mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-17 21:52:32 +00:00
Merge next lint fixes before updating the reviewed SDK PR
This commit is contained in:
@@ -23,5 +23,5 @@ jobs:
|
||||
# The codec and format code behind features that are off by default. Without these the
|
||||
# coverage build never compiles it, and new code there reads as untested.
|
||||
cargo-features: >-
|
||||
craft-video/dnx,craft-video/vp8,craft-video/prores,craft-video/h264,craft-video/hevc,craft-video/vp9,craft-video/av1,craft-video/threads,craft-audio/decode,craft-audio/encode,craft-audio/peaks,craft-audio/rayon,craft-audio/serde,craft-media/matroska,craft-media/ogg,craft-media/bitstream,craft-media/serde,craft-mesh/stl,craft-mesh/obj,craft-mesh/ply,craft-mesh/gltf,craft-metafile/write,craft-image-codecs/sgi
|
||||
craft-image-codecs/avif,craft-image-codecs/avif-decode,craft-video/dnx,craft-video/vp8,craft-video/prores,craft-video/h264,craft-video/hevc,craft-video/vp9,craft-video/av1,craft-video/threads,craft-audio/decode,craft-audio/encode,craft-audio/peaks,craft-audio/rayon,craft-audio/serde,craft-media/matroska,craft-media/ogg,craft-media/bitstream,craft-media/serde,craft-mesh/stl,craft-mesh/obj,craft-mesh/ply,craft-mesh/gltf,craft-metafile/write,craft-image-codecs/sgi,craft-image-codecs/sunraster,craft-jsonrpc/async-lines
|
||||
secrets: inherit
|
||||
|
||||
Generated
+85
-8
@@ -1152,9 +1152,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cfg-expr"
|
||||
version = "0.20.10"
|
||||
version = "0.20.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ba9e9ec16c447027685b1f897b720e18e9a8afd00bd7332c483537e38086c9f"
|
||||
checksum = "edd44d82755fb9210fb10d9daf8dd7e522da0efe1d5f263cffc5db07725663ff"
|
||||
dependencies = [
|
||||
"smallvec",
|
||||
"target-lexicon",
|
||||
@@ -1595,6 +1595,7 @@ dependencies = [
|
||||
"craft-dwg",
|
||||
"craft-dxf",
|
||||
"craft-ecproj-format",
|
||||
"craft-elf",
|
||||
"craft-eps",
|
||||
"craft-epub",
|
||||
"craft-ews",
|
||||
@@ -1611,6 +1612,7 @@ dependencies = [
|
||||
"craft-font-manager",
|
||||
"craft-font-store",
|
||||
"craft-font-subset",
|
||||
"craft-format",
|
||||
"craft-gainmap",
|
||||
"craft-gcode",
|
||||
"craft-gif",
|
||||
@@ -1637,11 +1639,13 @@ dependencies = [
|
||||
"craft-latex",
|
||||
"craft-layers",
|
||||
"craft-lottie",
|
||||
"craft-macho",
|
||||
"craft-markdown",
|
||||
"craft-materialx",
|
||||
"craft-math",
|
||||
"craft-mcmat",
|
||||
"craft-mcp",
|
||||
"craft-mcp-server",
|
||||
"craft-media",
|
||||
"craft-menu",
|
||||
"craft-mesh",
|
||||
@@ -1660,6 +1664,7 @@ dependencies = [
|
||||
"craft-pcraft-format",
|
||||
"craft-pdf 0.1.0",
|
||||
"craft-pdn",
|
||||
"craft-pe",
|
||||
"craft-penpot",
|
||||
"craft-pkcs7",
|
||||
"craft-platform",
|
||||
@@ -1687,11 +1692,13 @@ dependencies = [
|
||||
"craft-silhouette-studio",
|
||||
"craft-sketch",
|
||||
"craft-smime",
|
||||
"craft-sniff",
|
||||
"craft-stencilcraft-document",
|
||||
"craft-stencilcraft-format",
|
||||
"craft-stream-codecs",
|
||||
"craft-svg",
|
||||
"craft-table",
|
||||
"craft-tar",
|
||||
"craft-tcraft",
|
||||
"craft-text",
|
||||
"craft-texture",
|
||||
@@ -2164,6 +2171,15 @@ dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-elf"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-eps"
|
||||
version = "0.1.0"
|
||||
@@ -2383,6 +2399,16 @@ dependencies = [
|
||||
"craft-fonts-build",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-format"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"craft-zip",
|
||||
"proptest",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-gainmap"
|
||||
version = "0.1.0"
|
||||
@@ -2433,6 +2459,7 @@ name = "craft-heif"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"heic-rs",
|
||||
"proptest",
|
||||
"rayon",
|
||||
]
|
||||
|
||||
@@ -2669,6 +2696,15 @@ dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-macho"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-markdown"
|
||||
version = "0.1.0"
|
||||
@@ -2714,6 +2750,14 @@ dependencies = [
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-mcp-server"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"proptest",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-media"
|
||||
version = "0.1.0"
|
||||
@@ -2756,6 +2800,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"craft-opc",
|
||||
"craft-xml",
|
||||
"craft-zip",
|
||||
"miniz_oxide 0.9.1",
|
||||
"serde_json",
|
||||
]
|
||||
@@ -2982,6 +3027,15 @@ dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-pe"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-penpot"
|
||||
version = "0.1.0"
|
||||
@@ -3165,6 +3219,7 @@ name = "craft-ps"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"craft-eps",
|
||||
"flate2",
|
||||
"kurbo",
|
||||
"miniz_oxide 0.8.9",
|
||||
"pdf-writer",
|
||||
@@ -3233,6 +3288,13 @@ dependencies = [
|
||||
"zune-jpegxl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-render-parity"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"craft-cms",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-review"
|
||||
version = "0.1.0"
|
||||
@@ -3360,6 +3422,10 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-sniff"
|
||||
version = "0.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "craft-stencilcraft-document"
|
||||
version = "0.1.0"
|
||||
@@ -3432,6 +3498,13 @@ dependencies = [
|
||||
"egui",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-tar"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-tcraft"
|
||||
version = "0.1.0"
|
||||
@@ -3455,6 +3528,7 @@ dependencies = [
|
||||
"egui",
|
||||
"hypher",
|
||||
"icu_segmenter",
|
||||
"kurbo",
|
||||
"miniz_oxide 0.8.9",
|
||||
"regex",
|
||||
"regex-syntax",
|
||||
@@ -3841,6 +3915,7 @@ dependencies = [
|
||||
name = "craft-video"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"craft-jpeg",
|
||||
"craft-media",
|
||||
"craft-media-testkit",
|
||||
"rav1e",
|
||||
@@ -5173,9 +5248,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "fixedbitset"
|
||||
version = "0.5.7"
|
||||
version = "0.5.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
|
||||
checksum = "5905ec755cc8386d7928353e4d356d2be3e89e81e5ca01fe630a1ca4d08dc175"
|
||||
|
||||
[[package]]
|
||||
name = "flagset"
|
||||
@@ -7224,6 +7299,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"rayon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9269,6 +9345,7 @@ dependencies = [
|
||||
"loop9",
|
||||
"quick-error 2.0.1",
|
||||
"rav1e",
|
||||
"rayon",
|
||||
"rgb",
|
||||
]
|
||||
|
||||
@@ -9843,9 +9920,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f"
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.15"
|
||||
version = "0.103.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
|
||||
checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"ring",
|
||||
@@ -13433,9 +13510,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.1.0"
|
||||
version = "9.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2886b00003a72c4ccee3f42cfc34908d33b2508a74d827dc2a4244706475c704"
|
||||
checksum = "1156b39d142a34b999c2310a50455e0cb3760bf7efad0122f796d813bee26ccb"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
|
||||
+8
-2
@@ -17,6 +17,7 @@ craft-usd = { path = "crates/usd" }
|
||||
craft-smime = { path = "crates/smime" }
|
||||
craft-ber = { path = "crates/ber" }
|
||||
craft-proto = { path = "crates/proto" }
|
||||
craft-render-parity = { path = "crates/render-parity" }
|
||||
acadrust = "=0.6.6"
|
||||
log = "0.4"
|
||||
# internal: add each crate here as `craft-<name> = { path = "crates/<name>" }`
|
||||
@@ -28,6 +29,7 @@ craft-win32 = { path = "crates/win32" }
|
||||
craft-ai-core = { path = "crates/ai-core" }
|
||||
craft-anicraft-format = { path = "crates/anicraft-format" }
|
||||
craft-pdf = { path = "crates/pdf", version = "=0.1.0" }
|
||||
craft-pe = { path = "crates/pe" }
|
||||
craft-ps = { path = "crates/ps", version = "=0.1.0" }
|
||||
craft-pdn = { path = "crates/pdn" }
|
||||
craft-plate-test = { path = "crates/plate-test" }
|
||||
@@ -70,6 +72,7 @@ craft-designcraft-format = { path = "crates/designcraft-format" }
|
||||
craft-dxf = { path = "crates/dxf" }
|
||||
craft-drawingml = { path = "crates/drawingml" }
|
||||
craft-eps = { path = "crates/eps" }
|
||||
craft-elf = { path = "crates/elf" }
|
||||
craft-file-formats = { path = "crates/file-formats" }
|
||||
craft-file-import = { path = "crates/file-import" }
|
||||
vectorcraft-doc = { git = "https://github.com/storytold/vectorcraft.git", rev = "6b3a6b8788ebcb420d69449199cbdf323888d27e" }
|
||||
@@ -124,6 +127,7 @@ craft-layers = { path = "crates/layers" }
|
||||
craft-audio = { path = "crates/audio" }
|
||||
craft-latex = { path = "crates/latex" }
|
||||
craft-markdown = { path = "crates/markdown" }
|
||||
craft-macho = { path = "crates/macho" }
|
||||
craft-math = { path = "crates/math" }
|
||||
# No default features: craft-audio turns on the craft-media modules each of its features needs.
|
||||
craft-media = { path = "crates/media", default-features = false }
|
||||
@@ -204,6 +208,10 @@ craft-xcf = { path = "crates/xcf" }
|
||||
craft-xml = { path = "crates/xml" }
|
||||
craft-zip = { path = "crates/zip" }
|
||||
craft-y4m = { path = "crates/y4m" }
|
||||
craft-sniff = { path = "crates/sniff" }
|
||||
craft-tar = { path = "crates/tar" }
|
||||
craft-format = { path = "crates/format" }
|
||||
craft-mcp-server = { path = "crates/mcp-server" }
|
||||
craft-epub = { path = "crates/epub" }
|
||||
craft-ews = { path = "crates/ews" }
|
||||
craft-video = { path = "crates/video" }
|
||||
@@ -230,13 +238,11 @@ chrono = { version = "0.4.45", default-features = false, features = ["std"] }
|
||||
|
||||
[workspace.lints.rust]
|
||||
unsafe_code = "forbid"
|
||||
unused_workspace_dependencies = "allow"
|
||||
unexpected_cfgs = { level = "warn", check-cfg = ["cfg(craft_pdf_unicode_fixtures)"] }
|
||||
|
||||
[workspace.lints.clippy]
|
||||
cognitive_complexity = "deny"
|
||||
too_many_arguments = "deny"
|
||||
unused_workspace_dependencies = "allow"
|
||||
wildcard_imports = "deny"
|
||||
dbg_macro = "warn"
|
||||
todo = "warn"
|
||||
|
||||
@@ -61,6 +61,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
Clean room, from EffectCraft. Mapping onto an app's document stays in the app.
|
||||
|
||||
- [`craft-ai-core`](crates/ai-core/README.md): the data model and contract for AI in the Craft Apps (connections, models, tools, requests, jobs, consent, history, route spec). Pure, no I/O, wasm-safe. Contract in [docs/ai-shared-layer.md](docs/ai-shared-layer.md).
|
||||
- [`craft-pe`](crates/pe/src/lib.rs): PE/COFF headers (`.exe`, `.dll`, `.sys`, `.efi`) from untrusted bytes, from Craft Anything: the COFF header, PE32/PE32+ optional header with its data directories bounded by the header's real size, the section table, RVA-to-file mapping and the decoded Rich header.
|
||||
- [`craft-pdf`](crates/pdf/README.md): independently optional PDF parsing, interpretation, rasterization and guarded authoring. Apps retain layout, fonts, editing and export policy; writer-only consumers do not enable the shared reader.
|
||||
- [`craft-pdn`](crates/pdn/README.md): Paint.NET documents (`.pdn`, PDN3), read: the layer list, blend modes and chunked gzip pixel blocks, with a data-only MS-NRBF (.NET binary serialization) reader. From PhotoCraft.
|
||||
- [`craft-aaf`](crates/aaf/README.md): clean-room AAF (Edit Protocol, structured storage) and OMF Interchange 2.0 (Bento) readers and writers over one mob-style composition model with exact times, clip gain, markers, nested compositions and embedded PCM essence. From FilmCraft.
|
||||
@@ -117,6 +118,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
EMU units and `a:xfrm`, colours with theme slots and colour transforms, preset shape outlines
|
||||
(`a:prstGeom`), custom geometry with guide formulas (`a:custGeom`) and preset text warps
|
||||
(`a:prstTxWarp`). From DeckCraft. Takes `craft-xml` trees; packages stay with each format.
|
||||
- [`craft-elf`](crates/elf/src/lib.rs): ELF program headers and dynamic section entries from untrusted bytes, from Craft Anything: 32- or 64-bit in either byte order, bounded counts and overflow-checked offsets, virtual-address-to-file mapping through `PT_LOAD`, and segment, permission and dynamic tag names.
|
||||
- [`craft-eps`](crates/eps/README.md): Encapsulated PostScript files as a container: the binary
|
||||
preview header (read and write), the DSC header comments and bounding boxes (also `(atend)`),
|
||||
the EPSI preview bitmap, ASCII85 data, and telling EPS, PostScript and Illustrator files
|
||||
@@ -213,6 +215,9 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
Hasselblad 3FR/FFF, Minolta MRW, Epson ERF, Mamiya MEF, Kodak DCS, Pentax PEF, Samsung SRW, Canon CR2, Nikon NEF/NRW, Panasonic RW2, Fujifilm RAF, Olympus ORF (uncompressed), Sony ARW, Canon CR3 and Canon CRW (through
|
||||
`craft-crw`) so far; demosaicing and everything after stay with the develop
|
||||
pipeline.
|
||||
- [`craft-render-parity`](crates/render-parity/README.md): test support for the shared develop
|
||||
pipeline (#587): how far two renders of the same photo are apart (per-channel error, CIEDE2000,
|
||||
the worst pixel) and whether that is within a stated tolerance.
|
||||
- [`craft-tga`](crates/tga/src/lib.rs): Truevision Targa, from FilmCraft (reader) and PhotoCraft
|
||||
(writer): true-colour, grayscale and colour-mapped pictures, raw or run-length encoded, any
|
||||
origin, to RGBA8, and a run-length writer whose packets end with their scan line. Bounded on
|
||||
@@ -270,6 +275,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
AC-3/E-AC-3, MP1/MP2, MP3 encoding), streaming and seeking, and packet decoding for any demuxer. See its
|
||||
README for formats and features.
|
||||
- [`craft-latex`](crates/latex/README.md): LaTeX documents read (the everyday subset) and written (a standalone article), to and from the craft-flow model.
|
||||
- [`craft-macho`](crates/macho/src/lib.rs): Mach-O header pieces from untrusted bytes, from Craft Anything: universal (fat and fat64) slice tables checked against the file, CPU and load command names, the fixed size each load command needs before its fields are read, and the thread-state program counter.
|
||||
- [`craft-markdown`](crates/markdown/README.md): Markdown read and written (a CommonMark subset with GitHub tables and strikethrough), to and from the craft-flow model.
|
||||
- [`craft-math`](crates/math/README.md): equations as one tree (the Office Math object model) with its syntaxes, read and written: OMML, presentation MathML, LaTeX math and the UnicodeMath linear format.
|
||||
- [`craft-media`](crates/media/README.md): bit-level I/O (`bitstream`), clean-room MP4/MOV
|
||||
@@ -414,6 +420,8 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
|
||||
text-format crates read into and write from. WordCraft's model without its editing code.
|
||||
- [`craft-xlsx`](crates/xlsx/README.md): Excel workbooks (`.xlsx`, `.xlsm`, SpreadsheetML, transitional and strict) and binary workbooks (`.xlsb`) on `craft-opc`: the workbook part, shared strings, the styles tables and worksheet rows streamed one at a time, with limits on the package, the XML and the binary records.
|
||||
- [`craft-ora`](crates/ora/README.md): OpenRaster documents (`.ora`): the ZIP archive and the `stack.xml` layer tree as written, read with bounded sizes and written. From PhotoCraft.
|
||||
- [`craft-sniff`](crates/sniff/src/lib.rs): what a file is from its first bytes: one dependency-free, wasm-safe signature table naming the format, its media type, extension and family for images, audio, video, 3D models, archives, compressed streams, executables and databases. Identifies only; parsing stays with each format's reader.
|
||||
- [`craft-tar`](crates/tar/src/lib.rs): TAR (v7, ustar, GNU long names and base-256 sizes, local and global pax overrides) and cpio (newc, crc, odc) member walkers for hostile input, from Craft Anything: list each member, its kind, mode, link target and where its data lies, without extracting. Bounded by the input and a member cap; an optional `serde` feature serializes listings.
|
||||
- [`craft-y4m`](crates/y4m/src/lib.rs): YUV4MPEG2 (`.y4m`) reader, from FilmCraft: the stream
|
||||
header, colour space layouts at 8 to 16 bits, a frame index built from the `FRAME` lines without
|
||||
loading the file, and plane access. Bounded on hostile input.
|
||||
|
||||
+45
-2
@@ -58,13 +58,56 @@ pub struct WriteOptions<'a> {
|
||||
/// reported.
|
||||
pub fn write(doc: &Document, opts: &WriteOptions<'_>, report: &mut Report) -> Result<Vec<u8>> {
|
||||
let version = if opts.small_sectors { craft_cfb::Version::V3 } else { craft_cfb::Version::V4 };
|
||||
write::write(doc, version, &opts.identity, report)
|
||||
write::write(doc, version, &opts.identity, &ReferenceOptions::default(), report)
|
||||
}
|
||||
|
||||
/// Optional interpretation choices for applications retaining older interchange behavior.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct ReadOptions {
|
||||
/// Interpret a source clip's start using its target slot's edit rate.
|
||||
pub source_slot_rate: bool,
|
||||
/// Use the referenced mob name when the clip has no explicit name.
|
||||
pub infer_clip_name: bool,
|
||||
/// Decode percent escapes in relative network locators.
|
||||
pub decode_relative_locator: bool,
|
||||
/// Report selector and nested-scope flattening.
|
||||
pub report_flattening: bool,
|
||||
/// Accept truncated PCM WAVE essence summaries.
|
||||
pub permissive_pcm: bool,
|
||||
}
|
||||
impl Default for ReadOptions {
|
||||
fn default() -> Self {
|
||||
Self { source_slot_rate: true, infer_clip_name: true, decode_relative_locator: true, report_flattening: true, permissive_pcm: false }
|
||||
}
|
||||
}
|
||||
/// Read using explicitly selected interpretation choices.
|
||||
pub fn read_with_options(bytes: &[u8], options: &ReadOptions, report: &mut Report) -> Result<Document> {
|
||||
read::read(bytes, options, report)
|
||||
}
|
||||
|
||||
/// Optional compatibility choices for writing source references and locators.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct ReferenceOptions {
|
||||
/// Write source offsets in target-slot units.
|
||||
pub source_slot_rate: bool,
|
||||
/// Percent-encode relative locators.
|
||||
pub encode_relative_locator: bool,
|
||||
}
|
||||
impl Default for ReferenceOptions {
|
||||
fn default() -> Self {
|
||||
Self { source_slot_rate: true, encode_relative_locator: true }
|
||||
}
|
||||
}
|
||||
/// Write with explicitly selected reference and locator conventions.
|
||||
pub fn write_with_references(doc: &Document, opts: &WriteOptions<'_>, references: &ReferenceOptions, report: &mut Report) -> Result<Vec<u8>> {
|
||||
let version = if opts.small_sectors { craft_cfb::Version::V3 } else { craft_cfb::Version::V4 };
|
||||
write::write(doc, version, &opts.identity, references, report)
|
||||
}
|
||||
|
||||
/// Read an AAF file: its top-level compositions, the compositions nested in them and their
|
||||
/// sources. What cannot be represented is skipped and named in `report`.
|
||||
pub fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
|
||||
read::read(bytes, report)
|
||||
read::read(bytes, &ReadOptions::default(), report)
|
||||
}
|
||||
|
||||
/// Whether `bytes` look like an AAF file (a compound file whose root has the AAF root class or a
|
||||
|
||||
@@ -67,6 +67,7 @@ fn indirect_f64(d: &[u8]) -> Option<f64> {
|
||||
|
||||
struct R<'a> {
|
||||
report: &'a mut Report,
|
||||
options: super::ReadOptions,
|
||||
mobs: HashMap<MobId, &'a Obj>,
|
||||
essence: HashMap<MobId, &'a [u8]>,
|
||||
doc: Document,
|
||||
@@ -90,7 +91,7 @@ struct NestClip<'a> {
|
||||
/// Compositions nested deeper than this are read as gaps.
|
||||
const MAX_NESTING: usize = 16;
|
||||
|
||||
pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
|
||||
pub(crate) fn read(bytes: &[u8], options: &super::ReadOptions, report: &mut Report) -> Result<Document> {
|
||||
let root = store::read(bytes).map_err(|e| Error::Parse { format: "AAF", message: e })?;
|
||||
let header = root.strong(pid::ROOT_HEADER).ok_or_else(|| Error::Parse { format: "AAF", message: "no header object".into() })?;
|
||||
let content = header.strong(pid::CONTENT).ok_or_else(|| Error::Parse { format: "AAF", message: "no content storage".into() })?;
|
||||
@@ -106,7 +107,8 @@ pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
|
||||
essence.insert(id, d);
|
||||
}
|
||||
}
|
||||
let mut r = R { report, mobs, essence, doc: Document::default(), sources: HashMap::new(), nests: HashMap::new(), open: Vec::new() };
|
||||
let mut r =
|
||||
R { report, options: *options, mobs, essence, doc: Document::default(), sources: HashMap::new(), nests: HashMap::new(), open: Vec::new() };
|
||||
// Top-level compositions: those no other composition uses, among the ones tagged top-level
|
||||
// when any are. (Premiere Pro tags a nested sequence's composition top-level too; it is still
|
||||
// only a nested sequence.) Compositions that only use each other in a circle all count. The
|
||||
@@ -331,7 +333,9 @@ impl R<'_> {
|
||||
items
|
||||
}
|
||||
x if x == cls::SELECTOR => {
|
||||
self.report.info("a selector was flattened to its selected input");
|
||||
if self.options.report_flattening {
|
||||
self.report.info("a selector was flattened to its selected input");
|
||||
}
|
||||
match c.strong(pid::SELECTED) {
|
||||
Some(s) => {
|
||||
let mut items = self.segment(s, rate, kind, depth);
|
||||
@@ -342,7 +346,9 @@ impl R<'_> {
|
||||
}
|
||||
}
|
||||
x if x == cls::NESTED_SCOPE => {
|
||||
self.report.info("a nested scope was flattened to its final input");
|
||||
if self.options.report_flattening {
|
||||
self.report.info("a nested scope was flattened to its final input");
|
||||
}
|
||||
match c.objs(pid::NESTED_SLOTS).last() {
|
||||
Some(s) => {
|
||||
let mut items = self.segment(s, rate, kind, depth);
|
||||
@@ -388,7 +394,12 @@ impl R<'_> {
|
||||
let len = units_to_ticks(len_of(c), rate.0, rate.1);
|
||||
let tagged_name = tagged_values(c, pid::COMPONENT_USER_COMMENTS).into_iter().find(|(k, _)| k == "Clip Name").map(|(_, v)| v);
|
||||
let (Some(id), Some(slot)) = (c.mob_id(pid::SOURCE_ID), c.u32(pid::SOURCE_MOB_SLOT_ID)) else { return CItem::Filler(len) };
|
||||
let name = tagged_name.or_else(|| self.mobs.get(&id).and_then(|mob| mob.string(pid::MOB_NAME))).unwrap_or_default();
|
||||
let name = if self.options.infer_clip_name {
|
||||
tagged_name.or_else(|| self.mobs.get(&id).and_then(|mob| mob.string(pid::MOB_NAME)))
|
||||
} else {
|
||||
tagged_name
|
||||
}
|
||||
.unwrap_or_default();
|
||||
if id == [0; 32] {
|
||||
return CItem::Filler(len);
|
||||
}
|
||||
@@ -401,6 +412,7 @@ impl R<'_> {
|
||||
.and_then(|mob| mob.objs(pid::SLOTS).iter().find(|s| s.u32(pid::SLOT_ID) == Some(slot)))
|
||||
.and_then(|s| rate_of(s, pid::EDIT_RATE))
|
||||
.unwrap_or(rate);
|
||||
let source_rate = if self.options.source_slot_rate { source_rate } else { rate };
|
||||
let start = units_to_ticks(c.i64(pid::START_TIME).unwrap_or(0), source_rate.0, source_rate.1);
|
||||
match self.resolve(id, slot, start, kind, master_key, 0) {
|
||||
Some((source, t)) => CItem::Clip(CClip { len, source, start: t, gain: None, name }),
|
||||
@@ -506,6 +518,7 @@ impl R<'_> {
|
||||
.and_then(|m| m.objs(pid::SLOTS).iter().find(|s| s.u32(pid::SLOT_ID) == Some(next_slot)))
|
||||
.and_then(|s| rate_of(s, pid::EDIT_RATE))
|
||||
.unwrap_or(rate);
|
||||
let source_rate = if self.options.source_slot_rate { source_rate } else { rate };
|
||||
let inner = units_to_ticks(clip.i64(pid::START_TIME).unwrap_or(0), source_rate.0, source_rate.1);
|
||||
if next == [0; 32] {
|
||||
return None;
|
||||
@@ -545,7 +558,11 @@ impl R<'_> {
|
||||
DataKind::Sound => CKind::Sound,
|
||||
_ => kind,
|
||||
};
|
||||
let path = d.objs(pid::LOCATOR).iter().find_map(|l| l.string(pid::URL_STRING)).map(|u| locator_path(&u));
|
||||
let path = d
|
||||
.objs(pid::LOCATOR)
|
||||
.iter()
|
||||
.find_map(|l| l.string(pid::URL_STRING))
|
||||
.map(|u| if self.options.decode_relative_locator { locator_path(&u) } else { craft_timeline::path::file_url_to_path(&u) });
|
||||
let file_rate = rate_of(d, pid::SAMPLE_RATE).unwrap_or(rate);
|
||||
let length = d
|
||||
.i64(pid::FILE_LENGTH)
|
||||
@@ -604,7 +621,8 @@ impl R<'_> {
|
||||
if let Some(data) = self.essence.get(&id) {
|
||||
s.embedded = Some(data.to_vec());
|
||||
// embedded essence may be a whole WAVE / AIFF file
|
||||
if let Some((pcm, ch, sr, bits)) = crate::pcm::parse_wav(data) {
|
||||
let pcm = if self.options.permissive_pcm { craft_audio::essence_pcm::parse_wav(data) } else { crate::pcm::parse_wav(data) };
|
||||
if let Some((pcm, ch, sr, bits)) = pcm {
|
||||
s.embedded = Some(pcm);
|
||||
s.file_channels = ch as u32;
|
||||
s.channels = if s.channel.is_some() { 1 } else { ch as u32 };
|
||||
|
||||
@@ -557,3 +557,32 @@ fn relative_media_locator_round_trips_escaped_characters() {
|
||||
let got = read_clean(&bytes);
|
||||
assert!(got.sources.iter().any(|s| s.path.as_deref() == Some("Audio Files/100%20 #1.wav")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compatibility_options_keep_relative_paths_and_parent_rate() {
|
||||
let mut doc = crate::testdoc::sound_only(None);
|
||||
doc.compositions[0].sample_rate = 44_100;
|
||||
doc.sources[0].path = Some("Audio Files/100%20 #1.wav".into());
|
||||
let mut report = Report::default();
|
||||
let references = ReferenceOptions { source_slot_rate: false, encode_relative_locator: false };
|
||||
let bytes = write_with_references(&doc, &WriteOptions { identity: IDENTITY, small_sectors: false }, &references, &mut report).unwrap();
|
||||
let root = store::read(&bytes).unwrap();
|
||||
let mut all = Vec::new();
|
||||
walk(&root, &mut all);
|
||||
assert!(
|
||||
all.iter().filter(|o| o.class == cls::NETWORK_LOCATOR).any(|o| o.string(pid::URL_STRING).as_deref() == Some("Audio Files/100%20 #1.wav"))
|
||||
);
|
||||
let starts: Vec<i64> = all.iter().filter(|o| o.class == cls::SOURCE_CLIP).filter_map(|o| o.i64(pid::START_TIME)).collect();
|
||||
assert!(starts.contains(&17_640), "parent rate start: {starts:?}");
|
||||
let opts = ReadOptions {
|
||||
source_slot_rate: false,
|
||||
infer_clip_name: false,
|
||||
decode_relative_locator: false,
|
||||
report_flattening: false,
|
||||
permissive_pcm: true,
|
||||
};
|
||||
let back = read_with_options(&bytes, &opts, &mut report).unwrap();
|
||||
assert!(back.sources.iter().any(|s| s.path.as_deref() == Some("Audio Files/100%20 #1.wav")));
|
||||
let clip = back.compositions[0].tracks[0].items.iter().find_map(|i| if let CItem::Clip(c) = i { Some(c) } else { None }).unwrap();
|
||||
assert_eq!(clip.start, FrameRate::FPS_25.tick_of(10));
|
||||
}
|
||||
|
||||
@@ -75,6 +75,7 @@ fn definition(class: Auid, id: Auid, name: &str) -> Obj {
|
||||
struct W<'a> {
|
||||
report: &'a mut Report,
|
||||
inexact: bool,
|
||||
references: super::ReferenceOptions,
|
||||
}
|
||||
|
||||
struct IdGen {
|
||||
@@ -350,9 +351,15 @@ fn add_tape_mob(first: &Source, tape_id: [u8; 32], timecode: Option<(i64, FrameR
|
||||
}
|
||||
|
||||
/// Serialise a composition document.
|
||||
pub(crate) fn write(doc: &Document, version: craft_cfb::Version, identity: &Identity<'_>, report: &mut Report) -> Result<Vec<u8>> {
|
||||
pub(crate) fn write(
|
||||
doc: &Document,
|
||||
version: craft_cfb::Version,
|
||||
identity: &Identity<'_>,
|
||||
references: &super::ReferenceOptions,
|
||||
report: &mut Report,
|
||||
) -> Result<Vec<u8>> {
|
||||
let comp = doc.compositions.first().ok_or(Error::Empty)?;
|
||||
let mut w = W { report, inexact: false };
|
||||
let mut w = W { report, inexact: false, references: *references };
|
||||
let mut ids = IdGen::for_document(doc, comp);
|
||||
let MediaOutput { mut mobs, essence, master_ref } = write_media(doc, &mut w, &mut ids);
|
||||
let mut used_ops: Vec<Auid> = Vec::new();
|
||||
@@ -360,7 +367,8 @@ pub(crate) fn write(doc: &Document, version: craft_cfb::Version, identity: &Iden
|
||||
|
||||
// ---- nested compositions: a clip refers to one as it does to a master mob, through the
|
||||
// composition's first track of the clip's kind (a broken-out channel: that channel's track)
|
||||
let mut refs = Refs { mobs: master_ref, rates: doc.sources.iter().enumerate().map(|(i, s)| (i, source_rate(s))).collect() };
|
||||
let rates = if references.source_slot_rate { doc.sources.iter().enumerate().map(|(i, s)| (i, source_rate(s))).collect() } else { HashMap::new() };
|
||||
let mut refs = Refs { mobs: master_ref, rates };
|
||||
let nested_ids: Vec<[u8; 32]> = doc.nested.iter().map(|_| ids.next()).collect();
|
||||
for (si, s) in doc.sources.iter().enumerate() {
|
||||
let Some((comp, id)) = s.nested.and_then(|n| Some((doc.nested.get(n)?, nested_ids.get(n)?))) else { continue };
|
||||
@@ -795,8 +803,10 @@ fn descriptor(w: &mut W, s: &Source) -> Obj {
|
||||
if let Some(p) = &s.path {
|
||||
let url = if craft_timeline::path::is_absolute(p) {
|
||||
craft_timeline::path::path_to_file_url(p, false)
|
||||
} else {
|
||||
} else if w.references.encode_relative_locator {
|
||||
craft_timeline::path::encode_uri_path(p)
|
||||
} else {
|
||||
p.clone()
|
||||
};
|
||||
d.set(pid::LOCATOR, Value::StrongVec(vec![Obj::new(cls::NETWORK_LOCATOR).data_prop(pid::URL_STRING, utf16z(&url))]));
|
||||
}
|
||||
|
||||
@@ -30,7 +30,11 @@ mod roundtrip_tests;
|
||||
#[cfg(test)]
|
||||
mod testdoc;
|
||||
|
||||
/// Diagnostic types used by readers and writers.
|
||||
pub use craft_timeline::report;
|
||||
pub use craft_timeline::report::Report;
|
||||
/// Exact time types used by the composition model.
|
||||
pub use craft_timeline::time;
|
||||
|
||||
/// Error from a reader or writer. Unsupported *features* never error; they go in the [`Report`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
|
||||
+340
-10
@@ -12,6 +12,10 @@ type Uid = [u8; 12];
|
||||
|
||||
const MAX_REFERENCE_VISITS: usize = 65_536;
|
||||
|
||||
/// A source-clip search stopped at its work limit.
|
||||
#[derive(Clone, Copy)]
|
||||
struct SearchLimit;
|
||||
|
||||
struct R<'a, 'b> {
|
||||
c: &'b Container<'a>,
|
||||
report: &'b mut Report,
|
||||
@@ -21,15 +25,49 @@ struct R<'a, 'b> {
|
||||
media: HashMap<Uid, Vec<u8>>,
|
||||
doc: Document,
|
||||
sources: HashMap<(Uid, u32, String), usize>,
|
||||
/// Component visits left for [`R::segment`] over the whole document.
|
||||
budget: usize,
|
||||
/// Outcome of the source-clip search under each segment: deterministic, so it is made once.
|
||||
clip_search: HashMap<u32, std::result::Result<Option<u32>, SearchLimit>>,
|
||||
/// Visits and queued references left for source-clip searches over the whole document:
|
||||
/// distinct roots over one shared graph would otherwise each search it in full.
|
||||
search_budget: usize,
|
||||
}
|
||||
|
||||
/// Components one document may expand into timeline items; a small file can reference the same
|
||||
/// sequence from many places and would otherwise multiply without bound.
|
||||
const MAX_COMPONENT_VISITS: usize = 1_000_000;
|
||||
|
||||
/// Reported when the shared component budget runs out.
|
||||
const BUDGET_WARNING: &str = "OMF composition is too large to import completely (the remaining components are dropped)";
|
||||
/// Most visited components, and most queued references, one source-clip search may spend.
|
||||
const MAX_CLIP_SEARCH_VISITS: usize = 100_000;
|
||||
const MAX_CLIP_SEARCH_PENDING: usize = 1_000_000;
|
||||
/// Work all source-clip searches in one document may spend together.
|
||||
const MAX_DOCUMENT_SEARCH_WORK: usize = 2_000_000;
|
||||
|
||||
pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
|
||||
read_with_budget(bytes, report, MAX_COMPONENT_VISITS)
|
||||
}
|
||||
|
||||
fn read_with_budget(bytes: &[u8], report: &mut Report, budget: usize) -> Result<Document> {
|
||||
let perr = |m: String| Error::Parse { format: "OMF", message: m };
|
||||
let c = Container::open(bytes).map_err(perr)?;
|
||||
let class_of = |o: u32| c.get(o, "OMFI:OOBJ:ObjClass").and_then(|v| v.get(..4).map(|b| [b[0], b[1], b[2], b[3]]));
|
||||
let head = c.objects.keys().copied().find(|&o| class_of(o) == Some(*b"HEAD")).ok_or_else(|| perr("no OMF header object".into()))?;
|
||||
let be = c.get(head, "OMFI:HEAD:ByteOrder").is_none_or(|v| v.first() != Some(&0x49));
|
||||
let mut r = R { c: &c, report, be, mobs: HashMap::new(), media: HashMap::new(), doc: Document::default(), sources: HashMap::new() };
|
||||
let mut r = R {
|
||||
c: &c,
|
||||
report,
|
||||
be,
|
||||
mobs: HashMap::new(),
|
||||
media: HashMap::new(),
|
||||
doc: Document::default(),
|
||||
sources: HashMap::new(),
|
||||
budget,
|
||||
clip_search: HashMap::new(),
|
||||
search_budget: MAX_DOCUMENT_SEARCH_WORK,
|
||||
};
|
||||
let mob_list = r.refs(head, "OMFI:HEAD:Mobs");
|
||||
let mob_list = if mob_list.is_empty() {
|
||||
c.objects.keys().copied().filter(|&o| matches!(class_of(o).as_ref(), Some(b"CMOB" | b"MMOB" | b"SMOB"))).collect()
|
||||
@@ -66,8 +104,14 @@ pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
|
||||
cmobs.iter().copied().filter(|m| r.uid(*m, "OMFI:MOBJ:MobID").is_none_or(|id| !referenced.contains(&id))).collect()
|
||||
};
|
||||
for m in top {
|
||||
// every composition spends from the budget too: a list of up to 65,535 repeated entries must not yield that many
|
||||
if r.budget == 0 && !r.doc.compositions.is_empty() {
|
||||
r.report.warn(BUDGET_WARNING);
|
||||
break;
|
||||
}
|
||||
let comp = r.composition(m);
|
||||
r.doc.compositions.push(comp);
|
||||
r.budget = r.budget.saturating_sub(1);
|
||||
}
|
||||
if r.doc.compositions.is_empty() {
|
||||
return Err(Error::Empty);
|
||||
@@ -166,6 +210,13 @@ impl R<'_, '_> {
|
||||
let slots = self.refs(m, "OMFI:MOBJ:Slots");
|
||||
let mut na = 0;
|
||||
for s in slots {
|
||||
if self.budget == 0 {
|
||||
self.report.warn(BUDGET_WARNING);
|
||||
break;
|
||||
}
|
||||
// Every slot spends a unit, skipped ones too: a list of up to 65,535 repeated slots
|
||||
// must not cost that much work in each of 65,535 repeated compositions.
|
||||
self.budget -= 1;
|
||||
let Some(seg) = self.r#ref(s, "OMFI:MSLT:Segment") else { continue };
|
||||
let rate = self.rational(s, "OMFI:MSLT:EditRate").filter(|r| r.0 > 0 && r.1 > 0).unwrap_or((25, 1));
|
||||
let kind = self.data_kind(seg);
|
||||
@@ -213,13 +264,32 @@ impl R<'_, '_> {
|
||||
|
||||
fn segment(&mut self, seg: u32, rate: (i64, i64), depth: usize) -> Vec<CItem> {
|
||||
let len = units_to_ticks(self.len(seg), rate.0, rate.1);
|
||||
// Depth alone does not bound a graph that references the same component many times at
|
||||
// every level, so each visit also spends from a per-document budget.
|
||||
// A visit stopped by depth spends from it too.
|
||||
if self.budget == 0 {
|
||||
self.report.warn(BUDGET_WARNING);
|
||||
return vec![CItem::Filler(len)];
|
||||
}
|
||||
self.budget -= 1;
|
||||
if depth > 32 {
|
||||
return vec![CItem::Filler(len)];
|
||||
}
|
||||
match &self.class(seg) {
|
||||
b"SEQU" => {
|
||||
let mut out = Vec::new();
|
||||
let mut expanded = 0i64;
|
||||
for c in self.refs(seg, "OMFI:SEQU:Components") {
|
||||
if self.budget == 0 {
|
||||
// the rest of the sequence keeps its duration as one gap, not a gap per reference
|
||||
self.report.warn(BUDGET_WARNING);
|
||||
let rest = self.len(seg).saturating_sub(expanded);
|
||||
if rest > 0 {
|
||||
out.push(CItem::Filler(units_to_ticks(rest, rate.0, rate.1)));
|
||||
}
|
||||
break;
|
||||
}
|
||||
expanded = expanded.saturating_add(self.len(c));
|
||||
out.extend(self.segment(c, rate, depth + 1));
|
||||
}
|
||||
out
|
||||
@@ -306,12 +376,66 @@ impl R<'_, '_> {
|
||||
.find(|&s| self.r#ref(s, "OMFI:MSLT:TrackDesc").and_then(|t| self.u32(t, "OMFI:TRKD:TrackID")) == Some(track))
|
||||
}
|
||||
|
||||
fn first_clip(&self, seg: u32) -> Option<u32> {
|
||||
match &self.class(seg) {
|
||||
b"SCLP" => Some(seg),
|
||||
b"SEQU" => self.refs(seg, "OMFI:SEQU:Components").into_iter().find_map(|c| self.first_clip(c)),
|
||||
_ => None,
|
||||
/// First source clip under `seg`, depth-first; a sequence referencing itself (directly or
|
||||
/// through a cycle) ends the search rather than the stack. `Err` when the search spent its
|
||||
/// budget of visited components or queued references before finishing (the answer is unknown).
|
||||
fn first_clip(&mut self, seg: u32) -> std::result::Result<Option<u32>, SearchLimit> {
|
||||
if let Some(found) = self.clip_search.get(&seg) {
|
||||
return *found;
|
||||
}
|
||||
let mut spent = 0;
|
||||
let found = if self.search_budget == 0 {
|
||||
Err(SearchLimit)
|
||||
} else {
|
||||
let visits = MAX_CLIP_SEARCH_VISITS.min(self.search_budget);
|
||||
let pending = MAX_CLIP_SEARCH_PENDING.min(self.search_budget);
|
||||
self.first_clip_within(seg, visits, pending, &mut spent)
|
||||
};
|
||||
self.search_budget = self.search_budget.saturating_sub(spent);
|
||||
self.clip_search.insert(seg, found);
|
||||
found
|
||||
}
|
||||
|
||||
/// `spent` receives the visits plus queued references the search used.
|
||||
fn first_clip_within(&self, seg: u32, max_visits: usize, max_pending: usize, spent: &mut usize) -> std::result::Result<Option<u32>, SearchLimit> {
|
||||
let mut visited = HashSet::new();
|
||||
let mut todo = vec![seg];
|
||||
let mut queued = 1usize;
|
||||
let found = Self::search_from(self, &mut visited, &mut todo, &mut queued, max_visits, max_pending);
|
||||
*spent = visited.len().saturating_add(queued);
|
||||
found
|
||||
}
|
||||
|
||||
fn search_from(
|
||||
&self,
|
||||
visited: &mut HashSet<u32>,
|
||||
todo: &mut Vec<u32>,
|
||||
queued: &mut usize,
|
||||
max_visits: usize,
|
||||
max_pending: usize,
|
||||
) -> std::result::Result<Option<u32>, SearchLimit> {
|
||||
while let Some(seg) = todo.pop() {
|
||||
if !visited.insert(seg) {
|
||||
continue;
|
||||
}
|
||||
if visited.len() > max_visits {
|
||||
return Err(SearchLimit);
|
||||
}
|
||||
match &self.class(seg) {
|
||||
b"SCLP" => return Ok(Some(seg)),
|
||||
b"SEQU" => {
|
||||
let comps = self.refs(seg, "OMFI:SEQU:Components");
|
||||
// sequences may share one component list, so queued references grow with expansions, not visits
|
||||
*queued = queued.saturating_add(comps.len());
|
||||
if *queued > max_pending {
|
||||
return Err(SearchLimit);
|
||||
}
|
||||
todo.extend(comps.into_iter().rev());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn resolve(&mut self, id: Uid, track: u32, start: Tick, master: Option<String>, depth: usize) -> Option<(usize, Tick)> {
|
||||
@@ -328,7 +452,13 @@ impl R<'_, '_> {
|
||||
match &self.class(mob) {
|
||||
b"MMOB" => {
|
||||
let key = master.unwrap_or_else(|| id.iter().map(|b| format!("{b:02x}")).collect());
|
||||
let clip = self.first_clip(seg)?;
|
||||
let clip = match self.first_clip(seg) {
|
||||
Ok(c) => c?,
|
||||
Err(SearchLimit) => {
|
||||
self.report.warn("a master mob is too large to search for its source clip (imported as a gap)");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let inner = units_to_ticks(self.i32(clip, "OMFI:SCLP:StartTime").unwrap_or(0), rate.0, rate.1);
|
||||
let next = self.uid(clip, "OMFI:SCLP:SourceID")?;
|
||||
let next_track = self.u32(clip, "OMFI:SCLP:SourceTrackID")?;
|
||||
@@ -353,7 +483,9 @@ impl R<'_, '_> {
|
||||
let file_rate = sr.filter(|r| r.0 > 0 && r.1 > 0).unwrap_or(rate);
|
||||
let length = units_to_ticks(self.i32(desc, "OMFI:MDFL:Length").unwrap_or(0), file_rate.0, file_rate.1);
|
||||
let summary = self.c.get(desc, "OMFI:WAVD:Summary").or_else(|| self.c.get(desc, "OMFI:AIFD:Summary")).unwrap_or_default();
|
||||
let fmt = crate::pcm::parse_wav(&summary).or_else(|| crate::pcm::parse_aiff(&summary)).map(|(_, ch, sr, bits)| (ch as u32, sr, bits));
|
||||
let fmt = craft_audio::essence_pcm::parse_wav(&summary)
|
||||
.or_else(|| craft_audio::essence_pcm::parse_aiff(&summary))
|
||||
.map(|(_, ch, sr, bits)| (ch as u32, sr, bits));
|
||||
let mut path = None;
|
||||
for l in self.refs(desc, "OMFI:MDES:Locator") {
|
||||
let p = match &self.class(l) {
|
||||
@@ -373,7 +505,8 @@ impl R<'_, '_> {
|
||||
let (file_channels, sample_rate, bits) = fmt.unwrap_or((1, file_rate.0 as u32, 16));
|
||||
let channel = (file_channels > 1 && sound_slots > 1).then(|| track.saturating_sub(1).min(file_channels - 1));
|
||||
// timecode of the physical source
|
||||
let tc = self.first_clip(seg).and_then(|clip| {
|
||||
let first = self.first_clip(seg).ok().flatten();
|
||||
let tc = first.and_then(|clip| {
|
||||
let tid = self.uid(clip, "OMFI:SCLP:SourceID")?;
|
||||
let tape = *self.mobs.get(&tid)?;
|
||||
let (start_tc, fps, trate) = self.refs(tape, "OMFI:MOBJ:Slots").into_iter().find_map(|s| {
|
||||
@@ -415,7 +548,7 @@ impl R<'_, '_> {
|
||||
nested: None,
|
||||
};
|
||||
if let Some(d) = self.media.get(&id) {
|
||||
if let Some((pcm, ch, sr, b)) = crate::pcm::parse_wav(d).or_else(|| crate::pcm::parse_aiff(d)) {
|
||||
if let Some((pcm, ch, sr, b)) = craft_audio::essence_pcm::parse_wav(d).or_else(|| craft_audio::essence_pcm::parse_aiff(d)) {
|
||||
s.embedded = Some(pcm);
|
||||
s.file_channels = ch as u32;
|
||||
s.channels = if s.channel.is_some() { 1 } else { ch as u32 };
|
||||
@@ -437,3 +570,200 @@ impl R<'_, '_> {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::bento::Writer;
|
||||
use super::*;
|
||||
|
||||
fn obj(w: &mut Writer, class: &[u8; 4]) -> u32 {
|
||||
let id = w.new_object();
|
||||
w.set(id, "OMFI:OOBJ:ObjClass", "omfi:ClassID", class);
|
||||
id
|
||||
}
|
||||
fn list(w: &mut Writer, o: u32, p: &str, targets: &[u32]) {
|
||||
let mut v = (targets.len() as u16).to_be_bytes().to_vec();
|
||||
for t in targets {
|
||||
v.extend_from_slice(&t.to_be_bytes());
|
||||
}
|
||||
w.set(o, p, "omfi:ObjRefArray", &v);
|
||||
}
|
||||
|
||||
fn search(max_visits: usize, max_pending: usize, build: impl FnOnce(&mut Writer) -> u32) -> std::result::Result<Option<u32>, SearchLimit> {
|
||||
let mut w = Writer::new();
|
||||
let root = build(&mut w);
|
||||
let bytes = w.finish();
|
||||
let c = Container::open(&bytes).unwrap();
|
||||
let mut report = Report::default();
|
||||
let r = R {
|
||||
c: &c,
|
||||
report: &mut report,
|
||||
be: true,
|
||||
mobs: HashMap::new(),
|
||||
media: HashMap::new(),
|
||||
doc: Document::default(),
|
||||
sources: HashMap::new(),
|
||||
budget: 0,
|
||||
clip_search: HashMap::new(),
|
||||
search_budget: MAX_DOCUMENT_SEARCH_WORK,
|
||||
};
|
||||
r.first_clip_within(root, max_visits, max_pending, &mut 0)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clip_search_reports_its_limits() {
|
||||
// found within the limits
|
||||
assert!(matches!(
|
||||
search(10, 10, |w| {
|
||||
let (s, c) = (obj(w, b"SEQU"), obj(w, b"SCLP"));
|
||||
list(w, s, "OMFI:SEQU:Components", &[c]);
|
||||
s
|
||||
}),
|
||||
Ok(Some(_))
|
||||
));
|
||||
// too many distinct components before any clip
|
||||
assert!(
|
||||
search(5, 1000, |w| {
|
||||
let (s, f) = (obj(w, b"SEQU"), (0..20).map(|_| obj(w, b"FILL")).collect::<Vec<_>>());
|
||||
list(w, s, "OMFI:SEQU:Components", &f);
|
||||
s
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
// few visits, but sequences that each queue a long list of references
|
||||
assert!(
|
||||
search(1000, 500, |w| {
|
||||
let f = obj(w, b"FILL");
|
||||
let inner: Vec<u32> = (0..20).map(|_| obj(w, b"SEQU")).collect();
|
||||
for &i in &inner {
|
||||
list(w, i, "OMFI:SEQU:Components", &[f; 100]);
|
||||
}
|
||||
let s = obj(w, b"SEQU");
|
||||
list(w, s, "OMFI:SEQU:Components", &inner);
|
||||
s
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
// a search that ends without a clip is not a limit
|
||||
assert!(matches!(search(10, 10, |w| obj(w, b"FILL")), Ok(None)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skipped_slots_spend_the_budget() {
|
||||
let mut w = Writer::new();
|
||||
let head = obj(&mut w, b"HEAD");
|
||||
let cm = obj(&mut w, b"CMOB");
|
||||
let slot = obj(&mut w, b"MSLT");
|
||||
list(&mut w, cm, "OMFI:MOBJ:Slots", &[slot; 200]);
|
||||
list(&mut w, head, "OMFI:HEAD:Mobs", &[cm]);
|
||||
list(&mut w, head, "OMFI:HEAD:PrimaryMobs", &[cm]);
|
||||
let bytes = w.finish();
|
||||
let mut report = Report::default();
|
||||
read_with_budget(&bytes, &mut report, 50).unwrap();
|
||||
assert!(report.mentions(BUDGET_WARNING), "slots with no segment cost nothing before");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_source_clip_search_is_made_once_per_segment() {
|
||||
let mut w = Writer::new();
|
||||
let (s, c) = (obj(&mut w, b"SEQU"), obj(&mut w, b"SCLP"));
|
||||
list(&mut w, s, "OMFI:SEQU:Components", &[c]);
|
||||
let bytes = w.finish();
|
||||
let c = Container::open(&bytes).unwrap();
|
||||
let mut report = Report::default();
|
||||
let mut r = R {
|
||||
c: &c,
|
||||
report: &mut report,
|
||||
be: true,
|
||||
mobs: HashMap::new(),
|
||||
media: HashMap::new(),
|
||||
doc: Document::default(),
|
||||
sources: HashMap::new(),
|
||||
budget: 0,
|
||||
clip_search: HashMap::new(),
|
||||
search_budget: MAX_DOCUMENT_SEARCH_WORK,
|
||||
};
|
||||
let first = r.first_clip(s).ok();
|
||||
assert!(matches!(first, Some(Some(_))));
|
||||
assert_eq!(r.clip_search.len(), 1);
|
||||
assert_eq!(r.first_clip(s).ok(), first);
|
||||
assert_eq!(r.clip_search.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_slots_and_compositions_spend_the_budget() {
|
||||
let mut w = Writer::new();
|
||||
let head = obj(&mut w, b"HEAD");
|
||||
let cm = obj(&mut w, b"CMOB");
|
||||
let dd = obj(&mut w, b"DDEF");
|
||||
w.set(dd, "OMFI:DDEF:DataKindID", "omfi:UniqueName", b"omfi:data:Sound\0");
|
||||
let fill = obj(&mut w, b"FILL");
|
||||
w.set(fill, "OMFI:CPNT:DataKind", "omfi:ObjRef", &dd.to_be_bytes());
|
||||
let slot = obj(&mut w, b"MSLT");
|
||||
w.set(slot, "OMFI:MSLT:Segment", "omfi:ObjRef", &fill.to_be_bytes());
|
||||
list(&mut w, cm, "OMFI:MOBJ:Slots", &[slot; 200]);
|
||||
list(&mut w, head, "OMFI:HEAD:Mobs", &[cm]);
|
||||
list(&mut w, head, "OMFI:HEAD:PrimaryMobs", &[cm; 200]);
|
||||
let bytes = w.finish();
|
||||
let mut report = Report::default();
|
||||
let doc = read_with_budget(&bytes, &mut report, 50).unwrap();
|
||||
let tracks: usize = doc.compositions.iter().map(|c| c.tracks.len()).sum();
|
||||
assert!(tracks <= 50, "{tracks} tracks from a budget of 50");
|
||||
assert!(doc.compositions.len() <= 2, "{} compositions", doc.compositions.len());
|
||||
assert!(report.has_warnings());
|
||||
}
|
||||
|
||||
fn with_reader<T>(bytes: &[u8], budget: usize, search_budget: usize, f: impl FnOnce(&mut R) -> T) -> (T, Report) {
|
||||
let c = Container::open(bytes).unwrap();
|
||||
let mut report = Report::default();
|
||||
let mut r = R {
|
||||
c: &c,
|
||||
report: &mut report,
|
||||
be: true,
|
||||
mobs: HashMap::new(),
|
||||
media: HashMap::new(),
|
||||
doc: Document::default(),
|
||||
sources: HashMap::new(),
|
||||
budget,
|
||||
clip_search: HashMap::new(),
|
||||
search_budget,
|
||||
};
|
||||
let out = f(&mut r);
|
||||
(out, report)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn depth_limited_components_spend_the_budget() {
|
||||
// a chain 33 sequences deep whose last one repeats a depth-limited sequence many times
|
||||
let mut w = Writer::new();
|
||||
let deep = obj(&mut w, b"SEQU");
|
||||
let chain: Vec<u32> = (0..33).map(|_| obj(&mut w, b"SEQU")).collect();
|
||||
for pair in chain.windows(2) {
|
||||
list(&mut w, pair[0], "OMFI:SEQU:Components", &[pair[1]]);
|
||||
}
|
||||
list(&mut w, chain[32], "OMFI:SEQU:Components", &[deep; 1000]);
|
||||
let bytes = w.finish();
|
||||
let (items, report) = with_reader(&bytes, 100, MAX_DOCUMENT_SEARCH_WORK, |r| r.segment(chain[0], (48_000, 1), 0));
|
||||
assert!(items.len() <= 100, "{} items from a budget of 100", items.len());
|
||||
assert!(report.mentions(BUDGET_WARNING));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_clip_searches_share_a_document_budget() {
|
||||
// distinct roots over one shared clip-free graph
|
||||
let mut w = Writer::new();
|
||||
let fills: Vec<u32> = (0..50).map(|_| obj(&mut w, b"FILL")).collect();
|
||||
let shared = obj(&mut w, b"SEQU");
|
||||
list(&mut w, shared, "OMFI:SEQU:Components", &fills);
|
||||
let roots: Vec<u32> = (0..30).map(|_| obj(&mut w, b"SEQU")).collect();
|
||||
for &root in &roots {
|
||||
list(&mut w, root, "OMFI:SEQU:Components", &[shared]);
|
||||
}
|
||||
let bytes = w.finish();
|
||||
let (results, _) = with_reader(&bytes, 0, 500, |r| roots.iter().map(|&root| r.first_clip(root)).collect::<Vec<_>>());
|
||||
assert!(matches!(results[0], Ok(None)), "the first search fits");
|
||||
assert!(results.iter().any(|r| r.is_err()), "later searches run out of the shared budget");
|
||||
let (results, _) = with_reader(&bytes, 0, MAX_DOCUMENT_SEARCH_WORK, |r| roots.iter().map(|&root| r.first_clip(root)).collect::<Vec<_>>());
|
||||
assert!(results.iter().all(|r| matches!(r, Ok(None))));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,8 +4,3 @@
|
||||
pub(crate) fn parse_wav(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
|
||||
craft_audio::wav_pcm(b).map(|p| (p.data, p.channels, p.sample_rate, p.bits))
|
||||
}
|
||||
|
||||
/// The same for an AIFF / AIFF-C (`NONE`, `twos`, `sowt`) file.
|
||||
pub(crate) fn parse_aiff(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
|
||||
craft_audio::aiff_pcm(b).map(|p| (p.data, p.channels, p.sample_rate, p.bits))
|
||||
}
|
||||
|
||||
@@ -63,6 +63,7 @@ pub fn parse_aiff(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
|
||||
if b.len() < 12 || &b[..4] != b"FORM" || !(&b[8..12] == b"AIFF" || &b[8..12] == b"AIFC") {
|
||||
return None;
|
||||
}
|
||||
let aifc = &b[8..12] == b"AIFC";
|
||||
let mut at = 12;
|
||||
let mut comm = None;
|
||||
while at + 8 <= b.len() {
|
||||
@@ -73,7 +74,17 @@ pub fn parse_aiff(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
|
||||
let ch = u16::from_be_bytes([body[0], body[1]]);
|
||||
let bits = u16::from_be_bytes([body[6], body[7]]);
|
||||
let sr = from_extended(body[8..18].try_into().ok()?).round() as u32;
|
||||
let little = body.len() >= 22 && &body[18..22] == b"sowt";
|
||||
// AIFF has no compression code (trailing COMM bytes mean nothing); AIFF-C must name one
|
||||
// of the integer PCM codes, or the samples are not integer PCM at all
|
||||
let little = if aifc {
|
||||
match body.get(18..22)? {
|
||||
b"NONE" | b"twos" => false,
|
||||
b"sowt" => true,
|
||||
_ => return None,
|
||||
}
|
||||
} else {
|
||||
false
|
||||
};
|
||||
comm = Some((ch, sr, bits, little));
|
||||
} else if id == b"SSND" && body.len() >= 8 {
|
||||
let (ch, sr, bits, little) = comm?;
|
||||
@@ -144,4 +155,35 @@ mod tests {
|
||||
let _ = parse_aiff(&a[..cut.min(a.len())]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aifc_needs_an_integer_pcm_compression_code() {
|
||||
let file = |kind: &[u8; 4], code: &[u8]| {
|
||||
let mut comm = vec![0, 1, 0, 0, 0, 1, 0, 16];
|
||||
comm.extend_from_slice(&[0x40, 0x0E, 0xBB, 0x80, 0, 0, 0, 0, 0, 0]);
|
||||
comm.extend_from_slice(code);
|
||||
let mut body = kind.to_vec();
|
||||
body.extend_from_slice(b"COMM");
|
||||
body.extend_from_slice(&(comm.len() as u32).to_be_bytes());
|
||||
body.extend_from_slice(&comm);
|
||||
if comm.len() & 1 == 1 {
|
||||
body.push(0);
|
||||
}
|
||||
body.extend_from_slice(b"SSND");
|
||||
body.extend_from_slice(&10u32.to_be_bytes());
|
||||
body.extend_from_slice(&[0; 8]);
|
||||
body.extend_from_slice(&[0x12, 0x34]);
|
||||
let mut v = b"FORM".to_vec();
|
||||
v.extend_from_slice(&(body.len() as u32).to_be_bytes());
|
||||
v.extend_from_slice(&body);
|
||||
v
|
||||
};
|
||||
assert_eq!(parse_aiff(&file(b"AIFC", b"NONE")).map(|p| p.0), Some(vec![0x34, 0x12]));
|
||||
assert_eq!(parse_aiff(&file(b"AIFC", b"twos")).map(|p| p.0), Some(vec![0x34, 0x12]));
|
||||
assert_eq!(parse_aiff(&file(b"AIFC", b"sowt")).map(|p| p.0), Some(vec![0x12, 0x34]));
|
||||
assert!(parse_aiff(&file(b"AIFC", b"fl32")).is_none(), "float samples are not integer PCM");
|
||||
assert!(parse_aiff(&file(b"AIFC", b"")).is_none(), "AIFF-C without a compression code");
|
||||
// plain AIFF ignores trailing COMM bytes
|
||||
assert_eq!(parse_aiff(&file(b"AIFF", b"sowt")).map(|p| p.0), Some(vec![0x34, 0x12]));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,6 +79,8 @@ mod write;
|
||||
/// PCM WAVE and AIFF wrappers for interchange essence.
|
||||
#[cfg(feature = "pcm-encode")]
|
||||
pub mod essence_pcm;
|
||||
/// Streaming WAVE and AIFF headers and sample packing.
|
||||
pub mod pcm_export;
|
||||
/// Interleaved PCM packet decoding.
|
||||
pub mod pcm_interleaved;
|
||||
/// Range-backed PCM WAVE access and a 16-bit PCM writer.
|
||||
@@ -762,3 +764,21 @@ extern crate self as craft_audio;
|
||||
#[cfg(test)]
|
||||
#[path = "../tests/main.rs"]
|
||||
mod integration;
|
||||
|
||||
// An encoder refusing its settings is an encode error, not damaged input.
|
||||
#[cfg(all(test, feature = "aac-encode"))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn aac_encoder_settings_are_encode_errors() {
|
||||
let e = match aac::Encoder::new(aac::EncoderConfig::cbr(44_000, 2, 128_000)) {
|
||||
Err(e) => AudioError::from(e),
|
||||
Ok(_) => panic!("44 kHz is not an AAC rate"),
|
||||
};
|
||||
assert!(matches!(e, AudioError::Encode(_)), "{e}");
|
||||
}
|
||||
}
|
||||
|
||||
/// DVD and Blu-ray LPCM packet decoding.
|
||||
pub mod lpcm;
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
//! DVD-Video and Blu-ray LPCM packet grammar and sample decoding.
|
||||
|
||||
/// Blu-ray / AVCHD LPCM packet header (4 bytes before the samples).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct BlurayLpcm {
|
||||
/// Sample rate in hertz.
|
||||
pub sample_rate: u32,
|
||||
/// Number of decoded channels.
|
||||
pub channels: usize,
|
||||
/// Significant bits per sample.
|
||||
pub bits: u32,
|
||||
}
|
||||
|
||||
/// Channels of each channel_assignment code (mono, stereo, 3/0, 2/1, 3/1, 2/2, 3/2, 3/2+LFE,
|
||||
/// 3/4, 3/4+LFE).
|
||||
const BD_LPCM_CHANNELS: [usize; 12] = [0, 1, 0, 2, 3, 3, 4, 4, 5, 6, 7, 8];
|
||||
|
||||
/// Parse and validate the four-byte Blu-ray LPCM header.
|
||||
pub fn bluray_lpcm_header(p: &[u8]) -> Option<BlurayLpcm> {
|
||||
if p.len() < 4 {
|
||||
return None;
|
||||
}
|
||||
let channels = *BD_LPCM_CHANNELS.get((p[2] >> 4) as usize).filter(|&&c| c > 0)?;
|
||||
let sample_rate = match p[2] & 15 {
|
||||
1 => 48_000,
|
||||
4 => 96_000,
|
||||
5 => 192_000,
|
||||
_ => return None,
|
||||
};
|
||||
let bits = match p[3] >> 6 {
|
||||
1 => 16,
|
||||
2 => 20,
|
||||
3 => 24,
|
||||
_ => return None,
|
||||
};
|
||||
Some(BlurayLpcm { sample_rate, channels, bits })
|
||||
}
|
||||
|
||||
/// Decode one Blu-ray LPCM packet (big-endian; channels padded to an even count; 20-bit
|
||||
/// samples are stored in 24 bits).
|
||||
pub fn decode_bluray_lpcm(p: &[u8]) -> Option<Vec<Vec<f32>>> {
|
||||
let h = bluray_lpcm_header(p)?;
|
||||
let coded = (h.channels + 1) & !1;
|
||||
let bps = if h.bits == 16 { 2 } else { 3 };
|
||||
let data = &p[4..];
|
||||
let n = data.len() / (coded * bps);
|
||||
let mut out = vec![Vec::with_capacity(n); h.channels];
|
||||
for k in 0..n {
|
||||
for (c, o) in out.iter_mut().enumerate() {
|
||||
let at = (k * coded + c) * bps;
|
||||
let v = if bps == 2 {
|
||||
i16::from_be_bytes([data[at], data[at + 1]]) as f32 / 32768.0
|
||||
} else {
|
||||
(i32::from_be_bytes([data[at], data[at + 1], data[at + 2], 0]) >> 8) as f32 / 8_388_608.0
|
||||
};
|
||||
o.push(v);
|
||||
}
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Bytes per two sample frames of DVD LPCM. The caller must bound `channels` to the DVD channel range.
|
||||
pub fn dvd_lpcm_group_bytes(channels: usize, bits: u32) -> usize {
|
||||
match bits {
|
||||
16 => 4 * channels,
|
||||
20 => 5 * channels,
|
||||
_ => 6 * channels,
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode DVD-Video LPCM (big-endian). 16-bit samples are interleaved; 20- and 24-bit samples
|
||||
/// come in groups of two frames: the upper 16 bits of all the group's samples, then their low
|
||||
/// 4 / 8 bits. The caller must bound `channels` to the DVD channel range.
|
||||
pub fn decode_dvd_lpcm(data: &[u8], channels: usize, bits: u32) -> Vec<Vec<f32>> {
|
||||
let ch = channels.max(1);
|
||||
let group = dvd_lpcm_group_bytes(ch, bits);
|
||||
let groups = data.len() / group;
|
||||
let mut out = vec![Vec::with_capacity(groups * 2); ch];
|
||||
for g in 0..groups {
|
||||
let b = &data[g * group..(g + 1) * group];
|
||||
for f in 0..2 {
|
||||
for (c, o) in out.iter_mut().enumerate() {
|
||||
let k = f * ch + c;
|
||||
let hi = i16::from_be_bytes([b[2 * k], b[2 * k + 1]]) as i32;
|
||||
let v = match bits {
|
||||
16 => hi as f32 / 32768.0,
|
||||
20 => {
|
||||
let nib = b[4 * ch + k / 2];
|
||||
let lo = if k.is_multiple_of(2) { nib >> 4 } else { nib & 15 } as i32;
|
||||
((hi << 4) | lo) as f32 / 524_288.0
|
||||
}
|
||||
_ => ((hi << 8) | b[4 * ch + k] as i32) as f32 / 8_388_608.0,
|
||||
};
|
||||
o.push(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn bluray_header_and_even_channel_padding() {
|
||||
let packet = [0, 0, 0x31, 0x40, 0x40, 0, 0xc0, 0];
|
||||
assert_eq!(bluray_lpcm_header(&packet), Some(BlurayLpcm { sample_rate: 48_000, channels: 2, bits: 16 }));
|
||||
assert_eq!(decode_bluray_lpcm(&packet), Some(vec![vec![0.5], vec![-0.5]]));
|
||||
assert_eq!(decode_bluray_lpcm(&packet[..3]), None);
|
||||
}
|
||||
#[test]
|
||||
fn dvd_lpcm_16_20_and_24_bit_groups() {
|
||||
assert_eq!(decode_dvd_lpcm(&[0x40, 0, 0xc0, 0], 1, 16), vec![vec![0.5, -0.5]]);
|
||||
assert_eq!(decode_dvd_lpcm(&[0x40, 0, 0xc0, 0, 0], 1, 20), vec![vec![0.5, -0.5]]);
|
||||
assert_eq!(decode_dvd_lpcm(&[0x40, 0, 0xc0, 0, 0, 0], 1, 24), vec![vec![0.5, -0.5]]);
|
||||
assert_eq!(decode_dvd_lpcm(&[1, 2, 3], 1, 16), vec![Vec::<f32>::new()]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
//! Audio-only files: WAV (RIFF, little-endian PCM; RF64 past 4 GiB) and AIFF (big-endian PCM),
|
||||
//! 16, 24 or 32-bit integer or 32-bit float ([`SampleFormat`]), for streaming export.
|
||||
//! Float WAV is `WAVE_FORMAT_IEEE_FLOAT` with a `fact` chunk; float AIFF is AIFF-C (`fl32`).
|
||||
//!
|
||||
//! The header is written first from the known length (`header`), then the samples in chunks
|
||||
//! (`encode`) and the pad byte (`pad`), so an export streams to disk instead of building the
|
||||
//! file in memory.
|
||||
|
||||
/// PCM representation in an exported file.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum SampleFormat {
|
||||
/// 16-bit signed integer.
|
||||
Int16,
|
||||
/// 24-bit signed integer.
|
||||
Int24,
|
||||
/// 32-bit signed integer.
|
||||
Int32,
|
||||
/// 32-bit IEEE float.
|
||||
Float32,
|
||||
}
|
||||
|
||||
impl SampleFormat {
|
||||
/// Bits per sample.
|
||||
pub fn bits(self) -> u16 {
|
||||
match self {
|
||||
Self::Int16 => 16,
|
||||
Self::Int24 => 24,
|
||||
Self::Int32 | Self::Float32 => 32,
|
||||
}
|
||||
}
|
||||
/// Bytes per sample.
|
||||
pub fn bytes(self) -> usize {
|
||||
usize::from(self.bits() / 8)
|
||||
}
|
||||
/// Whether this is a floating point format.
|
||||
pub fn is_float(self) -> bool {
|
||||
self == Self::Float32
|
||||
}
|
||||
/// Clip and quantize one sample with symmetric integer full scale; NaN becomes silence.
|
||||
pub fn quantise(self, s: f32) -> i32 {
|
||||
let s = s.clamp(-1.0, 1.0);
|
||||
match self {
|
||||
Self::Int16 => (s * 32_767.0).round() as i32,
|
||||
Self::Int24 => (s * 8_388_607.0).round() as i32,
|
||||
Self::Int32 => (f64::from(s) * 2_147_483_647.0).round() as i32,
|
||||
Self::Float32 => 0,
|
||||
}
|
||||
}
|
||||
/// Sanitize a floating sample: NaN becomes silence, infinity becomes the largest finite value.
|
||||
pub fn float(s: f32) -> f32 {
|
||||
if s.is_nan() { 0.0 } else { s.clamp(f32::MIN, f32::MAX) }
|
||||
}
|
||||
}
|
||||
|
||||
/// The container of an audio-only export.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum PcmContainer {
|
||||
/// RIFF/WAVE or RF64.
|
||||
Wav,
|
||||
/// AIFF or AIFF-C.
|
||||
Aiff,
|
||||
}
|
||||
|
||||
/// Bytes of sample data for `frames` frames, or None when that does not fit 64 bits.
|
||||
fn data_len(frames: u64, channels: u16, format: SampleFormat) -> Option<u64> {
|
||||
frames.checked_mul(u64::from(channels))?.checked_mul(format.bytes() as u64)
|
||||
}
|
||||
|
||||
/// The pad byte after an odd-sized data chunk (RIFF and AIFF chunks are word-aligned).
|
||||
pub fn pad(frames: u64, channels: u16, format: SampleFormat) -> &'static [u8] {
|
||||
if data_len(frames, channels, format).is_some_and(|n| n & 1 == 1) { &[0] } else { &[] }
|
||||
}
|
||||
|
||||
/// `KSDATAFORMAT_SUBTYPE_PCM` / `_IEEE_FLOAT` after their first two bytes (the format tag).
|
||||
const SUBTYPE_TAIL: [u8; 14] = [0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71];
|
||||
|
||||
/// AIFF-C format version 1 (`FVER`).
|
||||
const AIFC_VERSION_1: u32 = 0xA280_5140;
|
||||
|
||||
/// The AIFF-C compression name of `fl32`, as a Pascal string of even length.
|
||||
const FL32_NAME: &[u8] = b"\x1532-bit floating point";
|
||||
|
||||
/// Everything before the samples of a file of `frames` frames. A WAV whose RIFF size would not
|
||||
/// fit 32 bits (4 GiB: 24-bit 5.1 at 96 kHz after about 41 minutes) is written as RF64 (EBU Tech
|
||||
/// 3306), with the 64-bit sizes in a `ds64` chunk. AIFF has no 64-bit form, so a longer AIFF is
|
||||
/// an error naming WAV instead.
|
||||
pub fn header(container: PcmContainer, channels: u16, sample_rate: u32, format: SampleFormat, frames: u64) -> Result<Vec<u8>, String> {
|
||||
let too_long = || "the audio is too long to write".to_string();
|
||||
if channels == 0 {
|
||||
return Err("audio needs at least one channel".into());
|
||||
}
|
||||
if sample_rate == 0 {
|
||||
return Err("audio needs a sample rate".into());
|
||||
}
|
||||
let bits = format.bits();
|
||||
let bps = bits / 8;
|
||||
let float = format.is_float();
|
||||
let data = data_len(frames, channels, format).ok_or_else(too_long)?;
|
||||
let block_align = channels.checked_mul(bps).ok_or("too many audio channels")?;
|
||||
match container {
|
||||
PcmContainer::Wav => {
|
||||
let byte_rate =
|
||||
u32::try_from(u64::from(sample_rate) * u64::from(block_align)).map_err(|_| "the audio sample rate is too high".to_string())?;
|
||||
let ext = channels > 2;
|
||||
// plain PCM has no cbSize; IEEE float has an empty one
|
||||
let fmt_len: u32 = if ext {
|
||||
40
|
||||
} else if float {
|
||||
18
|
||||
} else {
|
||||
16
|
||||
};
|
||||
// every format but integer PCM has a `fact` chunk (the frame count; RF64: in `ds64`)
|
||||
let fact_len: u64 = if float { 12 } else { 0 };
|
||||
let padded = data.checked_add(data & 1).ok_or_else(too_long)?;
|
||||
// "WAVE" + fmt chunk + fact chunk + data chunk header + samples
|
||||
let riff = (4 + 8 + u64::from(fmt_len) + fact_len + 8).checked_add(padded).ok_or_else(too_long)?;
|
||||
let rf64 = riff > u64::from(u32::MAX);
|
||||
let mut v = Vec::with_capacity(100);
|
||||
if rf64 {
|
||||
const DS64: u64 = 8 + 28;
|
||||
v.extend_from_slice(b"RF64");
|
||||
v.extend_from_slice(&u32::MAX.to_le_bytes());
|
||||
v.extend_from_slice(b"WAVEds64");
|
||||
v.extend_from_slice(&28u32.to_le_bytes());
|
||||
v.extend_from_slice(&riff.checked_add(DS64).ok_or_else(too_long)?.to_le_bytes());
|
||||
v.extend_from_slice(&data.to_le_bytes());
|
||||
v.extend_from_slice(&frames.to_le_bytes());
|
||||
v.extend_from_slice(&0u32.to_le_bytes()); // no table entries
|
||||
} else {
|
||||
v.extend_from_slice(b"RIFF");
|
||||
v.extend_from_slice(&(riff as u32).to_le_bytes());
|
||||
v.extend_from_slice(b"WAVE");
|
||||
}
|
||||
let tag: u16 = if float { 3 } else { 1 };
|
||||
v.extend_from_slice(b"fmt ");
|
||||
v.extend_from_slice(&fmt_len.to_le_bytes());
|
||||
v.extend_from_slice(&(if ext { 0xFFFE } else { tag }).to_le_bytes());
|
||||
v.extend_from_slice(&channels.to_le_bytes());
|
||||
v.extend_from_slice(&sample_rate.to_le_bytes());
|
||||
v.extend_from_slice(&byte_rate.to_le_bytes());
|
||||
v.extend_from_slice(&block_align.to_le_bytes());
|
||||
v.extend_from_slice(&bits.to_le_bytes());
|
||||
if ext {
|
||||
// cbSize, valid bits, channel mask, KSDATAFORMAT_SUBTYPE_PCM / _IEEE_FLOAT
|
||||
v.extend_from_slice(&22u16.to_le_bytes());
|
||||
v.extend_from_slice(&bits.to_le_bytes());
|
||||
let mask: u32 = if channels == 6 { 0x3F } else { 0 };
|
||||
v.extend_from_slice(&mask.to_le_bytes());
|
||||
v.extend_from_slice(&tag.to_le_bytes());
|
||||
v.extend_from_slice(&SUBTYPE_TAIL);
|
||||
} else if float {
|
||||
v.extend_from_slice(&0u16.to_le_bytes());
|
||||
}
|
||||
if float {
|
||||
v.extend_from_slice(b"fact");
|
||||
v.extend_from_slice(&4u32.to_le_bytes());
|
||||
v.extend_from_slice(&(if rf64 { u32::MAX } else { u32::try_from(frames).unwrap_or(u32::MAX) }).to_le_bytes());
|
||||
}
|
||||
v.extend_from_slice(b"data");
|
||||
v.extend_from_slice(&(if rf64 { u32::MAX } else { data as u32 }).to_le_bytes());
|
||||
Ok(v)
|
||||
}
|
||||
PcmContainer::Aiff => {
|
||||
let aiff_too_long = || "AIFF holds at most 4 GB of audio: export WAV, which has no such limit".to_string();
|
||||
let frames32 = u32::try_from(frames).map_err(|_| aiff_too_long())?;
|
||||
// SSND: offset + block size + samples
|
||||
let ssnd = u32::try_from(data.checked_add(8).ok_or_else(aiff_too_long)?).map_err(|_| aiff_too_long())?;
|
||||
// floats need AIFF-C: a version chunk, and the compression type and name in COMM
|
||||
let comm: u32 = if float { 18 + 4 + FL32_NAME.len() as u32 } else { 18 };
|
||||
let fver: u32 = if float { 8 + 4 } else { 0 };
|
||||
let form = (4u32 + fver + (8 + comm))
|
||||
.checked_add(8)
|
||||
.and_then(|n| n.checked_add(ssnd))
|
||||
.and_then(|n| n.checked_add(ssnd & 1))
|
||||
.ok_or_else(aiff_too_long)?;
|
||||
let mut v = Vec::with_capacity(90);
|
||||
v.extend_from_slice(b"FORM");
|
||||
v.extend_from_slice(&form.to_be_bytes());
|
||||
if float {
|
||||
v.extend_from_slice(b"AIFCFVER");
|
||||
v.extend_from_slice(&4u32.to_be_bytes());
|
||||
v.extend_from_slice(&AIFC_VERSION_1.to_be_bytes());
|
||||
} else {
|
||||
v.extend_from_slice(b"AIFF");
|
||||
}
|
||||
v.extend_from_slice(b"COMM");
|
||||
v.extend_from_slice(&comm.to_be_bytes());
|
||||
v.extend_from_slice(&channels.to_be_bytes());
|
||||
v.extend_from_slice(&frames32.to_be_bytes());
|
||||
v.extend_from_slice(&bits.to_be_bytes());
|
||||
v.extend_from_slice(&extended(sample_rate));
|
||||
if float {
|
||||
v.extend_from_slice(b"fl32");
|
||||
v.extend_from_slice(FL32_NAME);
|
||||
}
|
||||
v.extend_from_slice(b"SSND");
|
||||
v.extend_from_slice(&ssnd.to_be_bytes());
|
||||
v.extend_from_slice(&0u32.to_be_bytes()); // offset
|
||||
v.extend_from_slice(&0u32.to_be_bytes()); // block size
|
||||
Ok(v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Append interleaved samples in `format` (little-endian for WAV, big-endian for AIFF).
|
||||
pub fn encode(container: PcmContainer, interleaved: &[f32], format: SampleFormat, out: &mut Vec<u8>) {
|
||||
let bps = format.bytes();
|
||||
out.reserve(interleaved.len().saturating_mul(bps));
|
||||
for &s in interleaved {
|
||||
let q = if format.is_float() { SampleFormat::float(s).to_bits() as i32 } else { format.quantise(s) };
|
||||
match container {
|
||||
PcmContainer::Wav => out.extend_from_slice(q.to_le_bytes().get(..bps).unwrap_or_default()),
|
||||
PcmContainer::Aiff => out.extend_from_slice(q.to_be_bytes().get(4 - bps..).unwrap_or_default()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A whole file in memory (short audio; exports stream with [`header`] and [`encode`]).
|
||||
fn write_whole(container: PcmContainer, interleaved: &[f32], channels: u16, sample_rate: u32, format: SampleFormat) -> Result<Vec<u8>, String> {
|
||||
let per_frame = usize::from(channels.max(1));
|
||||
if !interleaved.len().is_multiple_of(per_frame) {
|
||||
return Err("the samples do not make whole frames".into());
|
||||
}
|
||||
let frames = (interleaved.len() / per_frame) as u64;
|
||||
let mut v = header(container, channels, sample_rate, format, frames)?;
|
||||
encode(container, interleaved, format, &mut v);
|
||||
v.extend_from_slice(pad(frames, channels, format));
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// A RIFF/WAVE file of interleaved samples. More than two channels are written as
|
||||
/// `WAVE_FORMAT_EXTENSIBLE` with the speaker mask of the layout (6 channels: L, R, C, LFE, Ls, Rs
|
||||
/// = `0x3F`).
|
||||
pub fn write_wav(interleaved: &[f32], channels: u16, sample_rate: u32, format: SampleFormat) -> Result<Vec<u8>, String> {
|
||||
write_whole(PcmContainer::Wav, interleaved, channels, sample_rate, format)
|
||||
}
|
||||
|
||||
/// IEEE 754 80-bit extended encoding of a sample rate (AIFF `COMM`).
|
||||
pub fn extended(rate: u32) -> [u8; 10] {
|
||||
let mut out = [0u8; 10];
|
||||
if rate == 0 {
|
||||
return out;
|
||||
}
|
||||
let shift = rate.leading_zeros();
|
||||
let mant = (rate as u64) << (32 + shift);
|
||||
let exp = 16383 + 31 - shift as u16;
|
||||
out[..2].copy_from_slice(&exp.to_be_bytes());
|
||||
out[2..].copy_from_slice(&mant.to_be_bytes());
|
||||
out
|
||||
}
|
||||
|
||||
/// An AIFF file (`FORM`/`AIFF`, `COMM` + `SSND`; AIFF-C for floats) of interleaved samples.
|
||||
pub fn write_aiff(interleaved: &[f32], channels: u16, sample_rate: u32, format: SampleFormat) -> Result<Vec<u8>, String> {
|
||||
write_whole(PcmContainer::Aiff, interleaved, channels, sample_rate, format)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn streaming_formats_preserve_samples_and_bounds() {
|
||||
let samples = [1.0, -1.0, f32::NAN];
|
||||
let wav = write_wav(&samples, 1, 48_000, SampleFormat::Int24).unwrap();
|
||||
assert_eq!(&wav[..4], b"RIFF");
|
||||
assert_eq!(&wav[44..53], &[255, 255, 127, 1, 0, 128, 0, 0, 0]);
|
||||
let h = header(PcmContainer::Wav, 6, 96_000, SampleFormat::Int24, 96_000 * 3600).unwrap();
|
||||
assert_eq!(&h[..4], b"RF64");
|
||||
assert!(header(PcmContainer::Aiff, 2, 48_000, SampleFormat::Int24, u64::MAX).is_err());
|
||||
let aiff = write_aiff(&[0.25, -3.0], 1, 44_100, SampleFormat::Float32).unwrap();
|
||||
assert_eq!(&aiff[8..12], b"AIFC");
|
||||
assert_eq!(&aiff[aiff.len() - 8..], [0.25f32.to_be_bytes(), (-3.0f32).to_be_bytes()].concat());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_channels_rates_and_partial_frames_are_refused() {
|
||||
for container in [PcmContainer::Wav, PcmContainer::Aiff] {
|
||||
assert!(header(container, 0, 48_000, SampleFormat::Int16, 1).is_err());
|
||||
assert!(header(container, 1, 0, SampleFormat::Int16, 1).is_err());
|
||||
assert!(header(container, 2, 48_000, SampleFormat::Int16, 0).is_ok(), "zero frames stay valid");
|
||||
}
|
||||
assert!(write_wav(&[0.5], 0, 48_000, SampleFormat::Int16).is_err());
|
||||
assert!(write_wav(&[0.1, 0.2, 0.3], 2, 48_000, SampleFormat::Int16).is_err());
|
||||
assert!(write_aiff(&[0.1, 0.2, 0.3], 2, 48_000, SampleFormat::Int16).is_err());
|
||||
assert!(write_aiff(&[0.1, 0.2], 2, 48_000, SampleFormat::Int16).is_ok());
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,7 @@ use super::model::Action;
|
||||
///
|
||||
/// A file PhotoCraft wrote with its folder occurrences (`set_rows`, and per action `set_slot`,
|
||||
/// `action_id`, `legacy_shortcut_owner`) loads too: those fields are read into
|
||||
/// [`FileOccurrences`] and [`ActionList::merge_file`](super::ActionList::merge_file) turns them
|
||||
/// `FileOccurrences` and [`ActionList::merge_file`](super::ActionList::merge_file) turns them
|
||||
/// into unique set and action names. Names are the list's only identity, so they are never
|
||||
/// written back.
|
||||
#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
|
||||
@@ -108,7 +108,7 @@ impl<'a> AgentsPage<'a> {
|
||||
let mut next = None;
|
||||
let mut field = Field::new(words.control_port.clone()).id_salt("port");
|
||||
if mcp_flag {
|
||||
field = field.help(Help::Text(words.port_hint.clone().into()));
|
||||
field = field.help(Help::Text(words.port_hint.clone()));
|
||||
}
|
||||
field.show(ui, |ui, _| {
|
||||
let number = NumberField::new(words.control_port.clone(), Bind::given(Some(port as f64)))
|
||||
@@ -140,7 +140,7 @@ impl<'a> AgentsPage<'a> {
|
||||
.map(|choice| {
|
||||
let label: Label = match choice {
|
||||
WhenAppClosed::Launch => words.launch.fill(&[("app", app)]).into(),
|
||||
WhenAppClosed::Headless => words.headless.clone().into(),
|
||||
WhenAppClosed::Headless => words.headless.clone(),
|
||||
WhenAppClosed::Error => words.ask.fill(&[("app", app)]).into(),
|
||||
};
|
||||
Entry::new(choice, label)
|
||||
|
||||
+35
-3
@@ -176,6 +176,13 @@ craft-tcraft = { workspace = true, optional = true }
|
||||
craft-texture = { workspace = true, optional = true }
|
||||
craft-usd = { workspace = true, optional = true }
|
||||
craft-xfl = { workspace = true, optional = true }
|
||||
craft-elf = { workspace = true, optional = true }
|
||||
craft-format = { workspace = true, optional = true }
|
||||
craft-macho = { workspace = true, optional = true }
|
||||
craft-mcp-server = { workspace = true, optional = true }
|
||||
craft-pe = { workspace = true, optional = true }
|
||||
craft-sniff = { workspace = true, optional = true }
|
||||
craft-tar = { workspace = true, optional = true }
|
||||
|
||||
[features]
|
||||
ui = [
|
||||
@@ -228,7 +235,6 @@ aep = ["dep:craft-aep"]
|
||||
affinity = ["dep:craft-affinity"]
|
||||
affinity-synth = ["affinity", "craft-affinity/synth"]
|
||||
|
||||
ai-agents = ["dep:craft-ai-agents"]
|
||||
ai-core = ["dep:craft-ai-core"]
|
||||
|
||||
anicraft-format = ["dep:craft-anicraft-format"]
|
||||
@@ -267,8 +273,7 @@ audio-symphonia = ["audio", "craft-audio/symphonia"]
|
||||
audio-vorbis-encode = ["audio", "craft-audio/vorbis-encode"]
|
||||
|
||||
automation = ["dep:craft-automation", "keymap"]
|
||||
# Agents in the open window: the launcher's switch, the connection file, the link host and the Agents page.
|
||||
automation-agents = ["automation", "craft-automation/agents", "ai-agents", "flags"]
|
||||
automation-agents = ["automation", "craft-automation/agents"]
|
||||
|
||||
ber = ["dep:craft-ber"]
|
||||
|
||||
@@ -323,6 +328,9 @@ dxf = ["dep:craft-dxf"]
|
||||
|
||||
ecproj-format = ["dep:craft-ecproj-format"]
|
||||
|
||||
elf = ["dep:craft-elf"]
|
||||
elf-serde = ["elf", "craft-elf/serde"]
|
||||
|
||||
eps = ["dep:craft-eps"]
|
||||
eps-paths = ["eps", "craft-eps/paths"]
|
||||
|
||||
@@ -377,6 +385,8 @@ font-store-bundled-index = ["font-store", "craft-font-store/bundled-index"]
|
||||
|
||||
font-subset = ["dep:craft-font-subset"] # native only: HarfBuzz's C++ builds with the target's C++ compiler
|
||||
|
||||
format = ["dep:craft-format"]
|
||||
|
||||
gainmap = ["dep:craft-gainmap"]
|
||||
|
||||
gcode = ["dep:craft-gcode"]
|
||||
@@ -412,6 +422,8 @@ image-codecs-avif-decode = ["image-codecs", "craft-image-codecs/avif-decode"]
|
||||
image-codecs-corpus = ["image-codecs", "craft-image-codecs/corpus"]
|
||||
image-codecs-heif = ["image-codecs", "craft-image-codecs/heif"]
|
||||
image-codecs-sgi = ["image-codecs", "craft-image-codecs/sgi"]
|
||||
image-codecs-sunraster = ["image-codecs", "craft-image-codecs/sunraster"]
|
||||
image-codecs-threads = ["image-codecs", "craft-image-codecs/threads"]
|
||||
|
||||
imaging = ["dep:craft-imaging", "color", "keymap"]
|
||||
|
||||
@@ -442,6 +454,9 @@ layers = ["dep:craft-layers"]
|
||||
|
||||
lottie = ["dep:craft-lottie"]
|
||||
|
||||
macho = ["dep:craft-macho"]
|
||||
macho-serde = ["macho", "craft-macho/serde"]
|
||||
|
||||
markdown = ["dep:craft-markdown"]
|
||||
|
||||
materialx = ["dep:craft-materialx"]
|
||||
@@ -452,6 +467,8 @@ mcmat = ["dep:craft-mcmat"]
|
||||
|
||||
mcp = ["dep:craft-mcp"]
|
||||
|
||||
mcp-server = ["dep:craft-mcp-server"]
|
||||
|
||||
media = ["dep:craft-media"]
|
||||
media-avi = ["media", "craft-media/avi"]
|
||||
media-bitstream = ["media", "craft-media/bitstream"]
|
||||
@@ -504,6 +521,7 @@ pcraft-bundle = ["dep:craft-pcraft-bundle"]
|
||||
pcraft-format = ["dep:craft-pcraft-format"]
|
||||
|
||||
pdf = ["dep:craft-pdf"]
|
||||
pdf-content = ["pdf", "craft-pdf/content"]
|
||||
pdf-cos = ["pdf", "craft-pdf/cos"]
|
||||
pdf-crypt = ["pdf", "craft-pdf/crypt"]
|
||||
pdf-filters = ["pdf", "craft-pdf/filters"]
|
||||
@@ -519,6 +537,9 @@ pdf-write-svg = ["pdf", "craft-pdf/write-svg"]
|
||||
|
||||
pdn = ["dep:craft-pdn"]
|
||||
|
||||
pe = ["dep:craft-pe"]
|
||||
pe-serde = ["pe", "craft-pe/serde"]
|
||||
|
||||
penpot = ["dep:craft-penpot"]
|
||||
|
||||
pkcs7 = ["dep:craft-pkcs7"]
|
||||
@@ -552,6 +573,7 @@ proto = ["dep:craft-proto"]
|
||||
|
||||
ps = ["dep:craft-ps"]
|
||||
ps-read = ["ps", "craft-ps/read"]
|
||||
ps-write = ["ps", "craft-ps/write"]
|
||||
|
||||
psd = ["dep:craft-psd"]
|
||||
psd-corpus = ["psd", "craft-psd/corpus"]
|
||||
@@ -575,6 +597,8 @@ raw-erf = ["raw", "craft-raw/erf"]
|
||||
raw-hasselblad = ["raw", "craft-raw/hasselblad"]
|
||||
raw-jxl = ["raw", "craft-raw/jxl"]
|
||||
raw-kodak = ["raw", "craft-raw/kodak"]
|
||||
raw-line-dng = ["raw", "craft-raw/line-dng"]
|
||||
raw-line-dng-threads = ["raw", "craft-raw/line-dng-threads"]
|
||||
raw-mef = ["raw", "craft-raw/mef"]
|
||||
raw-mrw = ["raw", "craft-raw/mrw"]
|
||||
raw-nef = ["raw", "craft-raw/nef"]
|
||||
@@ -608,6 +632,8 @@ sketch = ["dep:craft-sketch"]
|
||||
|
||||
smime = ["dep:craft-smime"]
|
||||
|
||||
sniff = ["dep:craft-sniff"]
|
||||
|
||||
stencilcraft-document = ["dep:craft-stencilcraft-document"]
|
||||
|
||||
stencilcraft-format = ["dep:craft-stencilcraft-format"]
|
||||
@@ -616,6 +642,7 @@ stream-codecs = ["dep:craft-stream-codecs"]
|
||||
|
||||
svg = ["dep:craft-svg"]
|
||||
svg-draw = ["svg", "craft-svg/draw"]
|
||||
svg-paths = ["svg", "craft-svg/paths"]
|
||||
svg-tree = ["svg", "craft-svg/tree"]
|
||||
svg-tree-core = ["svg", "craft-svg/tree-core"]
|
||||
svg-tree-render = ["svg", "craft-svg/tree-render"]
|
||||
@@ -625,6 +652,9 @@ svg-tree-text = ["svg", "craft-svg/tree-text"]
|
||||
|
||||
table = ["dep:craft-table", "keymap"]
|
||||
|
||||
tar = ["dep:craft-tar"]
|
||||
tar-serde = ["tar", "craft-tar/serde"]
|
||||
|
||||
tcraft = ["dep:craft-tcraft"]
|
||||
|
||||
text = ["dep:craft-text", "keymap"]
|
||||
@@ -673,6 +703,7 @@ text-oriya = ["text", "craft-text/oriya"]
|
||||
text-panjabi = ["text", "craft-text/panjabi"]
|
||||
text-polish = ["text", "craft-text/polish"]
|
||||
text-portuguese = ["text", "craft-text/portuguese"]
|
||||
text-positioned = ["text", "craft-text/positioned"]
|
||||
text-russian = ["text", "craft-text/russian"]
|
||||
text-sanskrit = ["text", "craft-text/sanskrit"]
|
||||
text-segment = ["text", "craft-text/segment"]
|
||||
@@ -737,6 +768,7 @@ video-h264-encode = ["video", "craft-video/h264-encode"]
|
||||
video-hevc = ["video", "craft-video/hevc"]
|
||||
video-hevc-decode = ["video", "craft-video/hevc-decode"]
|
||||
video-hevc-encode = ["video", "craft-video/hevc-encode"]
|
||||
video-mjpeg = ["video", "craft-video/mjpeg"]
|
||||
video-mpeg2 = ["video", "craft-video/mpeg2"]
|
||||
video-mpeg2-decode = ["video", "craft-video/mpeg2-decode"]
|
||||
video-mpeg2-encode = ["video", "craft-video/mpeg2-encode"]
|
||||
|
||||
@@ -26,7 +26,7 @@ The `[features]` table in `Cargo.toml` is written by `tests/features.rs` from th
|
||||
|
||||
Reached through other crates, not named in the facade: the six library crates (`craft-ui-contract` to `craft-ui-shell`), `craft-ui-macros` and `craft-behavior` through `craft-ui`, and `craft-win32` through `craft-window`.
|
||||
|
||||
Not in the facade, because they are tools or test harnesses: `craft-ui-gallery`, `story-render`, `feature-root` (the features local builds resolve with), `craft-ui-bench`, `craft-ui-test-kit`, `craft-fonts-build` (a build-script helper), `craft-design-lint`, `craft-plate-test`, `craft-media-testkit`.
|
||||
Not in the facade, because they are tools or test harnesses: `craft-ui-gallery`, `story-render`, `feature-root` (the features local builds resolve with), `craft-ui-bench`, `craft-ui-test-kit`, `craft-fonts-build` (a build-script helper), `craft-design-lint`, `craft-plate-test`, `craft-media-testkit`, `craft-render-parity`.
|
||||
|
||||
## Family dependencies
|
||||
|
||||
|
||||
@@ -79,6 +79,8 @@ pub use craft_dwg as dwg;
|
||||
pub use craft_dxf as dxf;
|
||||
#[cfg(feature = "ecproj-format")]
|
||||
pub use craft_ecproj_format as ecproj_format;
|
||||
#[cfg(feature = "elf")]
|
||||
pub use craft_elf as elf;
|
||||
#[cfg(feature = "eps")]
|
||||
pub use craft_eps as eps;
|
||||
#[cfg(feature = "epub")]
|
||||
@@ -111,6 +113,8 @@ pub use craft_font_manager as font_manager;
|
||||
pub use craft_font_store as font_store;
|
||||
#[cfg(feature = "font-subset")]
|
||||
pub use craft_font_subset as font_subset;
|
||||
#[cfg(feature = "format")]
|
||||
pub use craft_format as format;
|
||||
#[cfg(feature = "gainmap")]
|
||||
pub use craft_gainmap as gainmap;
|
||||
#[cfg(feature = "gcode")]
|
||||
@@ -161,6 +165,8 @@ pub use craft_latex as latex;
|
||||
pub use craft_layers as layers;
|
||||
#[cfg(feature = "lottie")]
|
||||
pub use craft_lottie as lottie;
|
||||
#[cfg(feature = "macho")]
|
||||
pub use craft_macho as macho;
|
||||
#[cfg(feature = "markdown")]
|
||||
pub use craft_markdown as markdown;
|
||||
#[cfg(feature = "materialx")]
|
||||
@@ -171,6 +177,8 @@ pub use craft_math as math;
|
||||
pub use craft_mcmat as mcmat;
|
||||
#[cfg(feature = "mcp")]
|
||||
pub use craft_mcp as mcp;
|
||||
#[cfg(feature = "mcp-server")]
|
||||
pub use craft_mcp_server as mcp_server;
|
||||
#[cfg(feature = "media")]
|
||||
pub use craft_media as media;
|
||||
#[cfg(feature = "menu")]
|
||||
@@ -207,6 +215,8 @@ pub use craft_pcraft_format as pcraft_format;
|
||||
pub use craft_pdf as pdf;
|
||||
#[cfg(feature = "pdn")]
|
||||
pub use craft_pdn as pdn;
|
||||
#[cfg(feature = "pe")]
|
||||
pub use craft_pe as pe;
|
||||
#[cfg(feature = "penpot")]
|
||||
pub use craft_penpot as penpot;
|
||||
#[cfg(feature = "pkcs7")]
|
||||
@@ -261,6 +271,8 @@ pub use craft_silhouette_studio as silhouette_studio;
|
||||
pub use craft_sketch as sketch;
|
||||
#[cfg(feature = "smime")]
|
||||
pub use craft_smime as smime;
|
||||
#[cfg(feature = "sniff")]
|
||||
pub use craft_sniff as sniff;
|
||||
#[cfg(feature = "stencilcraft-document")]
|
||||
pub use craft_stencilcraft_document as stencilcraft_document;
|
||||
#[cfg(feature = "stencilcraft-format")]
|
||||
@@ -271,6 +283,8 @@ pub use craft_stream_codecs as stream_codecs;
|
||||
pub use craft_svg as svg;
|
||||
#[cfg(feature = "table")]
|
||||
pub use craft_table as table;
|
||||
#[cfg(feature = "tar")]
|
||||
pub use craft_tar as tar;
|
||||
#[cfg(feature = "tcraft")]
|
||||
pub use craft_tcraft as tcraft;
|
||||
#[cfg(feature = "text")]
|
||||
|
||||
@@ -11,6 +11,7 @@ pub const NOT_IN_FACADE: &[(&str, &str)] = &[
|
||||
("fonts-build", "a build-script helper; apps call it from build.rs"),
|
||||
("media-testkit", "test support: ffmpeg discovery and fixture locations for the media crates"),
|
||||
("plate-test", "a test harness: reference PDFs and PDF structure checks"),
|
||||
("render-parity", "test support: compares two renders of a photo against a stated tolerance"),
|
||||
("story-render", "a tool: renders a component's stories to PNG files offscreen"),
|
||||
("ui-bench", "a tool: the release-mode benchmark of the UI library"),
|
||||
("ui-gallery", "a tool: the story gallery application"),
|
||||
|
||||
@@ -69,9 +69,18 @@ impl Writer<'_> {
|
||||
}
|
||||
|
||||
/// A shape's or text box's `wps:wsp`, at `off` in its group (or 0, 0). In a group it carries
|
||||
/// its own `wps:cNvPr` with drawing id `id`.
|
||||
pub(super) fn wsp(&mut self, w: &mut W, o: &InlineObject, id: Option<&str>, off: (f32, f32), rels: &mut PartRels, depth: usize) {
|
||||
let InlineObject::Shape { kind, w: sw, h: sh, fill, stroke, stroke_width, story, effects, float, freeform, extra } = o else { return };
|
||||
/// its own `wps:cNvPr` with drawing id `id`. Returns where in `w` its `w:txbxContent` was
|
||||
/// written, for a VML fallback to repeat.
|
||||
pub(super) fn wsp(
|
||||
&mut self,
|
||||
w: &mut W,
|
||||
o: &InlineObject,
|
||||
id: Option<&str>,
|
||||
off: (f32, f32),
|
||||
rels: &mut PartRels,
|
||||
depth: usize,
|
||||
) -> Option<std::ops::Range<usize>> {
|
||||
let InlineObject::Shape { kind, w: sw, h: sh, fill, stroke, stroke_width, story, effects, float, freeform, extra } = o else { return None };
|
||||
let geom = freeform.as_deref().filter(|_| *kind == ShapeKind::Freeform);
|
||||
w.open("wps:wsp", &[]);
|
||||
if let Some(id) = id {
|
||||
@@ -181,6 +190,7 @@ impl Writer<'_> {
|
||||
// its place in it instead.
|
||||
let part = (*story).and_then(|id| self.doc.parts.get(&id).filter(|p| p.kind == craft_wordmodel::PartKind::TextBox).map(|p| (id, p)));
|
||||
let link = part.and_then(|(id, _)| self.txbx_links.get(&id).copied());
|
||||
let mut content = None;
|
||||
if let Some((n, seq)) = link.filter(|(_, seq)| *seq > 0) {
|
||||
w.empty("wps:linkedTxbx", &[("id", &n.to_string()), ("seq", &seq.to_string())]);
|
||||
} else if let Some((id, part)) = part
|
||||
@@ -190,6 +200,7 @@ impl Writer<'_> {
|
||||
Some((n, _)) => w.open("wps:txbx", &[("id", &n.to_string())]),
|
||||
None => w.open("wps:txbx", &[]),
|
||||
}
|
||||
let start = w.s.len();
|
||||
w.open("w:txbxContent", &[]);
|
||||
if self.written_boxes.insert(id) {
|
||||
let blocks = part.blocks.clone();
|
||||
@@ -198,6 +209,7 @@ impl Writer<'_> {
|
||||
w.empty("w:p", &[]);
|
||||
}
|
||||
w.close("w:txbxContent");
|
||||
content = Some(start..w.s.len());
|
||||
w.close("wps:txbx");
|
||||
self.boxes.leave();
|
||||
}
|
||||
@@ -231,6 +243,7 @@ impl Writer<'_> {
|
||||
w.close("wps:bodyPr");
|
||||
}
|
||||
w.close("wps:wsp");
|
||||
content
|
||||
}
|
||||
|
||||
/// A `w:drawing` holding `graphic` (an `a:graphic`) in a frame of the object's size and place.
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use super::drawing::{DrawingInfo, shape_name, spin_attrs, write_fill, xfrm_spin_only};
|
||||
use super::field::{FieldData, field_run};
|
||||
use super::section::comment_reference_style;
|
||||
use super::{CharProps, Fill, InlineObject, NoteKind, PartRels, TextColor, W, Wrap, Writer, emu, n, rpr};
|
||||
use super::{Anchor, CharProps, Fill, Float, InlineObject, NoteKind, PartRels, ShapeKind, TextColor, W, Wrap, Writer, emu, n, rpr};
|
||||
|
||||
impl Writer<'_> {
|
||||
/// Write an inline object (`Bookmark`, `Comment`, `Equation`, `Field`, `Shape`, etc.).
|
||||
@@ -362,10 +362,24 @@ impl Writer<'_> {
|
||||
}
|
||||
|
||||
fn write_shape(&mut self, w: &mut W, o: &InlineObject, props: &CharProps, rels: &mut PartRels, depth: usize) {
|
||||
let InlineObject::Shape { kind, w: sw, h: sh, float, effects, freeform, extra, .. } = o else { return };
|
||||
let InlineObject::Shape { kind, w: sw, h: sh, float, effects, freeform, extra, story, .. } = o else { return };
|
||||
self.rev_open(w, props);
|
||||
w.open("w:r", &[]);
|
||||
self.rpr(w, props);
|
||||
// A Word 2010 shape (`wps`) with a VML form goes in `mc:AlternateContent`, as Word writes
|
||||
// it: a reader that doesn't know `wps` (Word 2007, the Office Compatibility Pack) takes the
|
||||
// VML fallback instead of refusing the file. A shape without one stays bare: guarded with
|
||||
// no fallback, that reader would open the file without the shape and drop it on its next
|
||||
// save, which is worse than refusing the file.
|
||||
// VML currently has no chain IDs or mso-next-textbox links. Keep every member of a
|
||||
// linked story bare, including its head: an unlinked fallback would silently lose
|
||||
// overflow text in older readers. Unlinked boxes still get the compatibility fallback.
|
||||
let linked = story.is_some_and(|id| self.txbx_links.contains_key(&id));
|
||||
let fallback = freeform.is_none() && vml_tag(*kind).is_some() && !linked;
|
||||
if fallback {
|
||||
w.open("mc:AlternateContent", &[]);
|
||||
w.open("mc:Choice", &[("Requires", "wps")]);
|
||||
}
|
||||
w.open("w:drawing", &[]);
|
||||
let docpr = self.shape_docpr(extra.id);
|
||||
let link = Self::drawing_link(props, rels);
|
||||
@@ -384,18 +398,164 @@ impl Writer<'_> {
|
||||
}
|
||||
let float = &float;
|
||||
self.drawing_open(w, DrawingInfo { float, size: (*sw, *sh), docpr: &docpr, name: &name, alt: o.alt_text(), link: link.as_deref() });
|
||||
// This shape's stacking order: floating objects in its text box move `self.z` on.
|
||||
let z = self.z;
|
||||
w.empty("wp:cNvGraphicFramePr", &[]);
|
||||
w.open("a:graphic", &[]);
|
||||
w.open("a:graphicData", &[("uri", "http://schemas.microsoft.com/office/word/2010/wordprocessingShape")]);
|
||||
self.wsp(w, o, None, (0.0, 0.0), rels, depth);
|
||||
let content = self.wsp(w, o, None, (0.0, 0.0), rels, depth);
|
||||
w.close("a:graphicData");
|
||||
w.close("a:graphic");
|
||||
w.close(if float.wrap == Wrap::Inline { "wp:inline" } else { "wp:anchor" });
|
||||
w.close("w:drawing");
|
||||
if fallback {
|
||||
w.close("mc:Choice");
|
||||
// The same text again: the fallback is a copy of the choice, not more of the story.
|
||||
let content = content.and_then(|r| w.s.get(r)).map(str::to_owned);
|
||||
w.open("mc:Fallback", &[]);
|
||||
self.vml_shape(w, o, float, z, content.as_deref());
|
||||
w.close("mc:Fallback");
|
||||
w.close("mc:AlternateContent");
|
||||
}
|
||||
w.close("w:r");
|
||||
self.rev_close(w, props);
|
||||
}
|
||||
|
||||
/// A shape as VML (`w:pict`), for readers older than Word 2010 shapes: its frame, place,
|
||||
/// wrapping, fill, outline and text box. Only for a kind [`vml_tag`] names.
|
||||
fn vml_shape(&self, w: &mut W, o: &InlineObject, float: &Float, z: u32, content: Option<&str>) {
|
||||
let InlineObject::Shape { kind, w: sw, h: sh, fill, stroke, stroke_width, story, extra, .. } = o else { return };
|
||||
let Some(tag) = vml_tag(*kind) else { return };
|
||||
let pt = |v: f32| format!("{:.2}pt", craft_wordmodel::geom::finite(v));
|
||||
let mut style = Vec::new();
|
||||
if float.wrap != Wrap::Inline {
|
||||
style.push("position:absolute".to_string());
|
||||
style.push(format!("margin-left:{}", pt(float.x)));
|
||||
style.push(format!("margin-top:{}", pt(float.y)));
|
||||
}
|
||||
style.push(format!("width:{}", pt(sw.max(0.0))));
|
||||
style.push(format!("height:{}", pt(sh.max(0.0))));
|
||||
let spin = float.spin();
|
||||
if spin.deg != 0.0 {
|
||||
style.push(format!("rotation:{}", spin.deg));
|
||||
}
|
||||
match (spin.flip_h, spin.flip_v) {
|
||||
(true, true) => style.push("flip:x y".into()),
|
||||
(true, false) => style.push("flip:x".into()),
|
||||
(false, true) => style.push("flip:y".into()),
|
||||
(false, false) => {}
|
||||
}
|
||||
if float.wrap != Wrap::Inline {
|
||||
// The stacking order the choice was given, as a signed 32-bit `z-index`. Behind the text
|
||||
// is negative, shifted down rather than negated, so a higher `z` still stacks higher.
|
||||
let z = i64::from(z.min(i32::MAX as u32));
|
||||
let z = if float.wrap == Wrap::BehindText { z - (i64::from(i32::MAX) + 1) } else { z };
|
||||
style.push(format!("z-index:{z}"));
|
||||
// The same clearances from the text as the choice's `distL`/`distR`/`distT`/`distB`.
|
||||
let d = pt(float.dist.clamp(0.0, 1584.0));
|
||||
style.push(format!("mso-wrap-distance-left:{d}"));
|
||||
style.push(format!("mso-wrap-distance-right:{d}"));
|
||||
style.push(format!("mso-wrap-distance-top:{}", pt(float.dist_top.clamp(0.0, 1584.0))));
|
||||
style.push(format!("mso-wrap-distance-bottom:{}", pt(float.dist_bottom.clamp(0.0, 1584.0))));
|
||||
if let Some(a) = float.h_align {
|
||||
style.push(format!("mso-position-horizontal:{}", a.ooxml(true)));
|
||||
}
|
||||
style.push(format!("mso-position-horizontal-relative:{}", vml_rel(float.h_rel, true)));
|
||||
if let Some(a) = float.v_align {
|
||||
style.push(format!("mso-position-vertical:{}", a.ooxml(false)));
|
||||
}
|
||||
style.push(format!("mso-position-vertical-relative:{}", vml_rel(float.v_rel, false)));
|
||||
}
|
||||
let body = (*story).and_then(|id| self.doc.parts.get(&id)).map(|p| p.body).unwrap_or_default();
|
||||
style.push(
|
||||
match body.anchor {
|
||||
craft_wordmodel::props::VAlign::Top => "v-text-anchor:top",
|
||||
craft_wordmodel::props::VAlign::Center => "v-text-anchor:middle",
|
||||
craft_wordmodel::props::VAlign::Bottom => "v-text-anchor:bottom",
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
if float.hidden {
|
||||
style.push("visibility:hidden".into());
|
||||
}
|
||||
let style = style.join(";");
|
||||
let mut attrs: Vec<(&str, String)> = vec![("style", style)];
|
||||
if *kind == ShapeKind::RoundedRectangle {
|
||||
// The same fraction of the shorter side: VML's in 1/65536ths, DrawingML's in 1/100000ths.
|
||||
let c = match extra.corner {
|
||||
Some(c) => c.min(craft_wordmodel::connector::MAX_CORNER) as f32 / 100_000.0,
|
||||
None => craft_wordmodel::connector::DEFAULT_CORNER,
|
||||
};
|
||||
attrs.push(("arcsize", format!("{}f", (c * 65_536.0).round() as u32)));
|
||||
}
|
||||
if float.wrap != Wrap::Inline {
|
||||
attrs.push(("o:allowincell", (if float.outside_cell { "f" } else { "t" }).into()));
|
||||
}
|
||||
let opacity = match fill.as_ref() {
|
||||
Some(f) => {
|
||||
attrs.push(("fillcolor", format!("#{}", f.main_color().hex())));
|
||||
match f {
|
||||
Fill::Solid { transparency, .. } if *transparency > 0.0 => Some(format!("{:.4}", (1.0 - transparency / 100.0).clamp(0.0, 1.0))),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
None => {
|
||||
attrs.push(("filled", "f".into()));
|
||||
None
|
||||
}
|
||||
};
|
||||
match stroke {
|
||||
Some(c) => {
|
||||
attrs.push(("strokecolor", format!("#{}", c.hex())));
|
||||
attrs.push(("strokeweight", pt(stroke_width.clamp(0.0, 100.0))));
|
||||
}
|
||||
None => attrs.push(("stroked", "f".into())),
|
||||
}
|
||||
let attrs: Vec<(&str, &str)> = attrs.iter().map(|(k, v)| (*k, v.as_str())).collect();
|
||||
w.open("w:pict", &[]);
|
||||
w.open(tag, &attrs);
|
||||
if let Some(opacity) = &opacity {
|
||||
w.empty("v:fill", &[("opacity", opacity)]);
|
||||
}
|
||||
if let Some(content) = content {
|
||||
let inset = body.inset.map(|i| i.map(|v| pt(v.min(craft_wordmodel::MAX_TEXT_BOX_INSET) as f32 / 12_700.0)).join(","));
|
||||
let mut tb: Vec<(&str, &str)> = Vec::new();
|
||||
if let Some(inset) = &inset {
|
||||
tb.push(("inset", inset));
|
||||
}
|
||||
// Text direction as the VML reader takes it (stacked letters have no VML form).
|
||||
let mut tb_style = match body.vert {
|
||||
craft_wordmodel::props::TextVert::Vert => vec!["layout-flow:vertical"],
|
||||
craft_wordmodel::props::TextVert::Vert270 => vec!["layout-flow:vertical", "mso-layout-flow-alt:bottom-to-top"],
|
||||
craft_wordmodel::props::TextVert::Horz | craft_wordmodel::props::TextVert::Stacked => Vec::new(),
|
||||
};
|
||||
if body.auto_fit {
|
||||
tb_style.push("mso-fit-shape-to-text:t");
|
||||
}
|
||||
let tb_style = tb_style.join(";");
|
||||
if !tb_style.is_empty() {
|
||||
tb.push(("style", &tb_style));
|
||||
}
|
||||
w.open("v:textbox", &tb);
|
||||
w.raw(content);
|
||||
w.close("v:textbox");
|
||||
}
|
||||
let wrap = match float.wrap {
|
||||
Wrap::Square => Some("square"),
|
||||
Wrap::Tight => Some("tight"),
|
||||
Wrap::Through => Some("through"),
|
||||
Wrap::TopAndBottom => Some("topAndBottom"),
|
||||
Wrap::Inline | Wrap::BehindText | Wrap::InFrontOfText => None,
|
||||
};
|
||||
if let Some(wrap) = wrap {
|
||||
let side = float.wrap_text.ooxml();
|
||||
let attrs: &[(&str, &str)] = if float.wrap_text.is_both() { &[("type", wrap)] } else { &[("type", wrap), ("side", side)] };
|
||||
w.empty("w10:wrap", attrs);
|
||||
}
|
||||
w.close(tag);
|
||||
w.close("w:pict");
|
||||
}
|
||||
|
||||
fn write_group(&mut self, w: &mut W, o: &InlineObject, props: &CharProps, rels: &mut PartRels, depth: usize) {
|
||||
let InlineObject::Group { w: gw, h: gh, float, ch_w, ch_h, children, canvas } = o else { return };
|
||||
self.rev_open(w, props);
|
||||
@@ -487,6 +647,33 @@ impl Writer<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
/// The VML element that draws a shape kind, for the kinds VML has one for.
|
||||
fn vml_tag(kind: ShapeKind) -> Option<&'static str> {
|
||||
match kind {
|
||||
ShapeKind::Rectangle | ShapeKind::TextBox => Some("v:rect"),
|
||||
ShapeKind::RoundedRectangle => Some("v:roundrect"),
|
||||
ShapeKind::Ellipse => Some("v:oval"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// A floating object's anchor as VML names it (`mso-position-horizontal-relative`, `-vertical-`).
|
||||
fn vml_rel(a: Anchor, horiz: bool) -> &'static str {
|
||||
match (a, horiz) {
|
||||
(Anchor::Margin, _) => "margin",
|
||||
(Anchor::Page, _) => "page",
|
||||
(Anchor::InsideMargin, _) => "inner-margin-area",
|
||||
(Anchor::OutsideMargin, _) => "outer-margin-area",
|
||||
(Anchor::LeftMargin, true) => "left-margin-area",
|
||||
(Anchor::RightMargin, true) => "right-margin-area",
|
||||
(Anchor::Character, true) => "char",
|
||||
(Anchor::TopMargin, false) => "top-margin-area",
|
||||
(Anchor::BottomMargin, false) => "bottom-margin-area",
|
||||
(Anchor::Line, false) => "line",
|
||||
_ => "text",
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn ink_name(tool: craft_wordmodel::freeform::InkTool) -> &'static str {
|
||||
match tool {
|
||||
craft_wordmodel::freeform::InkTool::Pen => "Pen",
|
||||
|
||||
@@ -49,3 +49,4 @@ mod undefined_style;
|
||||
mod vml_shapes;
|
||||
mod word_xml;
|
||||
mod wordml2003_write;
|
||||
mod wps_fallback;
|
||||
|
||||
@@ -956,7 +956,8 @@ fn toc_field_skips_nested_stories() {
|
||||
let mut xml = String::new();
|
||||
std::io::Read::read_to_string(&mut zip.by_name("word/document.xml").unwrap(), &mut xml).unwrap();
|
||||
let boxes: Vec<(usize, usize)> = xml.match_indices("<w:txbxContent>").map(|(a, _)| (a, a + xml[a..].find("</w:txbxContent>").unwrap())).collect();
|
||||
assert_eq!(boxes.len(), 2, "{xml}");
|
||||
// Each box twice: its shape and its VML fallback.
|
||||
assert_eq!(boxes.len(), 4, "{xml}");
|
||||
let in_box = |i: usize| boxes.iter().any(|(a, b)| (*a..*b).contains(&i));
|
||||
for kind in ["begin", "separate", "end"] {
|
||||
let at: Vec<usize> = xml.match_indices(&format!(r#"fldCharType="{kind}""#)).map(|(i, _)| i).collect();
|
||||
@@ -964,7 +965,7 @@ fn toc_field_skips_nested_stories() {
|
||||
assert!(!in_box(at[0]), "field {kind} written inside a text box: {xml}");
|
||||
}
|
||||
let (begin, end) = (xml.find(r#"fldCharType="begin""#).unwrap(), xml.find(r#"fldCharType="end""#).unwrap());
|
||||
assert!(begin > boxes[0].1 && end > boxes[1].1 && end < xml.find("</w:sdt>").unwrap(), "{xml}");
|
||||
assert!(begin > boxes[1].1 && end > boxes[3].1 && end < xml.find("</w:sdt>").unwrap(), "{xml}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -257,5 +257,6 @@ fn story_shared_by_many_shapes_is_written_once() {
|
||||
}
|
||||
let xml = document_xml(&craft_docx::write(&d).unwrap());
|
||||
assert!(xml.len() < 2_000_000, "document.xml is {} bytes", xml.len());
|
||||
assert_eq!(xml.matches(text.as_str()).count(), 1);
|
||||
// Once in the first shape and once in its VML fallback; the other shapes don't repeat it.
|
||||
assert_eq!(xml.matches(text.as_str()).count(), 2);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
//! Word 2010 shapes (`wps:wsp`) are written inside `mc:AlternateContent` with a VML fallback
|
||||
//! (ECMA-376 Part 3, Markup Compatibility; Part 4 §14.1, VML), so a reader that doesn't know
|
||||
//! `wps` — Word 2007, or Word 2000-2003 with the Office Compatibility Pack — takes the fallback
|
||||
//! instead of refusing the whole file. A shape with no VML form is left bare.
|
||||
|
||||
use std::io::Read;
|
||||
|
||||
use craft_wordmodel::para::{Anchor, FloatAlign, ShapeKind, Wrap};
|
||||
use craft_wordmodel::props::TextVert;
|
||||
use craft_wordmodel::{Document, InlineObject, PartKind, TextBody, para_block};
|
||||
|
||||
fn document_xml(bytes: &[u8]) -> String {
|
||||
let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap();
|
||||
let mut s = String::new();
|
||||
z.by_name("word/document.xml").unwrap().read_to_string(&mut s).unwrap();
|
||||
s
|
||||
}
|
||||
|
||||
fn shape(kind: ShapeKind, story: Option<u32>) -> InlineObject {
|
||||
let float = craft_wordmodel::para::Float {
|
||||
wrap: Wrap::Square,
|
||||
h_rel: Anchor::Margin,
|
||||
h_align: Some(FloatAlign::End),
|
||||
v_rel: Anchor::Paragraph,
|
||||
y: 0.05,
|
||||
..Default::default()
|
||||
};
|
||||
InlineObject::Shape {
|
||||
kind,
|
||||
w: 10.7,
|
||||
h: 12.2,
|
||||
fill: None,
|
||||
stroke: None,
|
||||
stroke_width: 0.75,
|
||||
float,
|
||||
story,
|
||||
effects: Default::default(),
|
||||
freeform: None,
|
||||
extra: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// A page number in a floating text box, as in a footer: the shape that made the
|
||||
/// Compatibility Pack refuse the file.
|
||||
#[test]
|
||||
fn a_text_box_gets_a_vml_fallback_with_its_text() {
|
||||
let mut d = Document::from_text("a");
|
||||
let id = d.add_part(PartKind::TextBox, vec![para_block(craft_wordmodel::Paragraph::with_text("29", Default::default()))]);
|
||||
d.parts.get_mut(&id).unwrap().body = TextBody { auto_fit: true, inset: Some([0; 4]), ..Default::default() };
|
||||
let mut o = shape(ShapeKind::TextBox, Some(id));
|
||||
if let InlineObject::Shape { float, .. } = &mut o {
|
||||
(float.dist, float.dist_top, float.dist_bottom) = (18.0, 12.0, 6.0);
|
||||
}
|
||||
d.insert_object(&craft_wordmodel::Pos::body(0, 1), o, &Default::default()).unwrap();
|
||||
let bytes = craft_docx::write(&d).unwrap();
|
||||
let xml = document_xml(&bytes);
|
||||
assert!(xml.contains(r#"<mc:AlternateContent><mc:Choice Requires="wps"><w:drawing>"#), "{xml}");
|
||||
assert!(!xml.contains("<w:r><w:drawing>"), "no bare Word 2010 shape: {xml}");
|
||||
let fallback = &xml[xml.find("<mc:Fallback>").expect("a fallback")..];
|
||||
assert!(fallback.starts_with("<mc:Fallback><w:pict><v:rect "), "{fallback}");
|
||||
for want in [
|
||||
"position:absolute",
|
||||
"mso-position-horizontal:right",
|
||||
"mso-position-horizontal-relative:margin",
|
||||
"mso-position-vertical-relative:text",
|
||||
"mso-wrap-distance-left:18.00pt;mso-wrap-distance-right:18.00pt;mso-wrap-distance-top:12.00pt;mso-wrap-distance-bottom:6.00pt",
|
||||
r#"filled="f""#,
|
||||
r#"stroked="f""#,
|
||||
r#"<v:textbox inset="0.00pt,0.00pt,0.00pt,0.00pt" style="mso-fit-shape-to-text:t"><w:txbxContent>"#,
|
||||
r#"<w10:wrap type="square"/></v:rect></w:pict></mc:Fallback></mc:AlternateContent>"#,
|
||||
] {
|
||||
assert!(fallback.contains(want), "{want} in {fallback}");
|
||||
}
|
||||
assert_eq!(xml.matches(">29</w:t>").count(), 2, "the text in the choice and again in the fallback");
|
||||
|
||||
// Read back: the choice, once.
|
||||
let r = craft_docx::read(&bytes).unwrap();
|
||||
let mut boxes = Vec::new();
|
||||
r.objects_in_reading_order(&mut |o| boxes.extend(o.text_boxes()));
|
||||
boxes.retain(|id| r.parts.get(id).is_some_and(|p| p.kind == PartKind::TextBox));
|
||||
assert_eq!(boxes.len(), 1, "one text box, not the fallback as well");
|
||||
assert_eq!(r.plain_text(craft_wordmodel::StoryRef::Part(boxes[0])), "29");
|
||||
}
|
||||
|
||||
/// The fallback keeps the box's own stacking order (not that of a floating object inside it),
|
||||
/// its hidden flag and its text direction.
|
||||
#[test]
|
||||
fn the_fallback_keeps_order_visibility_and_direction() {
|
||||
let mut d = Document::from_text("a");
|
||||
let inner = d.add_part(PartKind::TextBox, vec![para_block(craft_wordmodel::Paragraph::with_text("inner", Default::default()))]);
|
||||
let mut nested = shape(ShapeKind::TextBox, Some(inner));
|
||||
if let InlineObject::Shape { float, .. } = &mut nested {
|
||||
float.z = 100;
|
||||
}
|
||||
let mut p = craft_wordmodel::Paragraph::with_text("outer", Default::default());
|
||||
p.insert_object(0, nested, &Default::default()).unwrap();
|
||||
let outer = d.add_part(PartKind::TextBox, vec![para_block(p)]);
|
||||
d.parts.get_mut(&outer).unwrap().body = TextBody { vert: TextVert::Vert270, ..Default::default() };
|
||||
let mut o = shape(ShapeKind::TextBox, Some(outer));
|
||||
if let InlineObject::Shape { float, .. } = &mut o {
|
||||
float.z = 10;
|
||||
float.hidden = true;
|
||||
}
|
||||
d.insert_object(&craft_wordmodel::Pos::body(0, 1), o, &Default::default()).unwrap();
|
||||
let xml = document_xml(&craft_docx::write(&d).unwrap());
|
||||
// The outer box's VML: the `v:rect` holding the vertical text box (the inner box's fallback
|
||||
// is copied into the outer one's text, so it comes after it).
|
||||
let tb = xml.find(r#"<v:textbox style="layout-flow:vertical;mso-layout-flow-alt:bottom-to-top">"#).expect("the outer box's text, turned");
|
||||
let rect = &xml[xml[..tb].rfind("<v:rect ").expect("its v:rect")..tb];
|
||||
assert!(rect.contains("z-index:10;"), "{rect}");
|
||||
assert!(rect.contains("visibility:hidden"), "{rect}");
|
||||
}
|
||||
|
||||
/// Shapes behind the text keep their order in the fallback: a higher `z` stacks higher, below
|
||||
/// the text (a negative `z-index` in signed 32-bit range).
|
||||
#[test]
|
||||
fn behind_text_shapes_keep_their_order() {
|
||||
let mut d = Document::from_text("a");
|
||||
for z in [10, 20, u32::MAX] {
|
||||
let mut o = shape(ShapeKind::Rectangle, None);
|
||||
if let InlineObject::Shape { float, .. } = &mut o {
|
||||
float.wrap = Wrap::BehindText;
|
||||
float.z = z;
|
||||
}
|
||||
d.insert_object(&craft_wordmodel::Pos::body(0, 1), o, &Default::default()).unwrap();
|
||||
}
|
||||
let xml = document_xml(&craft_docx::write(&d).unwrap());
|
||||
let mut got: Vec<i64> = xml
|
||||
.match_indices("z-index:")
|
||||
.map(|(i, m)| {
|
||||
let v = &xml[i + m.len()..];
|
||||
v[..v.find([';', '"']).unwrap()].parse().unwrap()
|
||||
})
|
||||
.collect();
|
||||
got.sort_unstable();
|
||||
let low = -(1i64 << 31);
|
||||
assert_eq!(got, vec![low + 10, low + 20, -1], "{xml}");
|
||||
}
|
||||
|
||||
/// A shape VML has no simple form for stays a bare `wps:wsp`: guarded without a fallback, an old
|
||||
/// reader would open the file without it and drop it on its next save.
|
||||
#[test]
|
||||
fn a_shape_without_a_vml_form_is_not_guarded() {
|
||||
let mut d = Document::from_text("a");
|
||||
d.insert_object(&craft_wordmodel::Pos::body(0, 1), shape(ShapeKind::Star, None), &Default::default()).unwrap();
|
||||
let bytes = craft_docx::write(&d).unwrap();
|
||||
let xml = document_xml(&bytes);
|
||||
assert!(xml.contains("<w:r><w:drawing>") && xml.contains(r#"<a:prstGeom prst="star5">"#), "{xml}");
|
||||
assert!(!xml.contains("<mc:AlternateContent>"), "{xml}");
|
||||
let r = craft_docx::read(&bytes).unwrap();
|
||||
let mut kinds = Vec::new();
|
||||
r.objects_in_reading_order(&mut |o| {
|
||||
if let InlineObject::Shape { kind, .. } = o {
|
||||
kinds.push(*kind);
|
||||
}
|
||||
});
|
||||
assert_eq!(kinds, vec![ShapeKind::Star]);
|
||||
}
|
||||
|
||||
/// A partial fallback must not turn a linked story into independent boxes in older readers.
|
||||
#[test]
|
||||
fn linked_boxes_stay_bare_while_unlinked_boxes_keep_their_fallback() {
|
||||
let mut d = Document::from_text("a\nb\nc\nd");
|
||||
let mut ids = Vec::new();
|
||||
for (i, text) in ["Flows on", "", "", "Independent"].into_iter().enumerate() {
|
||||
let id = d.add_part(PartKind::TextBox, vec![para_block(craft_wordmodel::Paragraph::with_text(text, Default::default()))]);
|
||||
ids.push(id);
|
||||
d.insert_object(&craft_wordmodel::Pos::body(i, 1), shape(ShapeKind::TextBox, Some(id)), &Default::default()).unwrap();
|
||||
}
|
||||
d.parts.get_mut(&ids[0]).unwrap().body.next = Some(ids[1]);
|
||||
d.parts.get_mut(&ids[1]).unwrap().body.next = Some(ids[2]);
|
||||
let bytes = craft_docx::write(&d).unwrap();
|
||||
let xml = document_xml(&bytes);
|
||||
assert_eq!(xml.matches("<mc:AlternateContent>").count(), 1, "{xml}");
|
||||
assert_eq!(xml.matches("<mc:Fallback>").count(), 1, "{xml}");
|
||||
let fallback = xml.split("<mc:Fallback>").nth(1).unwrap().split("</mc:Fallback>").next().unwrap();
|
||||
assert!(fallback.contains(">Independent</w:t>"), "{fallback}");
|
||||
assert!(!fallback.contains("Flows on"), "the chain head must not get an unlinked fallback");
|
||||
assert_eq!(xml.matches(">Flows on</w:t>").count(), 1);
|
||||
assert!(xml.contains(r#"<wps:txbx id="1">"#), "{xml}");
|
||||
assert!(xml.contains(r#"<wps:linkedTxbx id="1" seq="1"/>"#), "{xml}");
|
||||
assert!(xml.contains(r#"<wps:linkedTxbx id="1" seq="2"/>"#), "{xml}");
|
||||
let read = craft_docx::read(&bytes).unwrap();
|
||||
let chains = read.text_box_chains();
|
||||
assert_eq!(chains.len(), 1);
|
||||
assert_eq!(chains[0].len(), 3);
|
||||
assert_eq!(read.plain_text(craft_wordmodel::StoryRef::Part(chains[0][0])), "Flows on");
|
||||
}
|
||||
@@ -9,6 +9,14 @@ pub enum Rgba8EncodeError {
|
||||
/// The output buffer could not be allocated.
|
||||
OutOfMemory,
|
||||
}
|
||||
/// Reserves the DPX output up front; a failed reservation becomes [`Rgba8EncodeError::OutOfMemory`].
|
||||
fn output_buffer(size: usize) -> std::result::Result<Vec<u8>, Rgba8EncodeError> {
|
||||
let mut buf = Vec::new();
|
||||
match buf.try_reserve_exact(size) {
|
||||
Ok(()) => Ok(buf),
|
||||
Err(_) => Err(Rgba8EncodeError::OutOfMemory),
|
||||
}
|
||||
}
|
||||
fn checked_len(rgba: &[u8], w: u32, h: u32) -> std::result::Result<usize, Rgba8EncodeError> {
|
||||
let n = (w as usize).checked_mul(h as usize).and_then(|n| n.checked_mul(4)).ok_or(Rgba8EncodeError::FrameSize)?;
|
||||
if w == 0 || h == 0 || rgba.len() != n {
|
||||
@@ -86,8 +94,7 @@ pub fn encode_rgba8_dpx10(rgba: &[u8], w: u32, h: u32, creator: &str) -> std::re
|
||||
hd[1928] = 0;
|
||||
// The caller already holds the frame; reserve the output fallibly so a large export reports
|
||||
// an error instead of aborting the process.
|
||||
let mut out = Vec::new();
|
||||
out.try_reserve_exact(size).map_err(|_| Rgba8EncodeError::OutOfMemory)?;
|
||||
let mut out = output_buffer(size)?;
|
||||
out.extend_from_slice(&hd);
|
||||
let ten = |v: u8| (u32::from(v) << 2) | (u32::from(v) >> 6);
|
||||
for &[r, g, b, _] in rgba.as_chunks::<4>().0 {
|
||||
@@ -107,4 +114,9 @@ mod tests {
|
||||
assert_eq!(&bytes[2048..], &((1023u32 << 22) | (514u32 << 2)).to_be_bytes());
|
||||
assert_eq!(encode_rgba8_dpx10(&[], 0, 1, ""), Err(Rgba8EncodeError::FrameSize));
|
||||
}
|
||||
#[test]
|
||||
fn output_reservation_failure_maps_to_out_of_memory() {
|
||||
assert!(matches!(output_buffer(usize::MAX), Err(Rgba8EncodeError::OutOfMemory)));
|
||||
assert_eq!(output_buffer(DPX_HEADER).map(|b| b.capacity() >= DPX_HEADER), Ok(true));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -184,7 +184,7 @@ impl CropKept {
|
||||
/// drawn one.
|
||||
/// - Dismissal ([`dismiss`](Self::dismiss)): after Discard or an accepted crop the default frame
|
||||
/// is still made but stays hidden until the next press, ratio change or size change
|
||||
/// ([`show`](Self::show)), for that document and size only.
|
||||
/// ([`show_frame`](Self::show_frame)), for that document and size only.
|
||||
/// - [`CropTool`](super::CropTool) is the one writer: the session has no edit type of its own.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct CropSession {
|
||||
@@ -464,7 +464,7 @@ impl CropSession {
|
||||
}
|
||||
|
||||
/// Discard or an accepted crop: the session is cleared and the document's default frame stays
|
||||
/// hidden until [`show`](Self::show). The next [`ensure_frame`](Self::ensure_frame) still makes
|
||||
/// hidden until [`show_frame`](Self::show_frame). The next [`ensure_frame`](Self::ensure_frame) still makes
|
||||
/// the frame, so the hidden overlay has one to show.
|
||||
pub fn dismiss(&mut self) {
|
||||
let made_for = self.frame_for;
|
||||
|
||||
@@ -109,7 +109,7 @@ pub(super) fn ruler_cursor(precise: bool, protractor: bool) -> Cursor {
|
||||
/// Shift 10, Delete and Backspace clear the line.
|
||||
/// - It paints a cross at each point, in the line's look; the app shows the line and the arm with `StraightLinePreview`
|
||||
/// ([`RulerPass::show_line`](super::RulerPass::show_line)).
|
||||
/// - [Read only](Self::read_only), it paints the crosses at rest and takes no press, key or focus: the ruler drawn
|
||||
/// - [Read only](craft_ui::contract::Props::read_only), it paints the crosses at rest and takes no press, key or focus: the ruler drawn
|
||||
/// under the other tools ([`show_ruler`](super::show_ruler)).
|
||||
/// - **AccessKit:** the group's value reads "Length L, angle A°" in the ruler's units; each point is named and valued
|
||||
/// "x, y" in the ruler's units.
|
||||
|
||||
@@ -276,9 +276,7 @@ fn the_gear_enables_only_the_chosen_sizes_and_escape_keeps_the_pick() {
|
||||
|
||||
#[trial]
|
||||
fn a_fixed_size_typed_past_its_range_keeps_the_end_and_reports_it_once_and_an_emptied_field_is_empty() {
|
||||
let mut start = BoxConstraints::default();
|
||||
start.constraint = BoxConstraint::FixedSize;
|
||||
start.fixed = (Some(100.0), Some(50.0));
|
||||
let start = BoxConstraints { constraint: BoxConstraint::FixedSize, fixed: (Some(100.0), Some(50.0)), ..Default::default() };
|
||||
let (mut fixture, host) = gear(start);
|
||||
type_into(&mut fixture, "Fixed Size W", "400000");
|
||||
assert_eq!(host.borrow().value.fixed, (Some(300_000.0), Some(50.0)), "a width past 300,000 keeps 300,000");
|
||||
|
||||
@@ -161,8 +161,7 @@ fn shift_squares_the_box_from_the_shorter_side() {
|
||||
|
||||
/// A Rectangle drag under the gear `constraint` with `change` applied to the gear's values.
|
||||
fn gear(constraint: BoxConstraint, change: impl FnOnce(&mut BoxConstraints)) -> Shapes {
|
||||
let mut constraints = BoxConstraints::default();
|
||||
constraints.constraint = constraint;
|
||||
let mut constraints = BoxConstraints { constraint, ..Default::default() };
|
||||
change(&mut constraints);
|
||||
Shapes::new(ShapeDragOptions::new(ShapeKind::Rectangle).constraints(constraints))
|
||||
}
|
||||
|
||||
@@ -776,7 +776,7 @@ fn a_secondary_click_on_the_box_opens_the_mode_menu() {
|
||||
});
|
||||
});
|
||||
fixture.settle();
|
||||
let mut right_click = |fixture: &mut Fixture, at: Pos2| {
|
||||
let right_click = |fixture: &mut Fixture, at: Pos2| {
|
||||
fixture.frame(vec![pointer::moved(at)]);
|
||||
fixture.frame(vec![pointer::secondary(at, true)]);
|
||||
fixture.frame(vec![pointer::secondary(at, false)]);
|
||||
|
||||
@@ -354,6 +354,10 @@ pub struct Chart {
|
||||
pub legend: Option<String>,
|
||||
/// `c:legend/c:overlay`: the legend is drawn over the plot instead of shrinking it.
|
||||
pub legend_overlay: bool,
|
||||
/// `c:legend/c:legendEntry`: entries whose text has its own size, weight or colour, over
|
||||
/// the legend's.
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub legend_entries: Vec<LegendEntry>,
|
||||
/// Text size in points of the axis labels and legend (from their `c:txPr`, else the chart's);
|
||||
/// `None`: the file gives none.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
@@ -431,6 +435,21 @@ pub struct Chart {
|
||||
pub kept: ChartKept,
|
||||
}
|
||||
|
||||
/// One legend entry's own text (`c:legendEntry/c:txPr`): what it sets wins over the legend's.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct LegendEntry {
|
||||
/// The entry's place in the legend, from 0 (`c:idx`).
|
||||
pub idx: u32,
|
||||
/// Text size in points.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub size: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bold: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub color: Option<ColorRef>,
|
||||
}
|
||||
|
||||
/// One relationship of a chart part, with the target string as it was written.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
@@ -483,6 +502,7 @@ impl Default for Chart {
|
||||
series: vec![],
|
||||
legend: Some("b".into()),
|
||||
legend_overlay: false,
|
||||
legend_entries: vec![],
|
||||
text_size: None,
|
||||
title_size: None,
|
||||
data_labels: false,
|
||||
|
||||
@@ -143,6 +143,7 @@ fn chart_within(root: &El, mut budget: usize) -> Option<Chart> {
|
||||
c.legend = ch.child(Local("legend")).map(|l| l.child(Local("legendPos")).and_then(|c| c.attr_local("val")).unwrap_or("r").to_string());
|
||||
c.legend_overlay =
|
||||
ch.child(Local("legend")).and_then(|l| l.child(Local("overlay"))).and_then(|o| o.attr_local("val").and_then(value::to_bool)).unwrap_or(false);
|
||||
c.legend_entries = legend_entries(ch.child(Local("legend")));
|
||||
c.chart_fill = simple_fill(root.child(Local("spPr")));
|
||||
c.plot_fill = simple_fill(pa.child(Local("spPr")));
|
||||
|
||||
@@ -552,6 +553,52 @@ fn text_color(e: Option<&El>) -> Option<crate::style::ColorRef> {
|
||||
rec(e?, 0)
|
||||
}
|
||||
|
||||
/// Legend entries read from one chart; a legend lists at most one per series or category.
|
||||
const MAX_LEGEND_ENTRIES: usize = 1024;
|
||||
|
||||
/// The legend's entries with text of their own (`c:legendEntry/c:txPr`), over the legend's.
|
||||
fn legend_entries(legend: Option<&El>) -> Vec<crate::chart::LegendEntry> {
|
||||
legend
|
||||
.into_iter()
|
||||
.flat_map(|l| l.children(Local("legendEntry")))
|
||||
.take(MAX_LEGEND_ENTRIES)
|
||||
.filter_map(|e| {
|
||||
// The paragraph's own run properties first, then the list style's defaults
|
||||
// (`a:lstStyle` comes before `a:p`, and is the less specific of the two).
|
||||
let tx = e.child(Local("txPr"));
|
||||
let para = tx.and_then(|t| t.child(Local("p")));
|
||||
let entry = crate::chart::LegendEntry {
|
||||
idx: e.child(Local("idx")).and_then(|i| i.attr_local("val")).and_then(|v| v.parse().ok())?,
|
||||
size: text_size(para).or_else(|| text_size(tx)),
|
||||
bold: text_bold(para).or_else(|| text_bold(tx)),
|
||||
color: text_color(para).or_else(|| text_color(tx)),
|
||||
};
|
||||
(entry.size.is_some() || entry.bold.is_some() || entry.color.is_some()).then_some(entry)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The weight (`b`) of the first `a:defRPr` or `a:rPr` that sets one under `e`.
|
||||
fn text_bold(e: Option<&El>) -> Option<bool> {
|
||||
fn rec(e: &El, depth: usize) -> Option<bool> {
|
||||
if depth > 8 {
|
||||
return None;
|
||||
}
|
||||
for c in e.elements() {
|
||||
if (c.is(Local("defRPr")) || c.is(Local("rPr")))
|
||||
&& let Some(b) = c.attr_local("b").and_then(value::to_bool)
|
||||
{
|
||||
return Some(b);
|
||||
}
|
||||
if let Some(v) = rec(c, depth + 1) {
|
||||
return Some(v);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
rec(e?, 0)
|
||||
}
|
||||
|
||||
/// The size in points of the first `a:defRPr` or `a:rPr` with one under `e` (`sz` in hundredths).
|
||||
fn text_size(e: Option<&El>) -> Option<f64> {
|
||||
fn rec(e: &El, depth: usize) -> Option<f64> {
|
||||
|
||||
@@ -101,6 +101,38 @@ fn tx_pr(w: &mut Writer, c: &Chart) {
|
||||
w.close();
|
||||
}
|
||||
|
||||
/// `c:legendEntry` with the entry's own text size, weight and colour.
|
||||
fn legend_entry(w: &mut Writer, e: &crate::chart::LegendEntry) {
|
||||
w.open("c:legendEntry", &[]);
|
||||
w.tag("c:idx").attr("val", e.idx).empty();
|
||||
w.open("c:txPr", &[]);
|
||||
w.empty("a:bodyPr", &[]);
|
||||
w.empty("a:lstStyle", &[]);
|
||||
w.open("a:p", &[]);
|
||||
w.open("a:pPr", &[]);
|
||||
let mut run = w.tag("a:defRPr");
|
||||
if let Some(sz) = e.size.filter(|v| v.is_finite() && *v > 0.0) {
|
||||
// 1 to 400 pt, as the reader takes it, so a size reads back as written.
|
||||
run = run.attr("sz", (sz * 100.0).round().clamp(100.0, 40_000.0) as u32);
|
||||
}
|
||||
if let Some(b) = e.bold {
|
||||
run = run.attr("b", u8::from(b));
|
||||
}
|
||||
match &e.color {
|
||||
Some(c) => {
|
||||
run.open();
|
||||
dml::solid(w, c);
|
||||
w.close();
|
||||
}
|
||||
None => run.empty(),
|
||||
}
|
||||
w.close();
|
||||
w.tag("a:endParaRPr").attr("lang", "en-US").empty();
|
||||
w.close();
|
||||
w.close();
|
||||
w.close();
|
||||
}
|
||||
|
||||
fn rich(w: &mut Writer, text: &str, color: Option<&crate::style::ColorRef>) {
|
||||
w.open("c:tx", &[]);
|
||||
w.open("c:rich", &[]);
|
||||
@@ -781,6 +813,9 @@ pub fn chart_xml(c: &Chart) -> (Vec<u8>, bool) {
|
||||
};
|
||||
w.open("c:legend", &[]);
|
||||
w.tag("c:legendPos").attr("val", p).empty();
|
||||
for e in &c.legend_entries {
|
||||
legend_entry(&mut w, e);
|
||||
}
|
||||
w.tag("c:overlay").attr("val", u8::from(c.legend_overlay)).empty();
|
||||
tx_pr(&mut w, c);
|
||||
w.close();
|
||||
@@ -1058,6 +1093,72 @@ mod tests {
|
||||
use crate::color::{ColorTransform, Rgba};
|
||||
use crate::style::ColorRef;
|
||||
|
||||
/// A legend entry's own text (`c:legendEntry/c:txPr`: size, bold, colour) is read, and a
|
||||
/// rebuilt part writes it back, in schema order before `c:overlay` (DeckCraft #2067).
|
||||
#[test]
|
||||
fn legend_entry_text_is_read_and_survives_an_edit() {
|
||||
let read = |xml: &[u8]| crate::chart::read::read_chart_xml(xml).unwrap();
|
||||
let entry = |idx: u32, rgb: &str| {
|
||||
format!(
|
||||
r#"<c:legendEntry><c:idx val="{idx}"/><c:txPr><a:bodyPr/><a:lstStyle/><a:p><a:pPr><a:defRPr sz="2000" b="1"><a:solidFill><a:srgbClr val="{rgb}"/></a:solidFill></a:defRPr></a:pPr><a:endParaRPr lang="en-US"/></a:p></c:txPr></c:legendEntry>"#
|
||||
)
|
||||
};
|
||||
let xml = format!(
|
||||
r#"<c:chartSpace xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"><c:chart><c:plotArea><c:lineChart><c:grouping val="standard"/><c:ser><c:idx val="0"/><c:val><c:numRef><c:numCache><c:ptCount val="1"/><c:pt idx="0"><c:v>1</c:v></c:pt></c:numCache></c:numRef></c:val></c:ser><c:axId val="1"/><c:axId val="2"/></c:lineChart><c:catAx><c:axId val="1"/><c:crossAx val="2"/></c:catAx><c:valAx><c:axId val="2"/><c:crossAx val="1"/></c:valAx></c:plotArea><c:legend><c:legendPos val="r"/>{}{}<c:legendEntry><c:idx val="2"/><c:txPr><a:bodyPr/><a:p><a:pPr><a:defRPr/></a:pPr></a:p></c:txPr></c:legendEntry><c:overlay val="0"/><c:txPr><a:bodyPr/><a:p><a:pPr><a:defRPr sz="1200"><a:solidFill><a:srgbClr val="7F7F7F"/></a:solidFill></a:defRPr></a:pPr></a:p></c:txPr></c:legend></c:chart></c:chartSpace>"#,
|
||||
entry(0, "C00000"),
|
||||
entry(1, "00B050")
|
||||
);
|
||||
let c = read(xml.as_bytes());
|
||||
let want = |idx: u32, rgb: (u8, u8, u8)| crate::chart::LegendEntry {
|
||||
idx,
|
||||
size: Some(20.0),
|
||||
bold: Some(true),
|
||||
color: Some(ColorRef::rgb(Rgba::rgb(rgb.0, rgb.1, rgb.2))),
|
||||
};
|
||||
// The third entry sets nothing of its own and is not kept.
|
||||
assert_eq!(c.legend_entries, vec![want(0, (0xC0, 0, 0)), want(1, (0, 0xB0, 0x50))]);
|
||||
assert_eq!(c.text_size, Some(12.0), "the legend's own size stays the legend's");
|
||||
let mut e = c.clone();
|
||||
e.title = Some("Units".into());
|
||||
e.raw = None;
|
||||
let (out, kept) = super::chart_xml(&e);
|
||||
assert!(!kept);
|
||||
let s = String::from_utf8(out.clone()).unwrap();
|
||||
let legend = &s[s.find("<c:legend>").unwrap()..];
|
||||
let (pos, entry0, overlay) =
|
||||
(legend.find("<c:legendPos").unwrap(), legend.find("<c:legendEntry>").unwrap(), legend.find("<c:overlay").unwrap());
|
||||
assert!(pos < entry0 && entry0 < overlay, "{s}");
|
||||
assert_eq!(read(&out).legend_entries, c.legend_entries);
|
||||
// None: nothing written.
|
||||
e.legend_entries.clear();
|
||||
assert!(!String::from_utf8(super::chart_xml(&e).0).unwrap().contains("legendEntry"));
|
||||
// The paragraph's own properties win over the list style's, which come first in the XML.
|
||||
let listed = xml.replacen(
|
||||
r#"<c:legendEntry><c:idx val="0"/><c:txPr><a:bodyPr/><a:lstStyle/>"#,
|
||||
r#"<c:legendEntry><c:idx val="0"/><c:txPr><a:bodyPr/><a:lstStyle><a:lvl1pPr><a:defRPr sz="1200" b="0"><a:solidFill><a:srgbClr val="0000FF"/></a:solidFill></a:defRPr></a:lvl1pPr></a:lstStyle>"#,
|
||||
1,
|
||||
);
|
||||
assert_ne!(listed, xml, "the list style was put in");
|
||||
let c = read(listed.as_bytes());
|
||||
assert_eq!(c.legend_entries[0], want(0, (0xC0, 0, 0)), "the paragraph's 20 pt, bold, C00000");
|
||||
// The list style alone still counts.
|
||||
let only_list =
|
||||
listed.replacen(r#"<a:defRPr sz="2000" b="1"><a:solidFill><a:srgbClr val="C00000"/></a:solidFill></a:defRPr>"#, "<a:defRPr/>", 1);
|
||||
let entry = &read(only_list.as_bytes()).legend_entries[0];
|
||||
assert_eq!((entry.size, entry.bold), (Some(12.0), Some(false)));
|
||||
// A size reads back as it was written, within the reader's 1-400 pt.
|
||||
for (size, back) in [(20.5, 20.5), (400.0, 400.0), (500.0, 400.0)] {
|
||||
let mut e = c.clone();
|
||||
e.raw = None;
|
||||
e.legend_entries = vec![crate::chart::LegendEntry { idx: 0, size: Some(size), bold: Some(false), color: None }];
|
||||
let written = super::chart_xml(&e).0;
|
||||
let sz = format!(r#"sz="{}""#, (back * 100.0) as u32);
|
||||
assert!(String::from_utf8_lossy(&written).contains(&sz), "{size} pt is written as {sz}");
|
||||
let back_entry = &read(&written).legend_entries[0];
|
||||
assert_eq!((back_entry.size, back_entry.bold), (Some(back), Some(false)), "{size} pt");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_cost_counts_the_caches_each_group_writes() {
|
||||
let s = Series { values: vec![Some(1.0); 600], ..Default::default() };
|
||||
|
||||
@@ -234,16 +234,18 @@ fn sci(toks: &[Tok], v: f64, sep: char, dec: char) -> Option<String> {
|
||||
m *= 10.0;
|
||||
exp -= 1;
|
||||
}
|
||||
// Rounding the mantissa to its digits can reach 10 (9.999 as `0.00`).
|
||||
if format!("{m:.mant_frac$}").starts_with("10") {
|
||||
m /= 10.0;
|
||||
exp += 1;
|
||||
}
|
||||
}
|
||||
let step = mant.iter().take_while(|t| **t != Tok::Dot).filter(|t| matches!(t, Tok::Digit(_))).count().max(1) as i32;
|
||||
let shift = exp.rem_euclid(step);
|
||||
exp -= shift;
|
||||
m *= 10f64.powi(shift);
|
||||
// Round after forming the engineering mantissa: 999.94 as `##0.0` stays 999.9.
|
||||
// Carry only when rounding exceeds all of the format's integer placeholders.
|
||||
let carry = 10f64.powi(step);
|
||||
if format!("{m:.mant_frac$}").parse::<f64>().is_ok_and(|rounded| rounded >= carry) {
|
||||
m /= carry;
|
||||
exp += step;
|
||||
}
|
||||
let mantissa = fixed(mant, mfirst, mlast, m, sep, dec);
|
||||
let sign = if exp < 0 {
|
||||
"-"
|
||||
@@ -404,6 +406,21 @@ mod tests {
|
||||
assert_eq!(format("##0.0E+0", 999.96, None), "1.0E+3");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engineering_formats_round_the_full_mantissa() {
|
||||
for (code, value, expected) in [
|
||||
("##0.0E+00", 999.94, "999.9E+00"),
|
||||
("##0.0E+00", 0.99994, "999.9E-03"),
|
||||
("##0.0E+00", -999.94, "-999.9E+00"),
|
||||
("#0.0E+00", 99.94, "99.9E+00"),
|
||||
("##0.0E+00", 999.96, "1.0E+03"),
|
||||
("#0.0E+00", 99.96, "1.0E+02"),
|
||||
] {
|
||||
assert_eq!(format(code, value, None), expected, "{code}, {value}");
|
||||
}
|
||||
assert_eq!(format("##0.0E+00", 999.94, Some("de-DE")), "999,9E+00");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn never_panics() {
|
||||
for code in ["", ";", ";;;", "\"", "[", "\\", "0.", ".", "#,", "%", "_", "*", "@", "0E+00", "[$", "\"unterminated"] {
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
[package]
|
||||
name = "craft-elf"
|
||||
description = "ELF program headers and dynamic section entries from hostile bytes, from Craft Anything: 32/64-bit, either byte order, with bounded counts and overflow-checked offsets, plus segment, permission and dynamic tag names. Pure Rust, no I/O."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
thiserror = { workspace = true }
|
||||
# `serde`: `Serialize` on `ProgramHeader` (addresses as `0x…` strings), for apps that report
|
||||
# segments as JSON.
|
||||
serde = { workspace = true, optional = true }
|
||||
|
||||
[features]
|
||||
serde = ["dep:serde"]
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
@@ -0,0 +1,37 @@
|
||||
//! Bounds-checked reads of fixed-width integers in both byte orders: out of range is `None`.
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(crate) struct Reader<'a> {
|
||||
buf: &'a [u8],
|
||||
}
|
||||
|
||||
macro_rules! int {
|
||||
($le:ident, $be:ident, $ty:ty) => {
|
||||
pub(crate) fn $le(&self, at: usize) -> Option<$ty> {
|
||||
self.take(at).map(<$ty>::from_le_bytes)
|
||||
}
|
||||
|
||||
pub(crate) fn $be(&self, at: usize) -> Option<$ty> {
|
||||
self.take(at).map(<$ty>::from_be_bytes)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
impl<'a> Reader<'a> {
|
||||
pub(crate) fn new(buf: &'a [u8]) -> Self {
|
||||
Self { buf }
|
||||
}
|
||||
|
||||
pub(crate) fn u8(&self, at: usize) -> Option<u8> {
|
||||
self.buf.get(at).copied()
|
||||
}
|
||||
|
||||
fn take<const N: usize>(&self, at: usize) -> Option<[u8; N]> {
|
||||
let end = at.checked_add(N)?;
|
||||
self.buf.get(at..end)?.try_into().ok()
|
||||
}
|
||||
|
||||
int!(u16_le, u16_be, u16);
|
||||
int!(u32_le, u32_be, u32);
|
||||
int!(u64_le, u64_be, u64);
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
//! Addresses serialised as `0x…` strings, as Craft Anything reports them.
|
||||
|
||||
pub(crate) fn serialize<S: serde::Serializer>(value: &u64, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
serializer.serialize_str(&format!("{value:#x}"))
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
//! ELF program headers and dynamic section entries from untrusted bytes, 32- or 64-bit in either
|
||||
//! byte order: [`program_headers`], [`dynamic_entries`], [`va_to_off`], and the segment type,
|
||||
//! permission and dynamic tag names.
|
||||
//!
|
||||
//! From Craft Anything (`crates/formats/src/elf.rs`), where the mitigations, dynamic symbols,
|
||||
//! symbol versions and core-file notes built on these stay. With the `serde` feature,
|
||||
//! [`ProgramHeader`] is `Serialize` with `vaddr` as a `0x…` string.
|
||||
|
||||
mod bytes;
|
||||
#[cfg(feature = "serde")]
|
||||
mod hex;
|
||||
|
||||
use crate::bytes::Reader;
|
||||
|
||||
/// Program headers read from one table.
|
||||
pub const MAX_PHDRS: usize = 4096;
|
||||
|
||||
/// Dynamic entries read from one segment.
|
||||
pub const MAX_DYN: usize = 65_536;
|
||||
|
||||
/// Why a read gave up.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum ElfError {
|
||||
/// The ELF header ends before the file does.
|
||||
#[error("the file ends early ({0})")]
|
||||
Truncated(&'static str),
|
||||
/// No ELF magic, or an invalid class, encoding or field.
|
||||
#[error("the file is malformed ({0})")]
|
||||
Malformed(String),
|
||||
}
|
||||
|
||||
/// Endian/class-aware reads over an ELF image.
|
||||
#[derive(Clone, Copy)]
|
||||
struct E<'a> {
|
||||
r: Reader<'a>,
|
||||
is64: bool,
|
||||
le: bool,
|
||||
}
|
||||
|
||||
impl<'a> E<'a> {
|
||||
fn new(data: &'a [u8]) -> Result<Self, ElfError> {
|
||||
if !data.starts_with(b"\x7fELF") {
|
||||
return Err(ElfError::Malformed("no ELF magic".into()));
|
||||
}
|
||||
let r = Reader::new(data);
|
||||
let class = r.u8(4).ok_or(ElfError::Truncated("ELF ident"))?;
|
||||
let enc = r.u8(5).ok_or(ElfError::Truncated("ELF ident"))?;
|
||||
if !matches!(class, 1 | 2) || !matches!(enc, 1 | 2) {
|
||||
return Err(ElfError::Malformed("bad ELF class or data encoding".into()));
|
||||
}
|
||||
Ok(E { r, is64: class == 2, le: enc == 1 })
|
||||
}
|
||||
|
||||
fn u16(&self, off: usize) -> Option<u16> {
|
||||
if self.le { self.r.u16_le(off) } else { self.r.u16_be(off) }
|
||||
}
|
||||
|
||||
fn u32(&self, off: usize) -> Option<u32> {
|
||||
if self.le { self.r.u32_le(off) } else { self.r.u32_be(off) }
|
||||
}
|
||||
|
||||
fn u64(&self, off: usize) -> Option<u64> {
|
||||
if self.le { self.r.u64_le(off) } else { self.r.u64_be(off) }
|
||||
}
|
||||
|
||||
fn word(&self, off: usize) -> Option<u64> {
|
||||
if self.is64 { self.u64(off) } else { self.u32(off).map(u64::from) }
|
||||
}
|
||||
|
||||
fn word_size(&self) -> usize {
|
||||
if self.is64 { 8 } else { 4 }
|
||||
}
|
||||
}
|
||||
|
||||
/// One program header.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize))]
|
||||
pub struct ProgramHeader {
|
||||
/// [`segment_type_name`] of `p_type`.
|
||||
pub kind: String,
|
||||
/// [`permissions`] of `p_flags`.
|
||||
pub flags: String,
|
||||
pub offset: u64,
|
||||
#[cfg_attr(feature = "serde", serde(serialize_with = "hex::serialize"))]
|
||||
pub vaddr: u64,
|
||||
pub file_size: u64,
|
||||
pub mem_size: u64,
|
||||
pub align: u64,
|
||||
/// The raw segment type.
|
||||
#[cfg_attr(feature = "serde", serde(skip))]
|
||||
pub p_type: u32,
|
||||
/// The raw `PF_R`/`PF_W`/`PF_X` flags.
|
||||
#[cfg_attr(feature = "serde", serde(skip))]
|
||||
pub p_flags: u32,
|
||||
}
|
||||
|
||||
/// A segment type's `PT_` name without the prefix (`LOAD`, `GNU_RELRO`…), or its number in hex.
|
||||
pub fn segment_type_name(t: u32) -> String {
|
||||
match t {
|
||||
0 => "NULL".into(),
|
||||
1 => "LOAD".into(),
|
||||
2 => "DYNAMIC".into(),
|
||||
3 => "INTERP".into(),
|
||||
4 => "NOTE".into(),
|
||||
5 => "SHLIB".into(),
|
||||
6 => "PHDR".into(),
|
||||
7 => "TLS".into(),
|
||||
0x6474_e550 => "GNU_EH_FRAME".into(),
|
||||
0x6474_e551 => "GNU_STACK".into(),
|
||||
0x6474_e552 => "GNU_RELRO".into(),
|
||||
0x6474_e553 => "GNU_PROPERTY".into(),
|
||||
0x7000_0001 => "ARM_EXIDX".into(),
|
||||
other => format!("{other:#x}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// `p_flags` as `rwx`, with `-` for each permission missing.
|
||||
pub fn permissions(f: u32) -> String {
|
||||
format!("{}{}{}", if f & 4 != 0 { 'r' } else { '-' }, if f & 2 != 0 { 'w' } else { '-' }, if f & 1 != 0 { 'x' } else { '-' })
|
||||
}
|
||||
|
||||
/// The program headers of an ELF file, at most [`MAX_PHDRS`]. A table that runs past the file
|
||||
/// ends at the last complete entry.
|
||||
pub fn program_headers(data: &[u8]) -> Result<Vec<ProgramHeader>, ElfError> {
|
||||
let e = E::new(data)?;
|
||||
let (phoff, phentsize, phnum) = if e.is64 { (e.u64(32), e.u16(54), e.u16(56)) } else { (e.u32(28).map(u64::from), e.u16(42), e.u16(44)) };
|
||||
let phoff = usize::try_from(phoff.ok_or(ElfError::Truncated("ELF header"))?).map_err(|_| ElfError::Malformed("phoff".into()))?;
|
||||
let phentsize = usize::from(phentsize.unwrap_or(0));
|
||||
let phnum = usize::from(phnum.unwrap_or(0)).min(MAX_PHDRS);
|
||||
let min = if e.is64 { 56 } else { 32 };
|
||||
if phnum > 0 && phentsize < min {
|
||||
return Err(ElfError::Malformed("program header entry too small".into()));
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
for i in 0..phnum {
|
||||
// The fields below sit within `phentsize` bytes of `o`, so their offsets can't overflow either.
|
||||
let Some(o) = i.checked_mul(phentsize).and_then(|x| x.checked_add(phoff)).filter(|&o| o.checked_add(phentsize).is_some()) else { break };
|
||||
let ph = if e.is64 {
|
||||
(e.u32(o), e.u32(o + 4), e.u64(o + 8), e.u64(o + 16), e.u64(o + 32), e.u64(o + 40), e.u64(o + 48))
|
||||
} else {
|
||||
(
|
||||
e.u32(o),
|
||||
e.u32(o + 24),
|
||||
e.u32(o + 4).map(u64::from),
|
||||
e.u32(o + 8).map(u64::from),
|
||||
e.u32(o + 16).map(u64::from),
|
||||
e.u32(o + 20).map(u64::from),
|
||||
e.u32(o + 28).map(u64::from),
|
||||
)
|
||||
};
|
||||
let (Some(t), Some(f), Some(off), Some(va), Some(fs), Some(ms), Some(al)) = ph else { break };
|
||||
out.push(ProgramHeader {
|
||||
kind: segment_type_name(t),
|
||||
flags: permissions(f),
|
||||
offset: off,
|
||||
vaddr: va,
|
||||
file_size: fs,
|
||||
mem_size: ms,
|
||||
align: al,
|
||||
p_type: t,
|
||||
p_flags: f,
|
||||
});
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// A dynamic tag's `DT_` name without the prefix (`NEEDED`, `GNU_HASH`…), or its number in hex.
|
||||
pub fn dynamic_tag_name(tag: u64) -> String {
|
||||
match tag {
|
||||
0 => "NULL",
|
||||
1 => "NEEDED",
|
||||
2 => "PLTRELSZ",
|
||||
3 => "PLTGOT",
|
||||
4 => "HASH",
|
||||
5 => "STRTAB",
|
||||
6 => "SYMTAB",
|
||||
7 => "RELA",
|
||||
8 => "RELASZ",
|
||||
9 => "RELAENT",
|
||||
10 => "STRSZ",
|
||||
11 => "SYMENT",
|
||||
12 => "INIT",
|
||||
13 => "FINI",
|
||||
14 => "SONAME",
|
||||
15 => "RPATH",
|
||||
16 => "SYMBOLIC",
|
||||
17 => "REL",
|
||||
18 => "RELSZ",
|
||||
19 => "RELENT",
|
||||
20 => "PLTREL",
|
||||
21 => "DEBUG",
|
||||
22 => "TEXTREL",
|
||||
23 => "JMPREL",
|
||||
24 => "BIND_NOW",
|
||||
25 => "INIT_ARRAY",
|
||||
26 => "FINI_ARRAY",
|
||||
27 => "INIT_ARRAYSZ",
|
||||
28 => "FINI_ARRAYSZ",
|
||||
29 => "RUNPATH",
|
||||
30 => "FLAGS",
|
||||
0x6fff_fef5 => "GNU_HASH",
|
||||
0x6fff_fffb => "FLAGS_1",
|
||||
0x6fff_fff0 => "VERSYM",
|
||||
0x6fff_fffe => "VERNEED",
|
||||
0x6fff_ffff => "VERNEEDNUM",
|
||||
_ => return format!("{tag:#x}"),
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
/// Map a virtual address to a file offset using `PT_LOAD` headers. Addresses in a segment's
|
||||
/// zero-filled tail, or whose offset would overflow, map to nothing.
|
||||
pub fn va_to_off(phdrs: &[ProgramHeader], va: u64) -> Option<u64> {
|
||||
phdrs.iter().filter(|p| p.p_type == 1).find(|p| va >= p.vaddr && va - p.vaddr < p.file_size).and_then(|p| p.offset.checked_add(va - p.vaddr))
|
||||
}
|
||||
|
||||
/// The (tag, value) pairs of the `PT_DYNAMIC` segment in `phdrs`, up to its `DT_NULL`, at most
|
||||
/// [`MAX_DYN`]. Empty without a dynamic segment.
|
||||
pub fn dynamic_entries(data: &[u8], phdrs: &[ProgramHeader]) -> Result<Vec<(u64, u64)>, ElfError> {
|
||||
Ok(entries(&E::new(data)?, phdrs))
|
||||
}
|
||||
|
||||
fn entries(e: &E<'_>, phdrs: &[ProgramHeader]) -> Vec<(u64, u64)> {
|
||||
let mut out = Vec::new();
|
||||
let Some(d) = phdrs.iter().find(|p| p.p_type == 2) else { return out };
|
||||
let ent = 2 * e.word_size();
|
||||
let n = usize::try_from(d.file_size).unwrap_or(0) / ent;
|
||||
let start = usize::try_from(d.offset).unwrap_or(usize::MAX);
|
||||
for i in 0..n.min(MAX_DYN) {
|
||||
let Some(o) = i.checked_mul(ent).and_then(|x| start.checked_add(x)) else { break };
|
||||
let (Some(tag), Some(val)) = (e.word(o), o.checked_add(e.word_size()).and_then(|v| e.word(v))) else { break };
|
||||
if tag == 0 {
|
||||
break;
|
||||
}
|
||||
out.push((tag, val));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,201 @@
|
||||
//! Program header and dynamic section tests. `an_address_past_the_last_file_offset_maps_to_nothing`
|
||||
//! is from Craft Anything's `elf.rs`; the image builder is a small stand-in for its testkit's.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Field writer in the image's class and byte order.
|
||||
struct Image {
|
||||
d: Vec<u8>,
|
||||
is64: bool,
|
||||
le: bool,
|
||||
}
|
||||
|
||||
impl Image {
|
||||
fn put(&mut self, at: usize, value: u64, width: usize) {
|
||||
let bytes = value.to_le_bytes();
|
||||
let mut field = bytes[..width].to_vec();
|
||||
if !self.le {
|
||||
field.reverse();
|
||||
}
|
||||
self.d[at..at + width].copy_from_slice(&field);
|
||||
}
|
||||
|
||||
fn word(&mut self, at: usize, value: u64) {
|
||||
self.put(at, value, if self.is64 { 8 } else { 4 });
|
||||
}
|
||||
}
|
||||
|
||||
/// An executable with PT_PHDR, PT_LOAD (r-x, whole file at 0x40_0000), PT_DYNAMIC at file 0x200
|
||||
/// and PT_GNU_STACK (rw-). The dynamic section holds NEEDED, STRTAB and a DT_NULL, then junk.
|
||||
fn elf(is64: bool, le: bool) -> Vec<u8> {
|
||||
let mut img = Image { d: vec![0u8; 0x300], is64, le };
|
||||
img.d[..4].copy_from_slice(b"\x7fELF");
|
||||
img.d[4] = if is64 { 2 } else { 1 };
|
||||
img.d[5] = if le { 1 } else { 2 };
|
||||
img.d[6] = 1;
|
||||
let (phoff_at, entsize_at, num_at, phoff, entsize) = if is64 { (32, 54, 56, 64, 56) } else { (28, 42, 44, 52, 32) };
|
||||
img.word(phoff_at, phoff as u64);
|
||||
img.put(entsize_at, entsize as u64, 2);
|
||||
img.put(num_at, 4, 2);
|
||||
let headers = [
|
||||
(6u32, 4u32, phoff as u64, 0x40_0040, 4 * entsize as u64),
|
||||
(1, 5, 0, 0x40_0000, 0x300),
|
||||
(2, 6, 0x200, 0x40_0200, 0x60),
|
||||
(0x6474_e551, 6, 0, 0, 0),
|
||||
];
|
||||
for (i, (t, f, off, va, size)) in headers.into_iter().enumerate() {
|
||||
let o = phoff + i * entsize;
|
||||
img.put(o, u64::from(t), 4);
|
||||
if is64 {
|
||||
img.put(o + 4, u64::from(f), 4);
|
||||
img.word(o + 8, off);
|
||||
img.word(o + 16, va);
|
||||
img.word(o + 32, size);
|
||||
img.word(o + 40, size);
|
||||
img.word(o + 48, 0x1000);
|
||||
} else {
|
||||
img.word(o + 4, off);
|
||||
img.word(o + 8, va);
|
||||
img.word(o + 16, size);
|
||||
img.word(o + 20, size);
|
||||
img.put(o + 24, u64::from(f), 4);
|
||||
img.word(o + 28, 0x1000);
|
||||
}
|
||||
}
|
||||
let w = if is64 { 8 } else { 4 };
|
||||
for (i, (tag, value)) in [(1u64, 1u64), (5, 0x40_0280), (0, 0), (1, 9)].into_iter().enumerate() {
|
||||
img.word(0x200 + i * 2 * w, tag);
|
||||
img.word(0x200 + i * 2 * w + w, value);
|
||||
}
|
||||
img.d
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn program_headers_in_every_class_and_byte_order() {
|
||||
for is64 in [false, true] {
|
||||
for le in [false, true] {
|
||||
let ph = program_headers(&elf(is64, le)).unwrap();
|
||||
let kinds: Vec<(&str, &str)> = ph.iter().map(|p| (p.kind.as_str(), p.flags.as_str())).collect();
|
||||
assert_eq!(kinds, [("PHDR", "r--"), ("LOAD", "r-x"), ("DYNAMIC", "rw-"), ("GNU_STACK", "rw-")], "is64={is64} le={le}");
|
||||
assert_eq!((ph[1].offset, ph[1].vaddr, ph[1].file_size, ph[1].mem_size, ph[1].align), (0, 0x40_0000, 0x300, 0x300, 0x1000));
|
||||
assert_eq!((ph[2].p_type, ph[2].p_flags), (2, 6));
|
||||
let dynamic = dynamic_entries(&elf(is64, le), &ph).unwrap();
|
||||
assert_eq!(dynamic, [(1, 1), (5, 0x40_0280)], "stops at DT_NULL");
|
||||
assert_eq!(va_to_off(&ph, 0x40_0280), Some(0x280));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_idents_and_headers_are_errors() {
|
||||
assert_eq!(program_headers(b"\x7fELX\x02\x01"), Err(ElfError::Malformed("no ELF magic".into())));
|
||||
assert_eq!(program_headers(b"\x7fELF"), Err(ElfError::Truncated("ELF ident")));
|
||||
assert_eq!(program_headers(b"\x7fELF\x02"), Err(ElfError::Truncated("ELF ident")));
|
||||
assert_eq!(program_headers(b"\x7fELF\x03\x01"), Err(ElfError::Malformed("bad ELF class or data encoding".into())));
|
||||
assert_eq!(program_headers(b"\x7fELF\x01\x00"), Err(ElfError::Malformed("bad ELF class or data encoding".into())));
|
||||
assert_eq!(program_headers(&elf(true, true)[..20]), Err(ElfError::Truncated("ELF header")));
|
||||
assert_eq!(ElfError::Truncated("ELF header").to_string(), "the file ends early (ELF header)");
|
||||
let mut small = elf(true, true);
|
||||
small[54] = 40;
|
||||
assert_eq!(program_headers(&small).unwrap_err().to_string(), "the file is malformed (program header entry too small)");
|
||||
assert!(dynamic_entries(b"nope", &[]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_table_cut_short_keeps_its_complete_entries() {
|
||||
let d = elf(true, true);
|
||||
assert_eq!(program_headers(&d[..64 + 56 * 2 + 10]).unwrap().len(), 2);
|
||||
let mut none = d.clone();
|
||||
none[56] = 0;
|
||||
none[54] = 0;
|
||||
assert!(program_headers(&none).unwrap().is_empty(), "no headers need no entry size");
|
||||
let mut far = d.clone();
|
||||
far[32..40].copy_from_slice(&u64::MAX.to_le_bytes());
|
||||
assert!(program_headers(&far).unwrap().is_empty(), "offsets that overflow stop the walk");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_entries_need_a_dynamic_segment_inside_the_file() {
|
||||
let d = elf(false, true);
|
||||
let mut ph = program_headers(&d).unwrap();
|
||||
assert!(dynamic_entries(&d, &ph[..2]).unwrap().is_empty());
|
||||
ph[2].offset = 0x2f8;
|
||||
assert!(dynamic_entries(&d, &ph).unwrap().is_empty(), "an entry cut off by the end of the file");
|
||||
ph[2].offset = u64::MAX;
|
||||
assert!(dynamic_entries(&d, &ph).unwrap().is_empty());
|
||||
}
|
||||
|
||||
fn load(offset: u64, vaddr: u64, file_size: u64) -> ProgramHeader {
|
||||
ProgramHeader { kind: "LOAD".into(), flags: String::new(), offset, vaddr, file_size, mem_size: file_size, align: 0, p_type: 1, p_flags: 0 }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_address_past_the_last_file_offset_maps_to_nothing() {
|
||||
assert_eq!(va_to_off(&[load(0x200, 0x1000, 0x100)], 0x1010), Some(0x210));
|
||||
assert_eq!(va_to_off(&[load(u64::MAX, 0x1000, 0x100)], 0x1000), Some(u64::MAX));
|
||||
assert_eq!(va_to_off(&[load(u64::MAX, 0x1000, 0x100)], 0x1001), None, "the offset would pass u64::MAX");
|
||||
assert_eq!(va_to_off(&[load(0x200, 0x1000, 0x100)], 0x1100), None, "the zero-filled tail has no file bytes");
|
||||
let mut dynamic = load(0, 0x1000, 0x100);
|
||||
dynamic.p_type = 2;
|
||||
assert_eq!(va_to_off(&[dynamic], 0x1010), None, "only PT_LOAD maps");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn names() {
|
||||
let segments: Vec<String> =
|
||||
[0, 1, 2, 3, 4, 5, 6, 7, 0x6474_e550, 0x6474_e551, 0x6474_e552, 0x6474_e553, 0x7000_0001, 0x99].into_iter().map(segment_type_name).collect();
|
||||
assert_eq!(
|
||||
segments,
|
||||
[
|
||||
"NULL",
|
||||
"LOAD",
|
||||
"DYNAMIC",
|
||||
"INTERP",
|
||||
"NOTE",
|
||||
"SHLIB",
|
||||
"PHDR",
|
||||
"TLS",
|
||||
"GNU_EH_FRAME",
|
||||
"GNU_STACK",
|
||||
"GNU_RELRO",
|
||||
"GNU_PROPERTY",
|
||||
"ARM_EXIDX",
|
||||
"0x99"
|
||||
]
|
||||
);
|
||||
assert_eq!([0, 1, 2, 4, 7].map(permissions), ["---", "--x", "-w-", "r--", "rwx"]);
|
||||
let tags: Vec<String> =
|
||||
(0..=30).chain([0x6fff_fef5, 0x6fff_fffb, 0x6fff_fff0, 0x6fff_fffe, 0x6fff_ffff, 0x7000_0000]).map(dynamic_tag_name).collect();
|
||||
assert_eq!(tags[..3], ["NULL", "NEEDED", "PLTRELSZ"]);
|
||||
assert_eq!(tags[29..], ["RUNPATH", "FLAGS", "GNU_HASH", "FLAGS_1", "VERSYM", "VERNEED", "VERNEEDNUM", "0x70000000"]);
|
||||
assert!(tags.iter().all(|t| !t.is_empty()));
|
||||
}
|
||||
|
||||
#[cfg(feature = "serde")]
|
||||
#[test]
|
||||
fn program_headers_serialize_with_hex_addresses() {
|
||||
let ph = program_headers(&elf(true, true)).unwrap();
|
||||
let value = serde_json::to_value(&ph[1]).unwrap();
|
||||
assert_eq!(value["kind"], "LOAD");
|
||||
assert_eq!(value["vaddr"], "0x400000");
|
||||
assert!(value.get("p_type").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn damaged_input_never_panics() {
|
||||
for sample in [elf(true, true), elf(false, false)] {
|
||||
for i in 0..sample.len() {
|
||||
if let Ok(ph) = program_headers(&sample[..i]) {
|
||||
let _ = dynamic_entries(&sample[..i], &ph);
|
||||
}
|
||||
for v in [0x00, 0x7f, 0xff] {
|
||||
let mut m = sample.clone();
|
||||
m[i] = v;
|
||||
if let Ok(ph) = program_headers(&m) {
|
||||
let _ = dynamic_entries(&m, &ph);
|
||||
let _ = va_to_off(&ph, 0x40_0280);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+263
-44
@@ -7,7 +7,7 @@
|
||||
//! - [`decode_part`] decodes the channels of one flat part that a filter keeps, in their stored
|
||||
//! sample type, over the part's data window. Every compression the `exr` crate reads is read,
|
||||
//! scan lines and tiles alike (the largest resolution level). Other parts' chunks are skipped
|
||||
//! unread, and channels the filter drops are never allocated.
|
||||
//! unread, and channels the filter drops are never allocated. Other parts may be deep.
|
||||
//! - [`fn@write`] writes one scan-line part.
|
||||
//!
|
||||
//! The `exr` crate sizes buffers from headers before it reads pixels, so everything that reaches
|
||||
@@ -18,7 +18,7 @@
|
||||
use std::io::Cursor;
|
||||
|
||||
use exr::block::UncompressedBlock;
|
||||
use exr::block::chunk::TileCoordinates;
|
||||
use exr::block::chunk::{Chunk, CompressedBlock, CompressedScanLineBlock, CompressedTileBlock, TileCoordinates};
|
||||
use exr::image::read::image::ReadImage;
|
||||
use exr::image::read::layers::{ChannelsReader, ReadAllLayers, ReadChannels};
|
||||
use exr::meta::MetaData;
|
||||
@@ -245,11 +245,21 @@ fn window_size(size: exr::math::Vec2<usize>) -> (u32, u32) {
|
||||
|
||||
/// Reads the headers and checks every part's declared size, alone and together, before any pixel
|
||||
/// allocation. Headers are read without the crate's validation, which refuses deep parts.
|
||||
fn read_meta(bytes: &[u8], limits: &Limits) -> Result<MetaData> {
|
||||
pub(super) fn read_meta(bytes: &[u8], limits: &Limits) -> Result<MetaData> {
|
||||
check_previews(bytes)?;
|
||||
let meta = MetaData::read_from_buffered(Cursor::new(bytes), false).map_err(map_exr)?;
|
||||
let (mut total, mut dwa) = (0u64, 0u64);
|
||||
for header in meta.headers.iter() {
|
||||
let (mut total, mut flat, mut dwa) = (0u64, 0u64, 0u64);
|
||||
for (index, header) in meta.headers.iter().enumerate() {
|
||||
// The crate sizes a flat chunk's bound as a block's pixels times the bytes a pixel takes,
|
||||
// without checking the product, before it reads the chunk.
|
||||
let block_pixels = match header.blocks {
|
||||
exr::meta::BlockDescription::Tiles(tiles) => (tiles.tile_size.0 as u64).checked_mul(tiles.tile_size.1 as u64),
|
||||
exr::meta::BlockDescription::ScanLines => (header.compression.scan_lines_per_block() as u64).checked_mul(header.layer_size.0 as u64),
|
||||
};
|
||||
let block_bytes = block_pixels.and_then(|n| n.checked_mul(header.channels.bytes_per_pixel as u64)).and_then(|n| usize::try_from(n).ok());
|
||||
if !header.deep && block_bytes.is_none() {
|
||||
return Err(Error::LimitExceeded(format!("a block of part {index} holds more bytes than this platform can address")));
|
||||
}
|
||||
let (w, h) = window_size(header.layer_size);
|
||||
if w > limits.max_width || h > limits.max_height {
|
||||
return Err(Error::LimitExceeded(format!("a {w}x{h} part exceeds {}x{}", limits.max_width, limits.max_height)));
|
||||
@@ -260,23 +270,24 @@ fn read_meta(bytes: &[u8], limits: &Limits) -> Result<MetaData> {
|
||||
}
|
||||
let bytes = pixels.saturating_mul(header.channels.list.len().max(1) as u64 * 4);
|
||||
total = total.saturating_add(bytes);
|
||||
if matches!(header.compression, ExrCompression::DWAA(_) | ExrCompression::DWAB(_)) {
|
||||
dwa = dwa.saturating_add(bytes);
|
||||
if !header.deep {
|
||||
flat = flat.saturating_add(bytes);
|
||||
if matches!(header.compression, ExrCompression::DWAA(_) | ExrCompression::DWAB(_)) {
|
||||
dwa = dwa.saturating_add(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
if total > limits.max_bytes {
|
||||
return Err(Error::LimitExceeded(format!("{total} bytes across {} parts exceed {}", meta.headers.len(), limits.max_bytes)));
|
||||
}
|
||||
// The crate allocates a flat part's samples before reading its chunks, so the file must be
|
||||
// able to hold them. DWA's run-length channels are zlib-packed on top and reach about
|
||||
// 132000:1, so DWA parts are counted at a quarter. Deep parts size their samples from the
|
||||
// file's own counts.
|
||||
if !meta.requirements.has_deep_data {
|
||||
let declared = total.saturating_sub(dwa).saturating_add(dwa / 4);
|
||||
let capacity = (bytes.len() as u64).saturating_mul(limits.max_ratio).saturating_add(1 << 20);
|
||||
if declared > capacity {
|
||||
return Err(Error::LimitExceeded(format!("a {} byte file cannot hold {declared} bytes of samples", bytes.len())));
|
||||
}
|
||||
// A flat part's samples are allocated before its chunks are read, so the file must be able
|
||||
// to hold the flat parts' samples. DWA's run-length channels are zlib-packed on top and reach
|
||||
// about 132000:1, so DWA parts are counted at a quarter. Deep parts size their samples from
|
||||
// the file's own counts and are not counted here.
|
||||
let declared = flat.saturating_sub(dwa).saturating_add(dwa / 4);
|
||||
let capacity = (bytes.len() as u64).saturating_mul(limits.max_ratio).saturating_add(1 << 20);
|
||||
if declared > capacity {
|
||||
return Err(Error::LimitExceeded(format!("a {} byte file cannot hold {declared} bytes of samples", bytes.len())));
|
||||
}
|
||||
Ok(meta)
|
||||
}
|
||||
@@ -459,19 +470,24 @@ pub fn decode_display_part(
|
||||
}
|
||||
let block_limit = std::cell::Cell::new(None);
|
||||
let select = DisplaySelect { index, keep, fill, limits, seen: std::cell::Cell::new(0), block_limit: &block_limit };
|
||||
let image = ReadImage::new(ReadAllLayers { read_channels: select }, |_| {})
|
||||
.from_buffered(Cursor::new(bytes))
|
||||
.map_err(|error| block_limit.take().map(Error::LimitExceeded).unwrap_or_else(|| map_exr(error)))?;
|
||||
let channels = image
|
||||
.layer_data
|
||||
.into_iter()
|
||||
.nth(index)
|
||||
.ok_or_else(|| malformed("the part was not read"))?
|
||||
.channel_data
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|(name, samples)| DisplayChannel { name, samples })
|
||||
.collect();
|
||||
let exr_error = |error| block_limit.take().map(Error::LimitExceeded).unwrap_or_else(|| map_exr(error));
|
||||
let planes = if meta.headers.iter().any(|header| header.deep) {
|
||||
let header = meta.headers.get(index).ok_or_else(|| malformed(format!("part {index}")))?;
|
||||
select.seen.set(index);
|
||||
let mut reader = select.create_channels_reader(header).map_err(exr_error)?;
|
||||
read_flat_part_chunks(bytes, &meta, index, &mut reader, limits)?;
|
||||
reader.into_channels()
|
||||
} else {
|
||||
ReadImage::new(ReadAllLayers { read_channels: select }, |_| {})
|
||||
.from_buffered(Cursor::new(bytes))
|
||||
.map_err(exr_error)?
|
||||
.layer_data
|
||||
.into_iter()
|
||||
.nth(index)
|
||||
.ok_or_else(|| malformed("the part was not read"))?
|
||||
.channel_data
|
||||
};
|
||||
let channels = planes.into_iter().flatten().map(|(name, samples)| DisplayChannel { name, samples }).collect();
|
||||
Ok(DisplayPart { info, channels })
|
||||
}
|
||||
|
||||
@@ -515,18 +531,23 @@ impl<'s> ReadChannels<'s> for DisplaySelect<'_> {
|
||||
u32s: vec![],
|
||||
});
|
||||
}
|
||||
// The EXR crate decompresses each accepted block before `read_block` can discard ignored
|
||||
// channels. Bound that all-channel input separately from the selected output planes.
|
||||
let Some(block_bytes) = layer_w
|
||||
.checked_mul(layer_h)
|
||||
.and_then(|pixels| pixels.checked_mul(header.channels.list.len().max(1)))
|
||||
.and_then(|samples| samples.checked_mul(std::mem::size_of::<f32>()))
|
||||
else {
|
||||
// The EXR crate decompresses each accepted block, with every channel, before `read_block`
|
||||
// can discard the ignored ones, so bound that all-channel block separately from the
|
||||
// selected output planes. Blocks are decompressed one at a time, so the bound is the
|
||||
// largest block (a tile within the data window, or a scan-line group across it), not the
|
||||
// whole part: a part whose full channel set is larger than `max_bytes` still decodes when
|
||||
// the kept planes and each block fit.
|
||||
let block_pixels = match header.blocks {
|
||||
exr::meta::BlockDescription::Tiles(tiles) => tiles.tile_size.0.min(layer_w).checked_mul(tiles.tile_size.1.min(layer_h)),
|
||||
// a part shorter than its compression's scan-line group has only its own rows
|
||||
exr::meta::BlockDescription::ScanLines => header.compression.scan_lines_per_block().min(layer_h).checked_mul(layer_w),
|
||||
};
|
||||
let Some(block_bytes) = block_pixels.and_then(|pixels| pixels.checked_mul(header.channels.bytes_per_pixel.max(1))) else {
|
||||
self.block_limit.set(Some(format!("data-window block size exceeds {}", self.limits.max_bytes)));
|
||||
return Err(exr::error::Error::Invalid("data-window block size overflow".into()));
|
||||
};
|
||||
if u64::try_from(block_bytes).map_or(true, |bytes| bytes > self.limits.max_bytes) || block_bytes > isize::MAX as usize {
|
||||
self.block_limit.set(Some(format!("{block_bytes} data-window bytes exceed {}", self.limits.max_bytes)));
|
||||
self.block_limit.set(Some(format!("{block_bytes} bytes in one data-window block exceed {}", self.limits.max_bytes)));
|
||||
return Err(exr::error::Error::Invalid("data-window blocks exceed limit".into()));
|
||||
}
|
||||
let pixels = display_w.checked_mul(display_h).filter(|n| *n > 0).ok_or(exr::error::Error::Invalid("display window".into()))?;
|
||||
@@ -727,22 +748,220 @@ impl ChannelsReader for SelectReader {
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the headers end and the offset tables begin: [`MetaData::read_from_buffered`] consumes
|
||||
/// exactly the version field and the headers.
|
||||
fn tables_start(bytes: &[u8]) -> Result<u64> {
|
||||
let mut cursor = Cursor::new(bytes);
|
||||
MetaData::read_from_buffered(&mut cursor, false).map_err(map_exr)?;
|
||||
Ok(cursor.position())
|
||||
}
|
||||
|
||||
/// Stable dense ordinal without materializing every tile. The library's block iterator
|
||||
/// internally allocates a Vec of all coordinates, which is unsuitable for cropped reads.
|
||||
pub(super) fn flat_block_ordinal(header: &Header, tile: TileCoordinates) -> Result<usize> {
|
||||
use exr::math::Vec2;
|
||||
use exr::meta::attribute::LevelMode;
|
||||
let mut start = 0usize;
|
||||
let tile_size = header.max_block_pixel_size();
|
||||
if tile_size.0 == 0 || tile_size.1 == 0 {
|
||||
return Err(malformed("flat block has a zero dimension"));
|
||||
}
|
||||
let mut visit = |level: Vec2<usize>, size: Vec2<usize>| -> Result<Option<usize>> {
|
||||
let nx = size.0.div_ceil(tile_size.0);
|
||||
let ny = size.1.div_ceil(tile_size.1);
|
||||
if tile.level_index == level {
|
||||
if tile.tile_index.0 >= nx || tile.tile_index.1 >= ny {
|
||||
return Err(malformed("flat tile lies outside its level"));
|
||||
}
|
||||
return tile
|
||||
.tile_index
|
||||
.1
|
||||
.checked_mul(nx)
|
||||
.and_then(|n| n.checked_add(tile.tile_index.0))
|
||||
.and_then(|n| start.checked_add(n))
|
||||
.map(Some)
|
||||
.ok_or_else(|| malformed("block ordinal overflow"));
|
||||
}
|
||||
start = nx.checked_mul(ny).and_then(|n| start.checked_add(n)).ok_or_else(|| malformed("block count overflow"))?;
|
||||
Ok(None)
|
||||
};
|
||||
match header.blocks {
|
||||
exr::meta::BlockDescription::Tiles(tiles) => match tiles.level_mode {
|
||||
LevelMode::Singular => {
|
||||
if let Some(i) = visit(Vec2(0, 0), header.layer_size)? {
|
||||
return Ok(i);
|
||||
}
|
||||
}
|
||||
LevelMode::MipMap => {
|
||||
for (level, size) in exr::meta::mip_map_levels(tiles.rounding_mode, header.layer_size) {
|
||||
if let Some(i) = visit(Vec2(level, level), size)? {
|
||||
return Ok(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
LevelMode::RipMap => {
|
||||
for (level, size) in exr::meta::rip_map_levels(tiles.rounding_mode, header.layer_size) {
|
||||
if let Some(i) = visit(level, size)? {
|
||||
return Ok(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
exr::meta::BlockDescription::ScanLines => {
|
||||
if let Some(i) = visit(Vec2(0, 0), header.layer_size)? {
|
||||
return Ok(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(malformed("flat tile names a missing level"))
|
||||
}
|
||||
|
||||
/// Reads the chunks of flat part `index` of a file that also holds deep parts into `reader`.
|
||||
///
|
||||
/// The `exr` crate's image reader validates every header before it reads a chunk and refuses
|
||||
/// deep headers, so here only the flat headers are validated, the offset tables (one `u64` per
|
||||
/// chunk, a table per part in header order) are walked from the end of the headers, and only the
|
||||
/// selected part's chunks are read. Each chunk must start past the last offset table and inside
|
||||
/// the file, carry the selected part's number, and hold one of the part's blocks that no other
|
||||
/// chunk of the part holds, so the part's chunks hold every one of its blocks. A scan-line
|
||||
/// chunk's y must be the first line of its block, and no chunk may declare more bytes than the
|
||||
/// file has left after it.
|
||||
fn read_flat_part_chunks(bytes: &[u8], meta: &MetaData, index: usize, reader: &mut impl ChannelsReader, limits: &Limits) -> Result<()> {
|
||||
let header = meta.headers.get(index).ok_or_else(|| malformed(format!("part {index}")))?;
|
||||
let multipart = meta.requirements.is_multilayer();
|
||||
let mut long_names = false;
|
||||
for flat in meta.headers.iter().filter(|h| !h.deep) {
|
||||
flat.validate(multipart, &mut long_names, false).map_err(map_exr)?;
|
||||
}
|
||||
let truncated = || malformed("the offset tables run past the end of the file");
|
||||
let (mut table, mut tables_end) = (0u64, tables_start(bytes)?);
|
||||
for (i, h) in meta.headers.iter().enumerate() {
|
||||
if i == index {
|
||||
table = tables_end;
|
||||
}
|
||||
let len = u64::try_from(h.chunk_count).ok().and_then(|n| n.checked_mul(8)).ok_or_else(truncated)?;
|
||||
tables_end = tables_end.checked_add(len).filter(|end| *end <= bytes.len() as u64).ok_or_else(truncated)?;
|
||||
}
|
||||
let offsets = usize::try_from(table)
|
||||
.ok()
|
||||
.and_then(|start| bytes.get(start..start.checked_add(header.chunk_count.checked_mul(8)?)?))
|
||||
.ok_or_else(truncated)?;
|
||||
// One bit per block, not a Vec/HashSet of every coordinate. A cropped display must not
|
||||
// permit unbounded bookkeeping for an enormous data window of tiny tiles.
|
||||
let tracking_bytes = header.chunk_count.div_ceil(8);
|
||||
if tracking_bytes as u64 > limits.max_bytes || tracking_bytes > isize::MAX as usize {
|
||||
return Err(Error::LimitExceeded("block tracking exceeds the byte budget".into()));
|
||||
}
|
||||
let mut seen = Vec::new();
|
||||
seen.try_reserve_exact(tracking_bytes).map_err(|_| Error::LimitExceeded("cannot allocate block tracking".into()))?;
|
||||
seen.resize(tracking_bytes, 0u8);
|
||||
for &entry in offsets.as_chunks::<8>().0 {
|
||||
let offset = u64::from_le_bytes(entry);
|
||||
let chunk = usize::try_from(offset)
|
||||
.ok()
|
||||
.filter(|_| offset >= tables_end)
|
||||
.and_then(|start| bytes.get(start..))
|
||||
.filter(|rest| !rest.is_empty())
|
||||
.ok_or_else(|| malformed(format!("chunk offset {offset} is outside the chunk area of the file")))?;
|
||||
if multipart {
|
||||
let part = chunk.get(..4).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(i32::from_le_bytes);
|
||||
let part = part.ok_or_else(|| malformed(format!("the chunk at offset {offset} is cut off")))?;
|
||||
if usize::try_from(part).ok() != Some(index) {
|
||||
return Err(malformed(format!("the chunk at offset {offset} belongs to part {part}, not part {index}")));
|
||||
}
|
||||
}
|
||||
// The crate allocates a chunk's declared payload before it reads it, so the payload must
|
||||
// fit in the rest of the file. The length follows the part number and the block's y, or
|
||||
// its four tile coordinates.
|
||||
let fields = if matches!(header.blocks, exr::meta::BlockDescription::Tiles(_)) { 16 } else { 4 };
|
||||
let size_at = if multipart { 4 } else { 0 } + fields;
|
||||
let size = chunk.get(size_at..size_at + 4).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(i32::from_le_bytes);
|
||||
let size = size.ok_or_else(|| malformed(format!("the chunk at offset {offset} is cut off")))?;
|
||||
let left = chunk.len() - (size_at + 4);
|
||||
if usize::try_from(size).is_ok_and(|size| size > left) {
|
||||
return Err(malformed(format!("the chunk at offset {offset} declares {size} bytes, but {left} remain in the file")));
|
||||
}
|
||||
let size = usize::try_from(size).map_err(|_| malformed("negative packed chunk length"))?;
|
||||
// Read coordinates without allocating the payload. Validate their real clipped block
|
||||
// and budgets before Chunk::read uses the nominal (possibly huge) tile dimensions.
|
||||
let coordinate_at = if multipart { 4 } else { 0 };
|
||||
let shape = if matches!(header.blocks, exr::meta::BlockDescription::Tiles(_)) {
|
||||
let coordinates = TileCoordinates::read(&mut Cursor::new(&chunk[coordinate_at..])).map_err(map_exr)?;
|
||||
CompressedBlock::Tile(CompressedTileBlock { coordinates, compressed_pixels_le: Vec::new() })
|
||||
} else {
|
||||
let y = chunk
|
||||
.get(coordinate_at..coordinate_at + 4)
|
||||
.and_then(|b| <[u8; 4]>::try_from(b).ok())
|
||||
.ok_or_else(|| malformed("truncated scan-line coordinate"))?;
|
||||
CompressedBlock::ScanLine(CompressedScanLineBlock { y_coordinate: i32::from_le_bytes(y), compressed_pixels_le: Vec::new() })
|
||||
};
|
||||
let tile = header.get_block_data_indices(&shape).map_err(map_exr)?;
|
||||
if let CompressedBlock::ScanLine(block) = &shape {
|
||||
// The block index is rounded toward zero, so a y that is not the first line of a
|
||||
// block, or lies just above the data window, would still name a block.
|
||||
let lines = header.compression.scan_lines_per_block() as i64;
|
||||
let first = i64::from(header.own_attributes.layer_position.y()) + tile.tile_index.y() as i64 * lines;
|
||||
if i64::from(block.y_coordinate) != first {
|
||||
return Err(malformed(format!(
|
||||
"the scan-line chunk at offset {offset} starts at y {}, which is not the first line of a block",
|
||||
block.y_coordinate
|
||||
)));
|
||||
}
|
||||
}
|
||||
let ordinal = flat_block_ordinal(header, tile)?;
|
||||
let flag = 1u8 << (ordinal % 8);
|
||||
let byte = seen.get_mut(ordinal / 8).ok_or_else(|| malformed("block is outside its chunk table"))?;
|
||||
if *byte & flag != 0 {
|
||||
return Err(malformed("duplicate flat block"));
|
||||
}
|
||||
*byte |= flag;
|
||||
if reader.filter_block(tile) {
|
||||
let area = header.get_absolute_block_pixel_coordinates(tile).map_err(map_exr)?.size;
|
||||
let raw_size = area
|
||||
.0
|
||||
.checked_mul(area.1)
|
||||
.and_then(|n| n.checked_mul(header.channels.bytes_per_pixel))
|
||||
.ok_or_else(|| Error::LimitExceeded("flat block byte size overflows".into()))?;
|
||||
if raw_size as u64 > limits.max_bytes || raw_size > isize::MAX as usize {
|
||||
return Err(Error::LimitExceeded("flat block exceeds the byte budget".into()));
|
||||
}
|
||||
// Packed size need not be less than raw size for every accepted compression.
|
||||
// Bound its allocation independently, rather than rejecting valid compressed data
|
||||
// solely because a clipped edge block has fewer raw bytes.
|
||||
if size as u64 > limits.max_bytes || size > isize::MAX as usize {
|
||||
return Err(Error::LimitExceeded("packed flat block exceeds the byte budget".into()));
|
||||
}
|
||||
let chunk = Chunk::read(&mut Cursor::new(chunk), meta).map_err(map_exr)?;
|
||||
let block = UncompressedBlock::decompress_chunk(chunk, meta, false).map_err(map_exr)?;
|
||||
reader.read_block(header, block).map_err(map_exr)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Decodes the channels of flat part `index` that `keep` accepts (by the name as stored), in
|
||||
/// their stored type. A file with deep parts, or a kept channel that is subsampled, is
|
||||
/// [`Error::Unsupported`].
|
||||
/// their stored type. A deep part, or a kept channel that is subsampled, is
|
||||
/// [`Error::Unsupported`]; the file's other parts may be deep.
|
||||
pub fn decode_part(bytes: &[u8], index: usize, keep: &dyn Fn(&str) -> bool, limits: &Limits) -> Result<Part> {
|
||||
let meta = read_meta(bytes, limits)?;
|
||||
if meta.requirements.has_deep_data {
|
||||
let header = meta.headers.get(index).ok_or_else(|| malformed(format!("part {index} of {}", meta.headers.len())))?;
|
||||
if header.deep {
|
||||
return Err(Error::Unsupported("deep data (use the deep reader)".into()));
|
||||
}
|
||||
let header = meta.headers.get(index).ok_or_else(|| malformed(format!("part {index} of {}", meta.headers.len())))?;
|
||||
if matches!(header.compression, ExrCompression::HTJ2K32 | ExrCompression::HTJ2K256) {
|
||||
return Err(Error::Unsupported("HTJ2K compression".into()));
|
||||
}
|
||||
let info = part_info(index, header);
|
||||
let select = Select { index, keep, seen: std::cell::Cell::new(0) };
|
||||
let image = ReadImage::new(ReadAllLayers { read_channels: select }, |_| {}).from_buffered(Cursor::new(bytes)).map_err(map_exr)?;
|
||||
let planes = image.layer_data.into_iter().nth(index).map(|l| l.channel_data).ok_or_else(|| malformed("the part was not read"))?;
|
||||
let planes = if meta.headers.iter().any(|header| header.deep) {
|
||||
select.seen.set(index);
|
||||
let mut reader = select.create_channels_reader(header).map_err(map_exr)?;
|
||||
read_flat_part_chunks(bytes, &meta, index, &mut reader, limits)?;
|
||||
reader.into_channels()
|
||||
} else {
|
||||
let image = ReadImage::new(ReadAllLayers { read_channels: select }, |_| {}).from_buffered(Cursor::new(bytes)).map_err(map_exr)?;
|
||||
image.layer_data.into_iter().nth(index).map(|l| l.channel_data).ok_or_else(|| malformed("the part was not read"))?
|
||||
};
|
||||
let channels = info.channels.iter().zip(planes).filter_map(|(c, p)| Some(Channel { name: c.name.clone(), samples: p? })).collect();
|
||||
Ok(Part { info, channels })
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
//! The preview check is PhotoCraft's test; the rest builds files with the `exr` crate.
|
||||
//! The preview check is PhotoCraft's test; the rest builds files with the `exr` crate, except the
|
||||
//! files that mix flat and deep parts, which are built by hand.
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
|
||||
use exr::meta::attribute::{Preview, Text};
|
||||
@@ -129,10 +130,30 @@ fn decoded_display_part_places_selected_channels_and_checks_total_budget() {
|
||||
let limits = Limits { max_bytes: 6 * 4 - 1, ..Limits::default() };
|
||||
assert!(matches!(decode_display_part(&bytes, 0, &all, &|_| 0.0, &limits), Err(Error::LimitExceeded(_))));
|
||||
|
||||
// One selected display plane fits exactly, but the EXR decoder must decompress both data
|
||||
// channels in its input block before discarding the ignored one.
|
||||
// One selected display plane fits exactly. Both data channels together (2x2x2 f32, 32 bytes)
|
||||
// don't, but they are decompressed one scan-line block (2 pixels x 2 channels) at a time.
|
||||
let block_limits = Limits { max_bytes: 6 * 4, ..Limits::default() };
|
||||
assert!(matches!(decode_display_part(&bytes, 0, &|name| name == "R", &|_| 0.0, &block_limits), Err(Error::LimitExceeded(_))));
|
||||
let part = decode_display_part(&bytes, 0, &|name| name == "R", &|_| 9.0, &block_limits).unwrap();
|
||||
assert_eq!(part.channels[0].samples, [0.0, 1.0, 9.0, 2.0, 3.0, 9.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoded_display_part_bounds_each_all_channel_block() {
|
||||
// The kept R plane is 4 pixels of f32 (16 bytes), but each scan-line block carries all four
|
||||
// channels (64 bytes) before the ignored ones are discarded.
|
||||
let bytes = file(vec![layer(None, 4, 1, &["R", "G", "B", "A"], false, exr::prelude::Compression::ZIP1)], (4, 1));
|
||||
let r = |name: &str| name == "R";
|
||||
assert!(matches!(decode_display_part(&bytes, 0, &r, &|_| 0.0, &Limits { max_bytes: 16, ..Limits::default() }), Err(Error::LimitExceeded(_))));
|
||||
assert_eq!(decode_display_part(&bytes, 0, &r, &|_| 0.0, &Limits { max_bytes: 64, ..Limits::default() }).unwrap().channels.len(), 1);
|
||||
// ZIP groups 16 scan lines, but a one-row part's block is that row: still 64 bytes, not 1024
|
||||
let zip16 = file(vec![layer(None, 4, 1, &["R", "G", "B", "A"], false, exr::prelude::Compression::ZIP16)], (4, 1));
|
||||
assert!(matches!(decode_display_part(&zip16, 0, &r, &|_| 0.0, &Limits { max_bytes: 63, ..Limits::default() }), Err(Error::LimitExceeded(_))));
|
||||
assert_eq!(decode_display_part(&zip16, 0, &r, &|_| 0.0, &Limits { max_bytes: 64, ..Limits::default() }).unwrap().channels.len(), 1);
|
||||
// A tile is the block of a tiled part: a 4x4 tile of four f32 channels is 256 bytes, while
|
||||
// the whole 8x8 part is 1024 and the kept R plane of the 2x2 display window only 16.
|
||||
let tiled = file(vec![layer(None, 8, 8, &["R", "G", "B", "A"], true, exr::prelude::Compression::ZIP1)], (2, 2));
|
||||
assert!(matches!(decode_display_part(&tiled, 0, &r, &|_| 0.0, &Limits { max_bytes: 255, ..Limits::default() }), Err(Error::LimitExceeded(_))));
|
||||
assert_eq!(decode_display_part(&tiled, 0, &r, &|_| 0.0, &Limits { max_bytes: 256, ..Limits::default() }).unwrap().channels[0].samples.len(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -238,3 +259,451 @@ fn hostile_files_never_panic() {
|
||||
}
|
||||
assert!(parts(b"not exr", &limits).is_err());
|
||||
}
|
||||
|
||||
// A multi-part file with a flat part and a deep scan-line part, built by hand per the OpenEXR file
|
||||
// layout (the `exr` crate's writer refuses deep headers): version 2 with the deep and multi-part
|
||||
// flags, the headers and an empty header, one offset table per part, then chunks that each start
|
||||
// with their part number. Both parts are uncompressed, two chunks a part: the deep part and a
|
||||
// scan-line flat part hold one scan line a chunk, and a tiled flat part one 2 × 2 tile a chunk.
|
||||
|
||||
fn mixed_attr(out: &mut Vec<u8>, name: &str, ty: &str, payload: &[u8]) {
|
||||
out.extend_from_slice(name.as_bytes());
|
||||
out.push(0);
|
||||
out.extend_from_slice(ty.as_bytes());
|
||||
out.push(0);
|
||||
out.extend_from_slice(&(payload.len() as i32).to_le_bytes());
|
||||
out.extend_from_slice(payload);
|
||||
}
|
||||
|
||||
/// The version field and the headers of a mixed file whose flat part is `w × h` with float
|
||||
/// `B`, `G`, `R` channels, in scan lines or (with `tiled`) in one level of 2 × 2 tiles, and whose
|
||||
/// deep part is 3 × 2 with one float `A` channel. Both parts are uncompressed.
|
||||
fn mixed_headers(deep_first: bool, tiled: bool, (w, h): (usize, usize)) -> Vec<u8> {
|
||||
mixed_headers_with(deep_first, tiled, (w, h), 0, &["B", "G", "R"], 2)
|
||||
}
|
||||
|
||||
/// [`mixed_headers`] with the flat part's compression code (0 for none, or 3 for ZIP, which
|
||||
/// holds 16 scan lines a block), its float channels, and the width and height of its tiles.
|
||||
fn mixed_headers_with(deep_first: bool, tiled: bool, (w, h): (usize, usize), compression: u8, flat_channels: &[&str], tile: u32) -> Vec<u8> {
|
||||
mixed_headers_with_deep_size(deep_first, tiled, (w, h), compression, flat_channels, tile, (3, 2))
|
||||
}
|
||||
|
||||
fn mixed_headers_with_deep_size(
|
||||
deep_first: bool,
|
||||
tiled: bool,
|
||||
(w, h): (usize, usize),
|
||||
compression: u8,
|
||||
flat_channels: &[&str],
|
||||
tile: u32,
|
||||
deep_size: (usize, usize),
|
||||
) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
out.extend_from_slice(&0x0131_2f76u32.to_le_bytes());
|
||||
out.extend_from_slice(&(2u32 | 0x800 | 0x1000).to_le_bytes());
|
||||
for deep in if deep_first { [true, false] } else { [false, true] } {
|
||||
let (w, h, channels): (usize, usize, &[&str]) = if deep { (deep_size.0, deep_size.1, &["A"]) } else { (w, h, flat_channels) };
|
||||
let mut chlist = Vec::new();
|
||||
for name in channels {
|
||||
chlist.extend_from_slice(name.as_bytes());
|
||||
chlist.push(0);
|
||||
chlist.extend_from_slice(&2i32.to_le_bytes()); // float
|
||||
chlist.extend_from_slice(&[0u8; 4]); // pLinear and reserved
|
||||
chlist.extend_from_slice(&1i32.to_le_bytes());
|
||||
chlist.extend_from_slice(&1i32.to_le_bytes());
|
||||
}
|
||||
chlist.push(0);
|
||||
let window: Vec<u8> = [0i32, 0, w as i32 - 1, h as i32 - 1].iter().flat_map(|v| v.to_le_bytes()).collect();
|
||||
let tiled = tiled && !deep;
|
||||
let compression = if deep { 0 } else { compression };
|
||||
let lines = if compression == 3 { 16 } else { 1 };
|
||||
let chunks = if tiled { w.div_ceil(tile as usize) * h.div_ceil(tile as usize) } else { h.div_ceil(lines) };
|
||||
mixed_attr(&mut out, "channels", "chlist", &chlist);
|
||||
mixed_attr(&mut out, "chunkCount", "int", &(chunks as i32).to_le_bytes());
|
||||
mixed_attr(&mut out, "compression", "compression", &[compression]);
|
||||
mixed_attr(&mut out, "dataWindow", "box2i", &window);
|
||||
mixed_attr(&mut out, "displayWindow", "box2i", &window);
|
||||
mixed_attr(&mut out, "lineOrder", "lineOrder", &[0]);
|
||||
if deep {
|
||||
mixed_attr(&mut out, "maxSamplesPerPixel", "int", &1i32.to_le_bytes());
|
||||
}
|
||||
mixed_attr(&mut out, "name", "string", if deep { b"deep" } else { b"flat" });
|
||||
mixed_attr(&mut out, "pixelAspectRatio", "float", &1f32.to_le_bytes());
|
||||
mixed_attr(&mut out, "screenWindowCenter", "v2f", &[0u8; 8]);
|
||||
mixed_attr(&mut out, "screenWindowWidth", "float", &1f32.to_le_bytes());
|
||||
if tiled {
|
||||
// `tile` × `tile` tiles, one level, rounded down.
|
||||
let mut desc: Vec<u8> = [tile, tile].iter().flat_map(|v| v.to_le_bytes()).collect();
|
||||
desc.push(0);
|
||||
mixed_attr(&mut out, "tiles", "tiledesc", &desc);
|
||||
}
|
||||
let ty: &[u8] = if deep {
|
||||
b"deepscanline"
|
||||
} else if tiled {
|
||||
b"tiledimage"
|
||||
} else {
|
||||
b"scanlineimage"
|
||||
};
|
||||
mixed_attr(&mut out, "type", "string", ty);
|
||||
if deep {
|
||||
mixed_attr(&mut out, "version", "int", &1i32.to_le_bytes());
|
||||
}
|
||||
out.push(0); // end of this header
|
||||
}
|
||||
out.push(0); // end of the header list
|
||||
out
|
||||
}
|
||||
|
||||
/// Sample `i` of a flat channel: `R` is 1, 2, 3, …, `G` 10 more and `B` 100 more, top row first.
|
||||
fn mixed_flat(channel: f32, i: usize) -> f32 {
|
||||
channel + 1.0 + i as f32
|
||||
}
|
||||
|
||||
/// A mixed file with a 3 × 2 flat part, and its chunk offsets as `offsets[part][chunk]`. The
|
||||
/// tiled flat part's chunks hold the tiles at (0, 0) and (1, 0).
|
||||
fn mixed_file(deep_first: bool, tiled: bool) -> (Vec<u8>, [[usize; 2]; 2]) {
|
||||
let (w, h) = (3usize, 2usize);
|
||||
let mut out = mixed_headers(deep_first, tiled, (w, h));
|
||||
let chunks: Vec<Vec<Vec<u8>>> = (0..2i32)
|
||||
.map(|part| {
|
||||
let deep = (part == 0) == deep_first;
|
||||
(0..h)
|
||||
.map(|y| {
|
||||
let mut c = Vec::new();
|
||||
c.extend_from_slice(&part.to_le_bytes());
|
||||
if tiled && !deep {
|
||||
// Tile (y, 0) at level (0, 0): columns 2y and 2y + 1 that lie inside.
|
||||
[y as i32, 0, 0, 0].iter().for_each(|v| c.extend_from_slice(&v.to_le_bytes()));
|
||||
let xs = 2 * y..(2 * y + 2).min(w);
|
||||
c.extend_from_slice(&((xs.len() * h * 3 * 4) as i32).to_le_bytes());
|
||||
for row in 0..h {
|
||||
for channel in [100.0, 10.0, 0.0] {
|
||||
xs.clone().for_each(|x| c.extend_from_slice(&mixed_flat(channel, row * w + x).to_le_bytes()));
|
||||
}
|
||||
}
|
||||
return c;
|
||||
}
|
||||
c.extend_from_slice(&(y as i32).to_le_bytes());
|
||||
if deep {
|
||||
// Packed count table, packed and unpacked sample data: one sample a pixel.
|
||||
for size in [w * 4, w * 4, w * 4] {
|
||||
c.extend_from_slice(&(size as u64).to_le_bytes());
|
||||
}
|
||||
(1..=w as u32).for_each(|n| c.extend_from_slice(&n.to_le_bytes()));
|
||||
(0..w).for_each(|_| c.extend_from_slice(&0.5f32.to_le_bytes()));
|
||||
} else {
|
||||
c.extend_from_slice(&((w * 3 * 4) as i32).to_le_bytes());
|
||||
for channel in [100.0, 10.0, 0.0] {
|
||||
(0..w).for_each(|x| c.extend_from_slice(&mixed_flat(channel, y * w + x).to_le_bytes()));
|
||||
}
|
||||
}
|
||||
c
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
let mut offsets = [[0usize; 2]; 2];
|
||||
let mut at = out.len() + 2 * h * 8;
|
||||
for (part, lines) in chunks.iter().enumerate() {
|
||||
for (y, c) in lines.iter().enumerate() {
|
||||
offsets[part][y] = at;
|
||||
out.extend_from_slice(&(at as u64).to_le_bytes());
|
||||
at += c.len();
|
||||
}
|
||||
}
|
||||
chunks.into_iter().flatten().for_each(|c| out.extend_from_slice(&c));
|
||||
(out, offsets)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_flat_part_beside_a_deep_part_decodes() {
|
||||
let limits = Limits::default();
|
||||
for (deep_first, tiled) in [(false, false), (true, false), (false, true), (true, true)] {
|
||||
let (bytes, _) = mixed_file(deep_first, tiled);
|
||||
let (flat, deep) = if deep_first { (1, 0) } else { (0, 1) };
|
||||
let parts = parts(&bytes, &limits).unwrap();
|
||||
assert_eq!((parts[flat].deep, parts[deep].deep, parts[flat].name.as_deref()), (false, true, Some("flat")));
|
||||
|
||||
let part = decode_part(&bytes, flat, &all, &limits).unwrap();
|
||||
for (name, channel) in [("R", 0.0), ("G", 10.0), ("B", 100.0)] {
|
||||
let expected: Vec<f32> = (0..6).map(|i| mixed_flat(channel, i)).collect();
|
||||
assert_eq!(part.channel(name).unwrap().samples, Samples::F32(expected), "deep first {deep_first} tiled {tiled} {name}");
|
||||
}
|
||||
let only_g = decode_part(&bytes, flat, &|n| n == "G", &limits).unwrap();
|
||||
assert_eq!(only_g.channels.iter().map(|c| c.name.as_str()).collect::<Vec<_>>(), ["G"]);
|
||||
|
||||
let display = decode_display_part(&bytes, flat, &|n| n == "flat.R", &|_| 9.0, &limits).unwrap();
|
||||
assert_eq!(display.channels.len(), 1);
|
||||
assert_eq!((display.channels[0].name.as_str(), display.channels[0].samples.as_slice()), ("flat.R", &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0][..]));
|
||||
|
||||
assert!(matches!(decode_part(&bytes, deep, &all, &limits), Err(Error::Unsupported(_))));
|
||||
assert!(matches!(decode_display_part(&bytes, deep, &all, &|_| 0.0, &limits), Err(Error::Unsupported(_))));
|
||||
assert!(matches!(decode_part(&bytes, 2, &all, &limits), Err(Error::Malformed(_))));
|
||||
let deep_image = crate::deep::decode_deep(&bytes, &limits).unwrap();
|
||||
assert_eq!((deep_image.width, deep_image.height), (3, 2));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_flat_part_s_chunks_beside_a_deep_part_are_checked_before_they_are_read() {
|
||||
let limits = Limits::default();
|
||||
for deep_first in [false, true] {
|
||||
let (good, offsets) = mixed_file(deep_first, false);
|
||||
let (flat, deep) = if deep_first { (1, 0) } else { (0, 1) };
|
||||
let tables = mixed_headers(deep_first, false, (3, 2)).len();
|
||||
let entry = tables + flat * 2 * 8;
|
||||
let with_offset = |slot: usize, value: u64| {
|
||||
let mut b = good.clone();
|
||||
b[entry + slot * 8..entry + slot * 8 + 8].copy_from_slice(&value.to_le_bytes());
|
||||
b
|
||||
};
|
||||
let refused = |b: &[u8]| {
|
||||
for r in [decode_part(b, flat, &all, &limits).map(|_| ()), decode_display_part(b, flat, &all, &|_| 0.0, &limits).map(|_| ())] {
|
||||
assert!(matches!(&r, Err(Error::Malformed(_))), "deep first {deep_first}: {r:?}");
|
||||
}
|
||||
};
|
||||
// The deep part's chunk, the end of the file and past it, the offset tables, the same
|
||||
// chunk twice.
|
||||
refused(&with_offset(0, offsets[deep][0] as u64));
|
||||
refused(&with_offset(0, good.len() as u64));
|
||||
refused(&with_offset(0, u64::MAX));
|
||||
refused(&with_offset(0, (tables + 8) as u64));
|
||||
refused(&with_offset(1, offsets[flat][0] as u64));
|
||||
// A chunk naming another part, and one declaring more bytes than a block holds.
|
||||
let mut b = good.clone();
|
||||
b[offsets[flat][1]..offsets[flat][1] + 4].copy_from_slice(&(deep as i32).to_le_bytes());
|
||||
refused(&b);
|
||||
let mut b = good.clone();
|
||||
b[offsets[flat][0] + 8..offsets[flat][0] + 12].copy_from_slice(&i32::MAX.to_le_bytes());
|
||||
refused(&b);
|
||||
// Offset tables longer than the file.
|
||||
refused(&good[..tables + 20]);
|
||||
|
||||
for n in 0..good.len() {
|
||||
let _ = decode_part(&good[..n], flat, &all, &limits);
|
||||
let _ = decode_display_part(&good[..n], flat, &all, &|_| 0.0, &limits);
|
||||
}
|
||||
for i in 0..good.len() {
|
||||
for v in [0u8, 1, 0x7F, 0xFF] {
|
||||
let mut b = good.clone();
|
||||
b[i] = v;
|
||||
if let Ok(p) = decode_part(&b, flat, &all, &limits) {
|
||||
let n = p.info.data_window.width as usize * p.info.data_window.height as usize;
|
||||
assert!(p.channels.iter().all(|c| c.samples.len() == n));
|
||||
}
|
||||
let _ = decode_display_part(&b, flat, &all, &|_| 0.0, &limits);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tiled_flat_part_beside_a_deep_part_must_hold_each_of_its_tiles() {
|
||||
let limits = Limits::default();
|
||||
for deep_first in [false, true] {
|
||||
let (good, offsets) = mixed_file(deep_first, true);
|
||||
let flat = usize::from(deep_first);
|
||||
// The second chunk names tile (0, 0) at level (1, 1), which a one-level part does not
|
||||
// have, or tile (0, 0) again, so tile (1, 0) has no chunk.
|
||||
for coordinates in [[0i32, 0, 1, 1], [0, 0, 0, 0]] {
|
||||
let mut b = good.clone();
|
||||
let at = offsets[flat][1] + 4;
|
||||
coordinates.iter().enumerate().for_each(|(i, v)| b[at + i * 4..at + i * 4 + 4].copy_from_slice(&v.to_le_bytes()));
|
||||
for r in [decode_part(&b, flat, &all, &limits).map(|_| ()), decode_display_part(&b, flat, &all, &|_| 0.0, &limits).map(|_| ())] {
|
||||
assert!(matches!(&r, Err(Error::Malformed(_))), "deep first {deep_first} {coordinates:?}: {r:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_scan_line_chunk_beside_a_deep_part_must_start_at_its_block_s_first_line() {
|
||||
// A ZIP flat part (16 lines a block) holds its 3 × 2 pixels in one chunk, stored raw, as a
|
||||
// block is when compressing it does not make it smaller; the deep part follows.
|
||||
let limits = Limits::default();
|
||||
let (w, h) = (3usize, 2usize);
|
||||
let (plain, offsets) = mixed_file(false, false);
|
||||
let deep = &plain[offsets[1][0]..];
|
||||
let with_y = |y: i32| {
|
||||
let mut out = mixed_headers_with(false, false, (w, h), 3, &["B", "G", "R"], 2);
|
||||
let mut chunk = Vec::new();
|
||||
chunk.extend_from_slice(&0i32.to_le_bytes());
|
||||
chunk.extend_from_slice(&y.to_le_bytes());
|
||||
chunk.extend_from_slice(&((w * h * 3 * 4) as i32).to_le_bytes());
|
||||
for row in 0..h {
|
||||
for channel in [100.0, 10.0, 0.0] {
|
||||
(0..w).for_each(|x| chunk.extend_from_slice(&mixed_flat(channel, row * w + x).to_le_bytes()));
|
||||
}
|
||||
}
|
||||
let flat_at = out.len() + 3 * 8;
|
||||
let deep_at = flat_at + chunk.len();
|
||||
for at in [flat_at, deep_at, deep_at + offsets[1][1] - offsets[1][0]] {
|
||||
out.extend_from_slice(&(at as u64).to_le_bytes());
|
||||
}
|
||||
out.extend_from_slice(&chunk);
|
||||
out.extend_from_slice(deep);
|
||||
out
|
||||
};
|
||||
let part = decode_part(&with_y(0), 0, &all, &limits).unwrap();
|
||||
assert_eq!(part.channel("R").unwrap().samples, Samples::F32((0..6).map(|i| mixed_flat(0.0, i)).collect()));
|
||||
// Each of these y values rounds to block 0, but none is that block's first line.
|
||||
for y in [-15, -1, 1, 15] {
|
||||
let b = with_y(y);
|
||||
for r in [decode_part(&b, 0, &all, &limits).map(|_| ()), decode_display_part(&b, 0, &all, &|_| 0.0, &limits).map(|_| ())] {
|
||||
assert!(matches!(&r, Err(Error::Malformed(_))), "y {y}: {r:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A mixed file whose flat part, first, is one pixel of these float channels (each 1.0) in one
|
||||
/// `tile` × `tile` tile, held by a chunk that declares `size` payload bytes.
|
||||
fn mixed_one_tile(channels: &[&str], tile: u32, size: i32) -> Vec<u8> {
|
||||
let (plain, offsets) = mixed_file(false, false);
|
||||
let mut out = mixed_headers_with(false, true, (1, 1), 0, channels, tile);
|
||||
let mut chunk = Vec::new();
|
||||
[0i32, 0, 0, 0, 0, size].iter().for_each(|v| chunk.extend_from_slice(&v.to_le_bytes()));
|
||||
channels.iter().for_each(|_| chunk.extend_from_slice(&1f32.to_le_bytes()));
|
||||
let flat_at = out.len() + 3 * 8;
|
||||
let deep_at = flat_at + chunk.len();
|
||||
for at in [flat_at, deep_at, deep_at + offsets[1][1] - offsets[1][0]] {
|
||||
out.extend_from_slice(&(at as u64).to_le_bytes());
|
||||
}
|
||||
out.extend_from_slice(&chunk);
|
||||
out.extend_from_slice(&plain[offsets[1][0]..]);
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_chunk_beside_a_deep_part_must_fit_the_rest_of_the_file() {
|
||||
// 2^14 × 2^14 tiles of three floats let a chunk hold up to 3 GiB (also on 32-bit), so the crate would
|
||||
// allocate the 2 GiB this chunk declares before it found the file too short.
|
||||
let limits = Limits::default();
|
||||
let channels = ["B", "G", "R"];
|
||||
let good = mixed_one_tile(&channels, 1 << 14, 12);
|
||||
assert_eq!(decode_part(&good, 0, &all, &limits).unwrap().channel("R").unwrap().samples, Samples::F32(vec![1.0]));
|
||||
let b = mixed_one_tile(&channels, 1 << 14, i32::MAX);
|
||||
for r in [decode_part(&b, 0, &all, &limits).map(|_| ()), decode_display_part(&b, 0, &all, &|_| 0.0, &limits).map(|_| ())] {
|
||||
assert!(matches!(&r, Err(Error::Malformed(m)) if m.contains("remain in the file")), "{r:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tile_too_large_to_address_is_refused_before_a_chunk_is_read() {
|
||||
// Five float channels in 10^9 × 10^9 tiles: 20 bytes a pixel times 10^18 pixels does not fit
|
||||
// in a 64-bit size, though the 1 × 1 part passes every size limit.
|
||||
let limits = Limits::default();
|
||||
let channels = ["A", "B", "C", "D", "E"];
|
||||
let tile = 1_000_000_000u32;
|
||||
let mixed = mixed_one_tile(&channels, tile, 20);
|
||||
|
||||
// A file of flat parts only, read by the crate's own reader, with its 4 × 4 tiles widened.
|
||||
let mut flat_only = file(vec![layer(None, 1, 1, &channels, true, exr::prelude::Compression::Uncompressed)], (1, 1));
|
||||
let at = flat_only.windows(9).position(|w| w == b"tiledesc\0").unwrap() + 9 + 4;
|
||||
[tile, tile].iter().enumerate().for_each(|(i, v)| flat_only[at + i * 4..at + i * 4 + 4].copy_from_slice(&v.to_le_bytes()));
|
||||
|
||||
for (name, b) in [("mixed", &mixed), ("flat only", &flat_only)] {
|
||||
let results = [
|
||||
parts(b, &limits).map(|_| ()),
|
||||
decode_part(b, 0, &all, &limits).map(|_| ()),
|
||||
decode_display_part(b, 0, &all, &|_| 0.0, &limits).map(|_| ()),
|
||||
];
|
||||
for r in results {
|
||||
assert!(matches!(&r, Err(Error::LimitExceeded(_))), "{name}: {r:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_flat_parts_of_a_mixed_file_must_fit_the_file() {
|
||||
// 1024 × 1024 × 3 float samples (12 MiB) declared in a file of a few hundred bytes: over a
|
||||
// ratio of 1 plus the 1 MiB slack, though another part is deep.
|
||||
let headers = mixed_headers(false, false, (1024, 1024));
|
||||
assert!(parts(&headers, &Limits::default()).is_ok());
|
||||
let tight = Limits { max_ratio: 1, ..Limits::default() };
|
||||
for r in [parts(&headers, &tight).map(|_| ()), decode_part(&headers, 0, &all, &tight).map(|_| ())] {
|
||||
assert!(matches!(&r, Err(Error::LimitExceeded(_))), "{r:?}");
|
||||
}
|
||||
// The deep part's samples are not counted, and a small flat part passes the same ratio.
|
||||
assert!(parts(&mixed_file(false, false).0, &tight).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deep_samples_do_not_count_against_the_flat_expansion_ratio() {
|
||||
// The 4 MiB deep plane would exceed max_ratio=1 plus the 1 MiB slack if counted.
|
||||
// Header inspection does not allocate those samples; deep decoding bounds its own counts.
|
||||
let headers = mixed_headers_with_deep_size(false, false, (1, 1), 0, &["R"], 2, (1024, 1024));
|
||||
let limits = Limits { max_ratio: 1, max_bytes: 8 * 1024 * 1024, ..Limits::default() };
|
||||
assert!(parts(&headers, &limits).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_display_decode_bounds_each_block_not_the_whole_flat_part() {
|
||||
for (tiled, block_bytes) in [(false, 36), (true, 48)] {
|
||||
for deep_first in [false, true] {
|
||||
let (bytes, _) = mixed_file(deep_first, tiled);
|
||||
let flat = usize::from(deep_first);
|
||||
let keep = |name: &str| name == "flat.R";
|
||||
// The full flat part is 72 bytes; the selected plane is 24 bytes.
|
||||
let limits = Limits { max_bytes: block_bytes, ..Limits::default() };
|
||||
let part = decode_display_part(&bytes, flat, &keep, &|_| 0.0, &limits).unwrap();
|
||||
assert_eq!(part.channels[0].samples, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]);
|
||||
let tight = Limits { max_bytes: block_bytes - 1, ..limits };
|
||||
assert!(matches!(decode_display_part(&bytes, flat, &keep, &|_| 0.0, &tight), Err(Error::LimitExceeded(_))));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn present_packed_payload_is_bounded_before_allocation() {
|
||||
let packed = 1024 * 1024;
|
||||
let mut bytes = mixed_one_tile(&["R"], 1 << 15, packed);
|
||||
// Make the declared payload present, not merely a truncated-file error. The nominal
|
||||
// huge tile allows this size in exr::Chunk::read, but the actual flat image is 1x1.
|
||||
bytes.resize(bytes.len() + packed as usize, 0);
|
||||
let limits = Limits { max_bytes: 4, ..Limits::default() };
|
||||
assert!(matches!(
|
||||
decode_display_part(&bytes, 0, &|_| true, &|_| 0.0, &limits),
|
||||
Err(Error::LimitExceeded(message)) if message.contains("packed flat block")
|
||||
));
|
||||
// decode_part also includes the deep header's 24 bytes in its aggregate header budget.
|
||||
let limits = Limits { max_bytes: 32, ..limits };
|
||||
assert!(matches!(decode_part(&bytes, 0, &all, &limits), Err(Error::LimitExceeded(message)) if message.contains("packed flat block")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cropped_mixed_display_bounds_block_tracking() {
|
||||
let mut bytes = mixed_headers_with(false, true, (100, 100), 0, &["R"], 1);
|
||||
// Both display windows are 1x1; the flat data window still has 10,000 one-pixel tiles.
|
||||
let needle = b"displayWindow\0box2i\0";
|
||||
let positions: Vec<usize> = bytes.windows(needle.len()).enumerate().filter_map(|(i, w)| (w == needle).then_some(i + needle.len() + 4)).collect();
|
||||
for at in positions {
|
||||
bytes[at..at + 16].fill(0);
|
||||
}
|
||||
// Supply all offset-table bytes so validation reaches bookkeeping, not truncation.
|
||||
bytes.resize(bytes.len() + (10_000 + 2) * 8, 0);
|
||||
let limits = Limits { max_bytes: 4, ..Limits::default() };
|
||||
assert!(matches!(
|
||||
decode_display_part(&bytes, 0, &|_| true, &|_| 0.0, &limits),
|
||||
Err(Error::LimitExceeded(message)) if message.contains("block tracking")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_block_ordinals_cover_scan_lines_and_all_tile_levels() {
|
||||
use exr::block::chunk::TileCoordinates;
|
||||
use exr::meta::attribute::LevelMode;
|
||||
for tiled in [false, true] {
|
||||
let bytes = file(vec![layer(None, 9, 7, &["R"], tiled, exr::prelude::Compression::ZIP1)], (9, 7));
|
||||
let mut meta = read_meta(&bytes, &Limits::default()).unwrap();
|
||||
let header = &mut meta.headers[0];
|
||||
for mode in [LevelMode::Singular, LevelMode::MipMap, LevelMode::RipMap] {
|
||||
if let exr::meta::BlockDescription::Tiles(tiles) = &mut header.blocks {
|
||||
tiles.level_mode = mode;
|
||||
}
|
||||
for (i, block) in header.blocks_increasing_y_order().enumerate() {
|
||||
assert_eq!(flat_block_ordinal(header, block.location).unwrap(), i);
|
||||
}
|
||||
let absent = TileCoordinates { tile_index: exr::math::Vec2(0, 0), level_index: exr::math::Vec2(100, 100) };
|
||||
assert!(flat_block_ordinal(header, absent).is_err());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+22
-7
@@ -391,19 +391,14 @@ fn decode_deep_meta(meta: &MetaData, bytes: &[u8], limits: &Limits) -> Result<De
|
||||
let bytes_per_sample: usize = exr_types.iter().map(|t| sample_bytes(*t)).sum();
|
||||
let multipart = meta.requirements.has_multiple_layers;
|
||||
|
||||
// The offset tables follow the headers: `chunk_count` u64 per header, in header order.
|
||||
// Our part's table starts after the tables of all earlier parts.
|
||||
let mut table_pos = header_end(bytes)?;
|
||||
for h_ in meta.headers.iter().take(header_index) {
|
||||
table_pos = table_pos.saturating_add((h_.chunk_count.min(1 << 24) as u64).saturating_mul(8));
|
||||
}
|
||||
// A chunk offset before the end of the last offset table points into table data.
|
||||
let (table_pos, tables_end) = offset_tables(meta, bytes, header_index)?;
|
||||
let mut cur = Cursor::new(bytes);
|
||||
seek_to(&mut cur, table_pos)?;
|
||||
let mut offsets = Vec::with_capacity(header.chunk_count.min(1 << 20));
|
||||
for _ in 0..header.chunk_count {
|
||||
offsets.push(u64(&mut cur)?);
|
||||
}
|
||||
let tables_end = cur.position();
|
||||
|
||||
// Block geometry.
|
||||
let (tiles_x, tile_w, tile_h, slots) = match &header.blocks {
|
||||
@@ -631,6 +626,26 @@ fn header_end(bytes: &[u8]) -> Result<u64> {
|
||||
Ok(cur.position())
|
||||
}
|
||||
|
||||
/// Where part `header_index`'s offset table starts and where the last part's table ends. The
|
||||
/// tables follow the headers, `chunk_count` u64 per header in header order, and the chunks
|
||||
/// follow the last table. Sizes that overflow, or tables that run past the end of the file,
|
||||
/// are errors.
|
||||
fn offset_tables(meta: &MetaData, bytes: &[u8], header_index: usize) -> Result<(u64, u64)> {
|
||||
let mut table_pos = header_end(bytes)?;
|
||||
let mut tables_end = table_pos;
|
||||
for (i, h) in meta.headers.iter().enumerate() {
|
||||
let len = u64::try_from(h.chunk_count).ok().and_then(|n| n.checked_mul(8));
|
||||
tables_end = len.and_then(|len| tables_end.checked_add(len)).ok_or_else(|| err("the offset tables are too large"))?;
|
||||
if i < header_index {
|
||||
table_pos = tables_end;
|
||||
}
|
||||
}
|
||||
if tables_end > bytes.len() as u64 {
|
||||
return Err(err(format!("the offset tables end at byte {tables_end}, past the end of the file")));
|
||||
}
|
||||
Ok((table_pos, tables_end))
|
||||
}
|
||||
|
||||
/// Composite the deep samples into a flat image: per pixel, samples sorted by Z (stable; the
|
||||
/// file is assumed messy), front-to-back `over` with premultiplied colour, un-premultiplied
|
||||
/// into straight alpha at the end. Without any alpha channel the front-most sample wins,
|
||||
|
||||
@@ -886,6 +886,106 @@ fn inconsistent_count_table_errors() {
|
||||
assert!(matches!(decode_deep_exr(&bytes, &Limits::none()), Err(CodecError::Malformed(_))));
|
||||
}
|
||||
|
||||
/// A two-part file made from the single-part deep file `deep` and a 1×1 flat scan-line part,
|
||||
/// the deep part first when `deep_first`: each header gains `name` and `chunkCount`, the
|
||||
/// version field the multipart bit, and each chunk its part number. The offset tables follow
|
||||
/// the headers in header order, then the chunks. Returns the file, where the first table
|
||||
/// starts, where the second table starts and where the last table ends.
|
||||
fn two_part(deep: &[u8], deep_first: bool) -> (Vec<u8>, usize, usize, usize) {
|
||||
let end = header_end_of(deep);
|
||||
let first = u64::from_le_bytes(deep[end..end + 8].try_into().expect("8")) as usize;
|
||||
let offsets: Vec<usize> =
|
||||
(0..(first - end) / 8).map(|i| u64::from_le_bytes(deep[end + 8 * i..end + 8 * i + 8].try_into().expect("8")) as usize).collect();
|
||||
let deep_chunks: Vec<&[u8]> = offsets.iter().enumerate().map(|(i, &o)| &deep[o..offsets.get(i + 1).copied().unwrap_or(deep.len())]).collect();
|
||||
|
||||
let mut deep_header = deep[8..end - 1].to_vec();
|
||||
attr(&mut deep_header, "name", "string", b"deep");
|
||||
attr(&mut deep_header, "chunkCount", "int", &(deep_chunks.len() as i32).to_le_bytes());
|
||||
let mut flat_header = Vec::new();
|
||||
let mut chlist = b"R\0".to_vec();
|
||||
chlist.extend_from_slice(&1i32.to_le_bytes()); // half
|
||||
chlist.extend_from_slice(&[0u8; 4]); // pLinear, reserved
|
||||
chlist.extend_from_slice(&1i32.to_le_bytes());
|
||||
chlist.extend_from_slice(&1i32.to_le_bytes());
|
||||
chlist.push(0);
|
||||
attr(&mut flat_header, "channels", "chlist", &chlist);
|
||||
attr(&mut flat_header, "compression", "compression", &[0]);
|
||||
let dw: Vec<u8> = [0i32; 4].iter().flat_map(|v| v.to_le_bytes()).collect();
|
||||
attr(&mut flat_header, "dataWindow", "box2i", &dw);
|
||||
attr(&mut flat_header, "displayWindow", "box2i", &dw);
|
||||
attr(&mut flat_header, "lineOrder", "lineOrder", &[0]);
|
||||
attr(&mut flat_header, "pixelAspectRatio", "float", &1f32.to_le_bytes());
|
||||
attr(&mut flat_header, "screenWindowCenter", "v2f", &[0f32.to_le_bytes(), 0f32.to_le_bytes()].concat());
|
||||
attr(&mut flat_header, "screenWindowWidth", "float", &1f32.to_le_bytes());
|
||||
attr(&mut flat_header, "name", "string", b"flat");
|
||||
attr(&mut flat_header, "type", "string", b"scanlineimage");
|
||||
attr(&mut flat_header, "chunkCount", "int", &1i32.to_le_bytes());
|
||||
// The flat part's one chunk: y, the data size, one half sample.
|
||||
let mut flat_chunk = 0i32.to_le_bytes().to_vec();
|
||||
flat_chunk.extend_from_slice(&2i32.to_le_bytes());
|
||||
flat_chunk.extend_from_slice(&f16::from_f32(0.5).to_le_bytes());
|
||||
let flat_chunks: Vec<&[u8]> = vec![&flat_chunk];
|
||||
|
||||
let parts: [(&[u8], &[&[u8]]); 2] = if deep_first {
|
||||
[(&deep_header, &deep_chunks), (&flat_header, &flat_chunks)]
|
||||
} else {
|
||||
[(&flat_header, &flat_chunks), (&deep_header, &deep_chunks)]
|
||||
};
|
||||
let mut out = deep[..4].to_vec();
|
||||
out.extend_from_slice(&(2u32 | 0x800 | 0x1000).to_le_bytes());
|
||||
for (header, _) in &parts {
|
||||
out.extend_from_slice(header);
|
||||
out.push(0); // end of this header
|
||||
}
|
||||
out.push(0); // end of the headers
|
||||
let first_table = out.len();
|
||||
let second_table = first_table + parts[0].1.len() * 8;
|
||||
let tables_end = second_table + parts[1].1.len() * 8;
|
||||
let mut pos = tables_end;
|
||||
for (_, chunks) in &parts {
|
||||
for c in chunks.iter() {
|
||||
out.extend_from_slice(&(pos as u64).to_le_bytes());
|
||||
pos += 4 + c.len();
|
||||
}
|
||||
}
|
||||
for (part, (_, chunks)) in parts.iter().enumerate() {
|
||||
for c in chunks.iter() {
|
||||
out.extend_from_slice(&(part as i32).to_le_bytes());
|
||||
out.extend_from_slice(c);
|
||||
}
|
||||
}
|
||||
(out, first_table, second_table, tables_end)
|
||||
}
|
||||
|
||||
/// The deep part of a two-part file decodes as the single-part file does, first or second.
|
||||
/// Its chunks start after every part's offset table: a chunk offset into the other part's
|
||||
/// table, or one byte before the end of the last table, is refused as table data, and a file
|
||||
/// that ends inside the last table is refused even when the deep part's own table is whole.
|
||||
#[test]
|
||||
fn multipart_deep_chunks_start_after_every_offset_table() {
|
||||
let single = gen_deep(4, 4, &RGBAZ, 0, None, COUNTS_ONE, &sample_rgba);
|
||||
let want = decode_deep_exr(&single, &Limits::none()).expect("single part");
|
||||
for deep_first in [true, false] {
|
||||
let (bytes, ..) = two_part(&single, deep_first);
|
||||
let deep = decode_deep_exr(&bytes, &Limits::none()).expect("two parts");
|
||||
assert_eq!(deep.counts, want.counts);
|
||||
assert_eq!(deep.channel("Z").expect("Z").samples, want.channel("Z").expect("Z").samples);
|
||||
}
|
||||
|
||||
let refused = |bytes: &[u8], what: &str| match decode_deep_exr(bytes, &Limits::none()) {
|
||||
Err(e @ CodecError::Malformed(_)) => assert!(e.to_string().contains(what), "{e}"),
|
||||
Err(e) => panic!("{e}, want an error about {what}"),
|
||||
Ok(_) => panic!("decoded, want an error about {what}"),
|
||||
};
|
||||
let (bytes, first_table, second_table, tables_end) = two_part(&single, true);
|
||||
for off in [second_table, tables_end - 1] {
|
||||
let mut into_table = bytes.clone();
|
||||
into_table[first_table..first_table + 8].copy_from_slice(&(off as u64).to_le_bytes());
|
||||
refused(&into_table, "outside the chunk area");
|
||||
}
|
||||
refused(&bytes[..tables_end - 4], "past the end of the file");
|
||||
}
|
||||
|
||||
/// `gen_deep` for a cropped render: the data window sits at `(dwx, dwy)` inside a larger
|
||||
/// display window of `sw × sh` (the stage the renderer composed onto). Scanline chunk Y
|
||||
/// coordinates are absolute display rows, so they carry the data-window offset.
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
//! Shared ownership of a font file and a face within it.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Font file bytes and the index of a face in that file.
|
||||
///
|
||||
/// Clones share owned bytes, so collection faces and variable instances do not copy the file.
|
||||
/// Static bytes let applications use embedded fonts without allocating a copy.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FaceData {
|
||||
bytes: Bytes,
|
||||
index: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum Bytes {
|
||||
Static(&'static [u8]),
|
||||
Owned(Arc<Vec<u8>>),
|
||||
Shared(Arc<[u8]>),
|
||||
}
|
||||
|
||||
impl FaceData {
|
||||
/// A face in statically embedded font bytes.
|
||||
pub fn from_static(bytes: &'static [u8], index: u32) -> Self {
|
||||
Self { bytes: Bytes::Static(bytes), index }
|
||||
}
|
||||
|
||||
/// A face in a shared vector, without copying its allocation.
|
||||
pub fn from_owned(bytes: Arc<Vec<u8>>, index: u32) -> Self {
|
||||
Self { bytes: Bytes::Owned(bytes), index }
|
||||
}
|
||||
|
||||
/// A face in bytes shared by a catalog.
|
||||
pub fn from_shared(bytes: Arc<[u8]>, index: u32) -> Self {
|
||||
Self { bytes: Bytes::Shared(bytes), index }
|
||||
}
|
||||
|
||||
/// The entire font file.
|
||||
pub fn bytes(&self) -> &[u8] {
|
||||
match &self.bytes {
|
||||
Bytes::Static(bytes) => bytes,
|
||||
Bytes::Owned(bytes) => bytes.as_slice(),
|
||||
Bytes::Shared(bytes) => bytes,
|
||||
}
|
||||
}
|
||||
|
||||
/// The index of this face in the file.
|
||||
pub fn index(&self) -> u32 {
|
||||
self.index
|
||||
}
|
||||
|
||||
/// Parse this face for font metadata and outlines.
|
||||
pub fn skrifa(&self) -> Option<skrifa::FontRef<'_>> {
|
||||
skrifa::FontRef::from_index(self.bytes(), self.index).ok()
|
||||
}
|
||||
|
||||
/// Parse this face for shaping.
|
||||
pub fn harfrust(&self) -> Option<harfrust::FontRef<'_>> {
|
||||
harfrust::FontRef::from_index(self.bytes(), self.index).ok()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<crate::FontData> for FaceData {
|
||||
fn from(data: crate::FontData) -> Self {
|
||||
Self::from_shared(data.bytes, data.index)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn sources_keep_bytes_and_face_index_without_copying() {
|
||||
static DATA: &[u8] = b"not a font";
|
||||
let owned = Arc::new(DATA.to_vec());
|
||||
let shared: Arc<[u8]> = Arc::from(DATA);
|
||||
for data in [FaceData::from_static(DATA, 2), FaceData::from_owned(owned.clone(), 2), FaceData::from_shared(shared.clone(), 2)] {
|
||||
let clone = data.clone();
|
||||
assert_eq!(clone.bytes(), DATA);
|
||||
assert_eq!(clone.bytes().as_ptr(), data.bytes().as_ptr());
|
||||
assert_eq!(clone.index(), 2);
|
||||
assert!(clone.skrifa().is_none());
|
||||
assert!(clone.harfrust().is_none());
|
||||
}
|
||||
assert_eq!(Arc::strong_count(&owned), 1);
|
||||
assert_eq!(Arc::strong_count(&shared), 1);
|
||||
}
|
||||
}
|
||||
@@ -23,11 +23,13 @@
|
||||
mod catalog;
|
||||
mod embedding;
|
||||
mod env;
|
||||
mod face_data;
|
||||
mod fallback;
|
||||
mod fontconfig;
|
||||
mod ligature;
|
||||
mod line_break;
|
||||
mod outline;
|
||||
mod plan_cache;
|
||||
mod registry;
|
||||
mod scan;
|
||||
mod segments;
|
||||
@@ -41,11 +43,13 @@ mod visual_order;
|
||||
pub use catalog::{Catalog, FontData, FontFileError, Limits, Lister, LoadedFace, Match, Watch};
|
||||
pub use embedding::{SubsetEmbedding, subset_embedding};
|
||||
pub use env::{FontEnv, Os, font_dirs, font_store_dir, office_font_dirs};
|
||||
pub use face_data::FaceData;
|
||||
pub use fallback::{cluster_face, drawable_characters};
|
||||
pub use fontconfig::fontconfig_dirs;
|
||||
pub use ligature::ligature_caret_stops;
|
||||
pub use line_break::settle_line_break;
|
||||
pub use outline::{FontOutline, GlyphPen, OutlineError};
|
||||
pub use plan_cache::ShapePlanCache;
|
||||
pub use scan::{FaceInfo, LocalizedName, Outlines, Source, is_font_file, style_italic, style_weight, style_width};
|
||||
pub use segments::SegmentBoundary;
|
||||
pub use shape::{RunGlyph, RunInput, ShapeRunError, shape_run};
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
//! Per-face bounded shaping plan reuse.
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
/// A cache for one font face and variation instance. Do not share it across faces or instances.
|
||||
/// Plans are created outside the lock, then checked again to avoid retaining duplicates.
|
||||
#[derive(Default)]
|
||||
pub struct ShapePlanCache {
|
||||
plans: Mutex<Vec<Arc<harfrust::ShapePlan>>>,
|
||||
}
|
||||
|
||||
impl ShapePlanCache {
|
||||
/// Maximum number of plans retained; the oldest is evicted when this is reached.
|
||||
pub const CAPACITY: usize = 32;
|
||||
|
||||
/// Returns a matching plan or builds and retains one for this face.
|
||||
pub fn get_or_create(
|
||||
&self,
|
||||
shaper: &harfrust::Shaper<'_>,
|
||||
instance: Option<&harfrust::ShaperInstance>,
|
||||
direction: harfrust::Direction,
|
||||
script: Option<harfrust::Script>,
|
||||
language: Option<&harfrust::Language>,
|
||||
features: &[harfrust::Feature],
|
||||
) -> Arc<harfrust::ShapePlan> {
|
||||
let key = harfrust::ShapePlanKey::new(script, direction).language(language).instance(instance).features(features);
|
||||
let kept = |plans: &[Arc<harfrust::ShapePlan>]| plans.iter().rev().find(|p| key.matches(p)).cloned();
|
||||
if let Some(plan) = kept(&self.plans.lock().unwrap_or_else(|e| e.into_inner())) {
|
||||
return plan;
|
||||
}
|
||||
let plan = Arc::new(harfrust::ShapePlan::new(shaper, direction, script, language, features));
|
||||
let mut plans = self.plans.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if let Some(made) = kept(&plans) {
|
||||
return made;
|
||||
}
|
||||
if plans.len() >= Self::CAPACITY {
|
||||
plans.remove(0);
|
||||
}
|
||||
plans.push(plan.clone());
|
||||
plan
|
||||
}
|
||||
|
||||
/// Number of retained plans.
|
||||
pub fn len(&self) -> usize {
|
||||
self.plans.lock().unwrap_or_else(|e| e.into_inner()).len()
|
||||
}
|
||||
|
||||
/// Whether this cache contains no plans.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
}
|
||||
@@ -351,6 +351,25 @@ fn a_read_is_capped_where_the_file_has_grown() {
|
||||
let _ = std::fs::remove_file(&path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shape_plan_cache_reuses_and_evicts_plans() {
|
||||
let data = font(&spec("Test", "Regular", 400));
|
||||
let face = harfrust::FontRef::from_index(&data, 0).unwrap();
|
||||
let shaper_data = harfrust::ShaperData::new(&face);
|
||||
let shaper = shaper_data.shaper(&face).build();
|
||||
let cache = ShapePlanCache::default();
|
||||
let make = |language: &harfrust::Language| cache.get_or_create(&shaper, None, harfrust::Direction::LeftToRight, None, Some(language), &[]);
|
||||
let languages: Vec<harfrust::Language> = ('a'..='z').flat_map(|a| ['a', 'b'].map(move |b| format!("{a}{b}").parse().unwrap())).collect();
|
||||
let first = make(&languages[0]);
|
||||
assert!(Arc::ptr_eq(&first, &make(&languages[0])));
|
||||
for language in languages.iter().skip(1) {
|
||||
make(language);
|
||||
}
|
||||
assert_eq!(cache.len(), ShapePlanCache::CAPACITY);
|
||||
assert!(!Arc::ptr_eq(&first, &make(&languages[0])));
|
||||
assert_eq!(cache.len(), ShapePlanCache::CAPACITY);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_sfnt_reader_preserves_face_names_and_collection_indices() {
|
||||
let mut bold = spec("Example", "Bold Italic", 700);
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "craft-format"
|
||||
description = "Versioned JSON file formats: the format tag, the registry of versions (hand-written steps, then one folder of changes per version, then the pending changes), the upgrade of an older file and the telling of a newer one"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde_json = { workspace = true }
|
||||
# The container: content-addressed blobs (SHA-256) in a ZIP package.
|
||||
sha2 = { version = "0.10", default-features = false, features = ["std"] }
|
||||
craft-zip = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = { workspace = true }
|
||||
@@ -0,0 +1,237 @@
|
||||
//! For a build script: the generated lists of a family's changes. Each change is one `.rs` file in
|
||||
//! the family's folder, in `v<N>/` once a release fixed its number, else in `pending/`; the lists
|
||||
//! are generated so that a change adds a file and never edits a line another change also edits.
|
||||
//!
|
||||
//! The generated code, for the app to `include!`, is per family a `FOLDED` list (each folded
|
||||
//! format and its parts) and a `PENDING` list, both prefixed by the family's `list_prefix`, then a
|
||||
//! module per change file. It names the app's `Part` type, which the including module has in scope
|
||||
//! (`type Part = craft_format::Part<AppError>;`), and a change file's step as `step` (a
|
||||
//! `pub fn step(v: &mut Value) -> Result<(), AppError>`, when the file has one).
|
||||
|
||||
use std::fmt::Write as _;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// A family's change files: their folder (relative to the crate's manifest), the last format
|
||||
/// before them, and the prefix of the family's lists.
|
||||
pub struct Family {
|
||||
pub folder: &'static str,
|
||||
pub base: u32,
|
||||
pub list_prefix: &'static str,
|
||||
}
|
||||
|
||||
/// What [`lists`] made: the code to write, and the folders the build script reruns on.
|
||||
pub struct Lists {
|
||||
pub code: String,
|
||||
pub watch: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
/// The `FOLDED` and `PENDING` lists of every family, and a module for each change file. Refuses
|
||||
/// versions that are not contiguous from a family's base, a change file not named with lowercase
|
||||
/// letters, digits and dashes, and one that doesn't start with a `//!` note.
|
||||
pub fn lists(manifest: &Path, families: &[Family]) -> Result<Lists, String> {
|
||||
let mut code = String::new();
|
||||
let mut mods = String::new();
|
||||
let mut watch = Vec::new();
|
||||
for family in families {
|
||||
let root = manifest.join(family.folder);
|
||||
family_lists(family, &root, &mut code, &mut mods)?;
|
||||
watch.push(root);
|
||||
}
|
||||
code.push_str(&mods);
|
||||
Ok(Lists { code, watch })
|
||||
}
|
||||
|
||||
/// A family's `FOLDED` and `PENDING` lists; its folded formats follow on from its base.
|
||||
fn family_lists(family: &Family, root: &Path, out: &mut String, mods: &mut String) -> Result<(), String> {
|
||||
let Family { folder, base, list_prefix } = family;
|
||||
let mut folded: Vec<(u32, PathBuf)> = Vec::new();
|
||||
for entry in std::fs::read_dir(root).map_err(|e| format!("{}: {e}", root.display()))? {
|
||||
// an entry that can't be read could be a version folder: fail rather than drop it
|
||||
let path = entry.map_err(|e| format!("{}: {e}", root.display()))?.path();
|
||||
let version = path.file_name().and_then(|n| n.to_str()).and_then(|n| n.strip_prefix('v')).and_then(|n| n.parse().ok());
|
||||
if let Some(version) = version
|
||||
&& path.is_dir()
|
||||
{
|
||||
folded.push((version, path));
|
||||
}
|
||||
}
|
||||
folded.sort();
|
||||
for (i, (version, _)) in folded.iter().enumerate() {
|
||||
let expected = base + 1 + i as u32;
|
||||
if *version != expected {
|
||||
return Err(format!("{folder}/: expected v{expected}/ next, found v{version}/ (versions are contiguous)"));
|
||||
}
|
||||
}
|
||||
// Module names start with the family's (`drawing_pending_…`), so two families' changes of one
|
||||
// name never clash.
|
||||
let module = list_prefix.to_ascii_lowercase();
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"/// The changes of each format after the base, oldest first (`{folder}/v<N>/`).\nconst {list_prefix}FOLDED: &[(u32, &[Part])] = &["
|
||||
);
|
||||
for (version, dir) in &folded {
|
||||
let _ = writeln!(out, " ({version}, &[");
|
||||
parts(dir, &format!("{module}v{version}"), false, out, mods)?;
|
||||
out.push_str(" ]),\n");
|
||||
}
|
||||
let _ = writeln!(out, "];\n\n/// The changes no release has a number for yet (`{folder}/pending/`).\nconst {list_prefix}PENDING: &[Part] = &[");
|
||||
parts(&root.join("pending"), &format!("{module}pending"), true, out, mods)?;
|
||||
out.push_str("];\n\n");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether a change file declares its `step` function (`fn step(`, `pub fn step (` …) on a line of
|
||||
/// code: a mention in a comment or a note is not a declaration.
|
||||
fn declares_step(text: &str) -> bool {
|
||||
let code = without_comments(text);
|
||||
let ident = |c: char| c.is_alphanumeric() || c == '_';
|
||||
// `fn`, whitespace, `step`, then `(` (or generics), across lines
|
||||
code.match_indices("fn").any(|(at, _)| {
|
||||
let before_ok = code[..at].chars().next_back().is_none_or(|c| !ident(c));
|
||||
let rest = &code[at + 2..];
|
||||
before_ok
|
||||
&& rest.starts_with(char::is_whitespace)
|
||||
&& rest
|
||||
.trim_start()
|
||||
.strip_prefix("step")
|
||||
.is_some_and(|rest| matches!(rest.trim_start().chars().next(), Some('(' | '<')) && !rest.starts_with(ident))
|
||||
})
|
||||
}
|
||||
|
||||
/// `text` as code only: its `//` and (nested) `/* */` comments become a space, and the contents
|
||||
/// of its string literals (plain, byte and raw) are dropped, line breaks kept throughout. A comment
|
||||
/// marker inside a string is not a comment, and `fn/* note */step` still reads as `fn step`.
|
||||
fn without_comments(text: &str) -> String {
|
||||
let chars: Vec<char> = text.chars().collect();
|
||||
let at = |i: usize| chars.get(i).copied();
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let (mut i, mut depth) = (0usize, 0usize);
|
||||
while let Some(c) = at(i) {
|
||||
let next = at(i + 1);
|
||||
if depth > 0 {
|
||||
match (c, next) {
|
||||
('/', Some('*')) => (depth, i) = (depth + 1, i + 2),
|
||||
('*', Some('/')) => (depth, i) = (depth - 1, i + 2),
|
||||
_ => {
|
||||
if c == '\n' {
|
||||
out.push('\n');
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
if depth == 0 {
|
||||
out.push(' ');
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let ident_before = i > 0 && at(i - 1).is_some_and(|p| p.is_alphanumeric() || p == '_');
|
||||
match (c, next) {
|
||||
('/', Some('/')) => {
|
||||
while at(i).is_some_and(|c| c != '\n') {
|
||||
i += 1;
|
||||
}
|
||||
out.push(' ');
|
||||
}
|
||||
('/', Some('*')) => (depth, i) = (1, i + 2),
|
||||
('"', _) => {
|
||||
out.push('"');
|
||||
i += 1;
|
||||
while let Some(c) = at(i) {
|
||||
i += 1;
|
||||
match c {
|
||||
'\\' => i += 1,
|
||||
'"' => break,
|
||||
'\n' => out.push('\n'),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
out.push('"');
|
||||
}
|
||||
// a raw string, `r"…"` or `r#"…"#`, after a byte prefix or on its own
|
||||
('r', Some('"' | '#'))
|
||||
if !ident_before || (i > 0 && at(i - 1) == Some('b') && !(i > 1 && at(i - 2).is_some_and(|p| p.is_alphanumeric() || p == '_'))) =>
|
||||
{
|
||||
let mut j = i + 1;
|
||||
let mut hashes = 0;
|
||||
while at(j) == Some('#') {
|
||||
hashes += 1;
|
||||
j += 1;
|
||||
}
|
||||
if at(j) != Some('"') {
|
||||
out.push(c);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
out.push('"');
|
||||
i = j + 1;
|
||||
while let Some(c) = at(i) {
|
||||
if c == '"' && (1..=hashes).all(|k| at(i + k) == Some('#')) {
|
||||
i += 1 + hashes;
|
||||
break;
|
||||
}
|
||||
if c == '\n' {
|
||||
out.push('\n');
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
out.push('"');
|
||||
}
|
||||
// a character literal (`'"'`, `'\''`), not a lifetime
|
||||
('\'', Some('\\')) => {
|
||||
i += 2;
|
||||
while at(i).is_some_and(|c| c != '\'') {
|
||||
i += 1;
|
||||
}
|
||||
i += 1;
|
||||
out.push_str("' '");
|
||||
}
|
||||
('\'', Some(_)) if at(i + 2) == Some('\'') => {
|
||||
i += 3;
|
||||
out.push_str("' '");
|
||||
}
|
||||
_ => {
|
||||
out.push(c);
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The `Part`s of the change files in `dir`, in file-name order, and a module for each. A missing
|
||||
/// folder is an empty list only where it may be absent (`optional`: `pending/`); any other error
|
||||
/// reading it fails the build rather than leave a change out of the lists.
|
||||
fn parts(dir: &Path, prefix: &str, optional: bool, out: &mut String, mods: &mut String) -> Result<(), String> {
|
||||
let entries = match std::fs::read_dir(dir) {
|
||||
Ok(entries) => entries,
|
||||
Err(e) if optional && e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(e) => return Err(format!("{}: {e}", dir.display())),
|
||||
};
|
||||
let mut files: Vec<PathBuf> = Vec::new();
|
||||
for entry in entries {
|
||||
let path = entry.map_err(|e| format!("{}: {e}", dir.display()))?.path();
|
||||
if path.extension().is_some_and(|x| x == "rs") {
|
||||
files.push(path);
|
||||
}
|
||||
}
|
||||
files.sort();
|
||||
for path in files {
|
||||
let name = path.file_stem().and_then(|s| s.to_str()).unwrap_or_default().to_string();
|
||||
if name.is_empty() || !name.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') {
|
||||
return Err(format!("{}: name a format change file with lowercase letters, digits and dashes", path.display()));
|
||||
}
|
||||
let text = std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
if !text.trim_start().starts_with("//!") {
|
||||
return Err(format!("{}: start a format change with a `//!` note", path.display()));
|
||||
}
|
||||
let module = format!("{prefix}_{}", name.replace('-', "_"));
|
||||
let step = if declares_step(&text) { format!("Some({module}::step)") } else { "None".into() };
|
||||
let _ = writeln!(out, " Part {{ name: {name:?}, step: {step} }},");
|
||||
let _ = writeln!(mods, "#[path = {:?}]\nmod {module};", path.display().to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "build_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,115 @@
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use super::*;
|
||||
|
||||
/// A fresh, empty folder for one test.
|
||||
fn scratch(name: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("craft-format-build-{}-{name}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir
|
||||
}
|
||||
|
||||
fn write(path: &Path, text: &str) {
|
||||
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
std::fs::write(path, text).unwrap();
|
||||
}
|
||||
|
||||
/// The error of a run that must fail.
|
||||
fn refused(dir: &Path) -> String {
|
||||
match lists(dir, &[DRAWING]) {
|
||||
Ok(_) => panic!("expected an error"),
|
||||
Err(e) => e,
|
||||
}
|
||||
}
|
||||
|
||||
const DRAWING: Family = Family { folder: "drawing", base: 3, list_prefix: "DRAWING_" };
|
||||
|
||||
#[test]
|
||||
fn lists_folded_and_pending_changes_with_their_modules() {
|
||||
let dir = scratch("ok");
|
||||
write(&dir.join("drawing/v4/add-layers.rs"), "//! Layers.\npub fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n");
|
||||
write(&dir.join("drawing/v4/notes.txt"), "not a change");
|
||||
write(&dir.join("drawing/v5/rename.rs"), "\n//! A rename with no step.\n");
|
||||
write(&dir.join("drawing/pending/next-1.rs"), "//! Pending.\nfn step(v: &mut Value) {}\n");
|
||||
std::fs::create_dir_all(dir.join("drawing/v6x")).unwrap();
|
||||
write(&dir.join("drawing/v7"), "a file, not a version folder");
|
||||
let lists = lists(&dir, &[DRAWING]).unwrap();
|
||||
assert_eq!(lists.watch, vec![dir.join("drawing")]);
|
||||
let code = &lists.code;
|
||||
assert!(code.contains("const DRAWING_FOLDED: &[(u32, &[Part])] = &["), "{code}");
|
||||
assert!(code.contains(" (4, &[\n Part { name: \"add-layers\", step: Some(drawing_v4_add_layers::step) },\n ]),"), "{code}");
|
||||
assert!(code.contains("Part { name: \"rename\", step: None }"), "{code}");
|
||||
assert!(
|
||||
code.contains("const DRAWING_PENDING: &[Part] = &[\n Part { name: \"next-1\", step: Some(drawing_pending_next_1::step) },"),
|
||||
"{code}"
|
||||
);
|
||||
assert!(code.contains("mod drawing_v5_rename;"), "{code}");
|
||||
assert!(!code.contains("notes"), "{code}");
|
||||
assert!(code.find("const DRAWING_PENDING").unwrap() < code.find("#[path").unwrap(), "modules follow the lists");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_family_has_empty_lists() {
|
||||
let dir = scratch("empty");
|
||||
std::fs::create_dir_all(dir.join("drawing")).unwrap();
|
||||
let code = lists(&dir, &[DRAWING]).unwrap().code;
|
||||
assert!(code.contains("const DRAWING_FOLDED: &[(u32, &[Part])] = &[\n];"), "{code}");
|
||||
assert!(code.contains("const DRAWING_PENDING: &[Part] = &[\n];"), "{code}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refuses_gaps_bad_names_missing_notes_and_missing_folders() {
|
||||
let dir = scratch("bad");
|
||||
assert!(refused(&dir).contains("drawing"));
|
||||
write(&dir.join("drawing/v5/a.rs"), "//! a");
|
||||
assert!(refused(&dir).contains("expected v4/ next, found v5/"));
|
||||
std::fs::rename(dir.join("drawing/v5"), dir.join("drawing/v4")).unwrap();
|
||||
write(&dir.join("drawing/v4/Bad_Name.rs"), "//! b");
|
||||
assert!(refused(&dir).contains("lowercase letters"));
|
||||
std::fs::remove_file(dir.join("drawing/v4/Bad_Name.rs")).unwrap();
|
||||
write(&dir.join("drawing/pending/no-note.rs"), "pub fn step() {}");
|
||||
assert!(refused(&dir).contains("`//!` note"));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_step_is_a_declaration_not_a_mention() {
|
||||
assert!(!declares_step("//! No fn step() is needed.\n"));
|
||||
assert!(!declares_step("//! a note\n// fn step(v) was removed\nconst X: u8 = 1;\n"));
|
||||
assert!(declares_step("//! a note\npub fn step (v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
|
||||
// a declaration inside a block comment, nested or not, is not one
|
||||
assert!(!declares_step("//! Documentation only.\n/*\nfn step() {}\n*/\n"));
|
||||
assert!(!declares_step("/* outer /* inner */\nfn step() {}\n*/\nconst X: u8 = 1;\n"));
|
||||
assert!(declares_step("/* a note */\nfn step(v: &mut Value) {}\n"));
|
||||
// comment markers inside strings and character literals are not comments
|
||||
assert!(declares_step("//! A change with a step.\nconst MARKER: &str = \"/*\";\npub fn step(_v: &mut Value) -> Result { Ok(()) }\n"));
|
||||
assert!(declares_step("const A: &str = r#\"/* \"quoted\" \"#;\nconst B: &[u8] = br\"/*\";\nfn step(v: &mut Value) {}\n"));
|
||||
assert!(declares_step("const Q: char = '\"';\nconst E: char = '\\'';\nfn step(v: &mut Value) { let _: &'static str = \"\"; }\n"));
|
||||
assert!(declares_step("const S: &str = \"a \\\" /* b\";\nfn step(v: &mut Value) {}\n"));
|
||||
assert!(!declares_step("const S: &str = \"\nfn step(v) \";\n"), "inside a string");
|
||||
// a comment between the words is whitespace
|
||||
assert!(declares_step("pub fn/* migration */step(v: &mut Value) {}\n"));
|
||||
// across lines
|
||||
assert!(declares_step("//! A change with a step.\npub fn step\n (_v: &mut Value) -> Result {\n Ok(())\n}\n"));
|
||||
assert!(declares_step("pub fn\nstep(v: &mut Value) {}\n"));
|
||||
assert!(!declares_step("fn stepper(v: &mut Value) {}\nfn prefn step() {}\n"), "other names");
|
||||
assert!(declares_step("const X: u8 = 1; // fn step() is below\nfn step(v: &mut Value) {}\n"));
|
||||
assert!(declares_step("//! a note\npub(crate) fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
|
||||
assert!(declares_step("//! a note\n fn step(v: &mut Value) {}\n"));
|
||||
assert!(!declares_step("//! a note\nfn stepper(v: &mut Value) {}\nfn fnstep() {}\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_documentation_only_change_has_no_step_and_a_missing_version_folder_fails() {
|
||||
let dir = scratch("doc-only");
|
||||
write(&dir.join("drawing/v4/doc.rs"), "//! No fn step() is needed.\n");
|
||||
let code = lists(&dir, &[DRAWING]).unwrap().code;
|
||||
assert!(code.contains("Part { name: \"doc\", step: None }"), "{code}");
|
||||
// a version folder that can't be listed is an error, not an empty format
|
||||
let missing = parts(&dir.join("drawing/v9"), "x", false, &mut String::new(), &mut String::new());
|
||||
assert!(missing.is_err());
|
||||
assert!(parts(&dir.join("drawing/pending"), "x", true, &mut String::new(), &mut String::new()).is_ok());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
@@ -0,0 +1,440 @@
|
||||
//! The file container: a ZIP package of named parts (the file's JSON, binary payloads, optional
|
||||
//! caches), never extracted to filesystem paths. Payloads are content-addressed blobs,
|
||||
//! `blobs/<sha256>.<extension>`, stored once however often they are referred to. Every size is
|
||||
//! bounded by the app's [`Limits`]: the file, each part, all parts expanded, the payloads resolved,
|
||||
//! and the number of entries; a part is read no further than its declared size (a decompression
|
||||
//! bomb stops there). What a part holds, and how a payload is encoded, is the app's.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::FormatError;
|
||||
|
||||
/// How hard the payloads are compressed: Fast favours interactive file operations; Compact trades
|
||||
/// CPU time for space.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum Compression {
|
||||
#[default]
|
||||
Fast,
|
||||
Compact,
|
||||
}
|
||||
|
||||
/// An app's budgets for its containers. The functions are read each time they are needed, so a
|
||||
/// limit the app lets the user raise takes effect at once.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct Limits {
|
||||
/// The most entries a container holds.
|
||||
pub max_entries: usize,
|
||||
/// The most bytes all parts may expand to, and all payloads may resolve to.
|
||||
pub max_expanded: fn() -> usize,
|
||||
/// The most bytes a part of this name may expand to.
|
||||
pub part_limit: fn(&str) -> usize,
|
||||
/// The Deflate level of a compressed part of this name and size.
|
||||
pub level: fn(&str, usize) -> i64,
|
||||
/// Whether a container of this many bytes may be read, or why not.
|
||||
pub check_size: fn(u64) -> Result<(), String>,
|
||||
/// Whether the work is cancelled (a save the person stopped): checked before each part, and
|
||||
/// a long part is written a piece at a time and stops before the next piece when this says
|
||||
/// so ([`FormatError::Cancelled`]). `None`: it cannot be cancelled here.
|
||||
pub cancelled: Option<fn() -> bool>,
|
||||
}
|
||||
|
||||
/// The most bytes of a part written between two cancellation checks.
|
||||
const PIECE: usize = 1 << 20;
|
||||
|
||||
fn invalid(message: impl std::fmt::Display) -> FormatError {
|
||||
FormatError::Invalid(message.to_string())
|
||||
}
|
||||
|
||||
/// Whether the bytes start as a container does, whatever the file is called.
|
||||
pub fn is_container(bytes: &[u8]) -> bool {
|
||||
bytes.starts_with(b"PK\x03\x04")
|
||||
}
|
||||
|
||||
/// The SHA-256 of the bytes.
|
||||
pub fn sha256(bytes: &[u8]) -> [u8; 32] {
|
||||
Sha256::digest(bytes).into()
|
||||
}
|
||||
|
||||
fn hex(digest: &[u8; 32]) -> String {
|
||||
use std::fmt::Write as _;
|
||||
let mut out = String::with_capacity(64);
|
||||
for byte in digest {
|
||||
// Writing to a `String` cannot fail.
|
||||
let _ = write!(&mut out, "{byte:02x}");
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The SHA-256 of the bytes, in lowercase hexadecimal: a blob's name, and how a cache is keyed to
|
||||
/// the file it was written for.
|
||||
pub fn digest(bytes: &[u8]) -> String {
|
||||
hex(&sha256(bytes))
|
||||
}
|
||||
|
||||
/// One part a [`Writer`] wrote, with the SHA-256 of its uncompressed content. A caller that signs
|
||||
/// a container uses these to bind the parts without reading the file back.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct PartDigest {
|
||||
pub name: String,
|
||||
pub len: u64,
|
||||
pub digest: [u8; 32],
|
||||
}
|
||||
|
||||
/// A container being written.
|
||||
pub struct Writer {
|
||||
zip: craft_zip::Writer,
|
||||
names: BTreeSet<String>,
|
||||
expanded: usize,
|
||||
resolved: usize,
|
||||
limits: Limits,
|
||||
/// Which written parts to record for a signature, and the records so far.
|
||||
tracked: Option<TrackedParts>,
|
||||
}
|
||||
|
||||
/// A tracking closure and the part digests collected so far.
|
||||
type TrackedParts = (fn(&str) -> bool, Vec<PartDigest>);
|
||||
|
||||
impl Writer {
|
||||
pub fn new(limits: Limits) -> Writer {
|
||||
Writer { zip: craft_zip::Writer::new(), names: BTreeSet::new(), expanded: 0, resolved: 0, limits, tracked: None }
|
||||
}
|
||||
|
||||
/// Record the parts written from now on whose name `include` accepts, with their SHA-256, for
|
||||
/// [`Writer::take_parts`]. Used by a caller that signs the container.
|
||||
pub fn track_parts(&mut self, include: fn(&str) -> bool) {
|
||||
self.tracked = Some((include, Vec::new()));
|
||||
}
|
||||
|
||||
/// The tracked parts written so far, oldest first, and stop tracking.
|
||||
pub fn take_parts(&mut self) -> Option<Vec<PartDigest>> {
|
||||
self.tracked.take().map(|(_, parts)| parts)
|
||||
}
|
||||
|
||||
/// The container's bytes.
|
||||
pub fn finish(self) -> Result<Vec<u8>, FormatError> {
|
||||
self.zip.finish().map_err(invalid)
|
||||
}
|
||||
|
||||
/// Add a part, compressed or stored. A name already written is written once.
|
||||
pub fn part(&mut self, name: &str, bytes: &[u8], compressed: bool) -> Result<(), FormatError> {
|
||||
self.part_with_digest(name, bytes, compressed, None)
|
||||
}
|
||||
|
||||
/// [`Writer::part`] with a digest the caller already worked out (a blob's name is its digest),
|
||||
/// so a signed save does not hash the same bytes twice.
|
||||
fn part_with_digest(&mut self, name: &str, bytes: &[u8], compressed: bool, digest: Option<[u8; 32]>) -> Result<(), FormatError> {
|
||||
if bytes.len() > (self.limits.part_limit)(name) {
|
||||
return Err(invalid("part too large"));
|
||||
}
|
||||
if self.names.contains(name) {
|
||||
return Ok(());
|
||||
}
|
||||
// Check the counters this part would leave, and commit them only once it is written: a
|
||||
// refused part is not remembered as written.
|
||||
let expanded = self.expanded.checked_add(bytes.len()).ok_or_else(|| invalid("expanded size overflow"))?;
|
||||
if expanded > (self.limits.max_expanded)() || self.names.len().saturating_add(1) > self.limits.max_entries {
|
||||
return Err(invalid("package too large"));
|
||||
}
|
||||
let cancelled = self.limits.cancelled;
|
||||
if cancelled.is_some_and(|c| c()) {
|
||||
return Err(FormatError::Cancelled);
|
||||
}
|
||||
// the configured level, clamped to the Deflate levels craft-zip writes (1 to 9)
|
||||
let compression = if compressed {
|
||||
craft_zip::Compression::Level((self.limits.level)(name, bytes.len()).clamp(1, 9) as u8)
|
||||
} else {
|
||||
craft_zip::Compression::Stored
|
||||
};
|
||||
match cancelled {
|
||||
Some(cancelled) if bytes.len() > PIECE => {
|
||||
let mut stopped = false;
|
||||
let written = self.zip.add_with(name, compression, |out| {
|
||||
for piece in bytes.chunks(PIECE) {
|
||||
if cancelled() {
|
||||
stopped = true;
|
||||
return Err(std::io::Error::other("cancelled"));
|
||||
}
|
||||
out.write_all(piece)?;
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
if stopped {
|
||||
return Err(FormatError::Cancelled);
|
||||
}
|
||||
written.map_err(invalid)?;
|
||||
}
|
||||
_ => self.zip.add(name, bytes, compression).map_err(invalid)?,
|
||||
}
|
||||
self.names.insert(name.to_owned());
|
||||
self.expanded = expanded;
|
||||
if let Some((include, parts)) = &mut self.tracked
|
||||
&& include(name)
|
||||
{
|
||||
let digest = digest.unwrap_or_else(|| sha256(bytes));
|
||||
parts.push(PartDigest { name: name.to_owned(), len: bytes.len() as u64, digest });
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Would `entries` more optional parts of `extra` bytes, stored uncompressed, still keep the
|
||||
/// file within `budget` and its expanded size within the reader's budget? The parts already
|
||||
/// written count at their expanded size (an upper bound on what they take in the file).
|
||||
pub fn room(&self, entries: usize, extra: usize, budget: usize) -> bool {
|
||||
self.remaining(entries, budget).is_some_and(|left| extra <= left)
|
||||
}
|
||||
|
||||
/// The bytes `entries` more optional parts may hold together ([`Writer::room`]).
|
||||
pub fn space(&self, entries: usize, budget: usize) -> usize {
|
||||
self.remaining(entries, budget).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// The bytes left for `entries` more parts, or `None` when the entries don't fit or what is
|
||||
/// written already is past `budget` or the reader's budget.
|
||||
fn remaining(&self, entries: usize, budget: usize) -> Option<usize> {
|
||||
let count = self.names.len().checked_add(entries).filter(|&n| n <= self.limits.max_entries)?;
|
||||
let overhead = count.saturating_mul(256).saturating_add(65_536);
|
||||
let in_file = budget.checked_sub(overhead.saturating_add(self.expanded))?;
|
||||
let expanded = (self.limits.max_expanded)().checked_sub(self.expanded)?;
|
||||
Some(in_file.min(expanded))
|
||||
}
|
||||
|
||||
/// Add a payload as a blob, `blobs/<sha256>.<extension>`, and say its name, for the JSON that
|
||||
/// refers to it.
|
||||
pub fn blob(&mut self, bytes: &[u8], extension: &str, compressed: bool) -> Result<Value, FormatError> {
|
||||
let resolved = self
|
||||
.resolved
|
||||
.checked_add(bytes.len())
|
||||
.filter(|&n| n <= (self.limits.max_expanded)())
|
||||
.ok_or_else(|| invalid("resolved payloads too large"))?;
|
||||
let digest = sha256(bytes);
|
||||
let name = format!("blobs/{}.{extension}", hex(&digest));
|
||||
self.part_with_digest(&name, bytes, compressed, Some(digest))?;
|
||||
self.resolved = resolved;
|
||||
Ok(Value::String(name))
|
||||
}
|
||||
}
|
||||
|
||||
/// A container being read. Opening reads all bytes into memory and checks the directory
|
||||
/// (the entry count, unique and plain names, every part's
|
||||
/// declared size), not the parts.
|
||||
pub struct Reader {
|
||||
archive: craft_zip::Archive<'static>,
|
||||
blobs: BTreeMap<String, Vec<u8>>,
|
||||
/// The bytes of payloads resolved so far, repeats counted: bounded by the expanded budget.
|
||||
pub hydrated: usize,
|
||||
document: Option<Arc<[u8]>>,
|
||||
limits: Limits,
|
||||
/// Every byte part reads have expanded to, whatever the budget was at the time.
|
||||
read_total: u64,
|
||||
}
|
||||
|
||||
impl Reader {
|
||||
pub fn new(mut source: impl Read + Seek, limits: Limits) -> Result<Self, FormatError> {
|
||||
let length = source.seek(SeekFrom::End(0)).map_err(invalid)?;
|
||||
(limits.check_size)(length).map_err(invalid)?;
|
||||
|
||||
// Read all bytes into memory for the archive, no more than the length just approved: a file
|
||||
// that grew since is refused rather than read past its budget.
|
||||
source.seek(SeekFrom::Start(0)).map_err(invalid)?;
|
||||
let capacity = usize::try_from(length).map_err(|_| invalid("file too large"))?;
|
||||
let mut bytes = Vec::new();
|
||||
bytes.try_reserve_exact(capacity).map_err(|_| invalid("file too large"))?;
|
||||
source.take(length.saturating_add(1)).read_to_end(&mut bytes).map_err(invalid)?;
|
||||
if bytes.len() as u64 != length {
|
||||
return Err(invalid("the file changed while it was read"));
|
||||
}
|
||||
|
||||
let mut archive = craft_zip::Archive::open_owned(
|
||||
bytes,
|
||||
craft_zip::Limits { max_entries: limits.max_entries as u64, max_total_bytes: (limits.max_expanded)() as u64 },
|
||||
)
|
||||
.map_err(invalid)?;
|
||||
// the directory lists distinct names, so a repeated name is caught here
|
||||
if archive.has_duplicate_names() {
|
||||
return Err(invalid("duplicate or invalid archive entry"));
|
||||
}
|
||||
|
||||
// Validate entry names match format rules: no empty, no backslashes,
|
||||
// no path components that are "." or "..", and no duplicates; and every part's declared
|
||||
// size within its configured limit.
|
||||
let mut name_set = BTreeSet::new();
|
||||
for entry_idx in 0..archive.len() {
|
||||
// an entry whose name can't be read would be left out of the package unnoticed
|
||||
let Ok(Some(name)) = archive.entry_name(entry_idx) else {
|
||||
return Err(invalid("duplicate or invalid archive entry"));
|
||||
};
|
||||
{
|
||||
if name.is_empty()
|
||||
|| name.contains('\\')
|
||||
|| name.split('/').any(|c| c.is_empty() || c == "." || c == "..")
|
||||
|| !name_set.insert(name.clone())
|
||||
{
|
||||
return Err(invalid("duplicate or invalid archive entry"));
|
||||
}
|
||||
if archive.declared_size(&name).is_some_and(|size| size > (limits.part_limit)(&name) as u64) {
|
||||
return Err(invalid("part too large"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Self { archive, blobs: BTreeMap::new(), hydrated: 0, document: None, limits, read_total: 0 })
|
||||
}
|
||||
|
||||
/// The file's own part, `document.json`, read once and kept: provisional caches and the
|
||||
/// authoritative reading are bound to the same bytes, even if another process overwrites the
|
||||
/// open file in place. Blobs remain hash-checked.
|
||||
pub fn document(&mut self, limit: usize) -> Result<Arc<[u8]>, FormatError> {
|
||||
if self.document.is_none() {
|
||||
let bytes = self.limited_part("document.json", limit)?;
|
||||
self.document = Some(bytes.into());
|
||||
}
|
||||
// a cached document still keeps the current part limit
|
||||
let limit = limit.min((self.limits.part_limit)("document.json"));
|
||||
self.document.as_ref().filter(|d| d.len() <= limit).cloned().ok_or_else(|| invalid("metadata budget"))
|
||||
}
|
||||
|
||||
/// Whether the container has a part of this name.
|
||||
pub fn has_part(&self, name: &str) -> bool {
|
||||
self.archive.contains(name)
|
||||
}
|
||||
|
||||
/// The names of every part, in the container's order.
|
||||
pub fn part_names(&self) -> Vec<String> {
|
||||
self.archive.names().map(|n| n.into_owned()).collect()
|
||||
}
|
||||
|
||||
/// The part of this name, at most `limit` bytes and never more than its configured limit.
|
||||
pub fn limited_part(&mut self, name: &str, limit: usize) -> Result<Vec<u8>, FormatError> {
|
||||
let limit = limit.min((self.limits.part_limit)(name));
|
||||
let expected = self.archive.declared_size(name).ok_or_else(|| invalid("part not found"))?;
|
||||
if expected > limit as u64 {
|
||||
return Err(invalid("part too large"));
|
||||
}
|
||||
// the expanded budget is read each time: a raised (or lowered) one applies at once, the
|
||||
// bytes already read still counting against it
|
||||
let budget = self.archive.budget_mut();
|
||||
*budget = craft_zip::Budget::new(budget.what(), (self.limits.max_expanded)() as u64);
|
||||
if budget.charge(self.read_total).is_err() {
|
||||
return Err(invalid("package too large"));
|
||||
}
|
||||
// Read within budget. craft-zip's read() handles decompression bombs via the limit.
|
||||
let bytes = self.archive.read(name, limit as u64).map_err(invalid)?.ok_or_else(|| invalid("part not found"))?;
|
||||
if bytes.len() as u64 != expected {
|
||||
return Err(invalid("part length mismatch"));
|
||||
}
|
||||
self.read_total = self.read_total.saturating_add(expected);
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
/// The blob `value`'s `field` names, of this extension, within its part limit.
|
||||
pub fn blob(&mut self, value: &Value, field: &str, extension: &str) -> Result<&[u8], FormatError> {
|
||||
let limit = (self.limits.part_limit)(&format!("blob.{extension}"));
|
||||
self.limited_blob(value, field, extension, limit)
|
||||
}
|
||||
|
||||
/// [`Reader::blob`] within `limit` bytes. Its hash is checked when it is first read, and every
|
||||
/// reference counts against the resolved budget.
|
||||
pub fn limited_blob(&mut self, value: &Value, field: &str, extension: &str, limit: usize) -> Result<&[u8], FormatError> {
|
||||
let name = value.get(field).and_then(Value::as_str).ok_or_else(|| invalid(format!("missing `{field}` reference")))?;
|
||||
// a cached blob still keeps the part's current limit
|
||||
let limit = limit.min((self.limits.part_limit)(name));
|
||||
let hash = name
|
||||
.strip_prefix("blobs/")
|
||||
.and_then(|n| n.strip_suffix(&format!(".{extension}")))
|
||||
.filter(|h| h.len() == 64 && h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))
|
||||
.ok_or_else(|| invalid("invalid blob reference"))?;
|
||||
if !self.blobs.contains_key(name) {
|
||||
let bytes = self.limited_part(name, limit)?;
|
||||
if digest(&bytes) != hash {
|
||||
return Err(invalid(format!("blob hash mismatch: {name}")));
|
||||
}
|
||||
self.blobs.insert(name.to_owned(), bytes);
|
||||
}
|
||||
let bytes = self.blobs.get(name).ok_or_else(|| invalid("missing blob"))?;
|
||||
if bytes.len() > limit {
|
||||
return Err(invalid("blob exceeds its resource budget"));
|
||||
}
|
||||
// Count repeated references too: a tiny archive must not expand into unbounded owned
|
||||
// strings or arrays when the same blob is referenced repeatedly.
|
||||
self.hydrated = self
|
||||
.hydrated
|
||||
.checked_add(bytes.len())
|
||||
.filter(|&n| n <= (self.limits.max_expanded)())
|
||||
.ok_or_else(|| invalid("resolved payloads too large"))?;
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
/// A small PNG picture of the file, kept in the container as a disposable cache keyed to the
|
||||
/// exact file it shows: `cache/thumbnail.json` (its key) and `cache/thumbnail.png`. Reading it
|
||||
/// never resolves a payload; a picture that isn't one, or is stale, is a miss.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Thumbnail {
|
||||
/// The key's format tag.
|
||||
pub format: &'static str,
|
||||
/// The build that wrote it: only the same build reads it back.
|
||||
pub app: &'static str,
|
||||
/// Its width and height in pixels.
|
||||
pub size: [usize; 2],
|
||||
/// The most bytes it may have.
|
||||
pub max_bytes: usize,
|
||||
}
|
||||
|
||||
impl Thumbnail {
|
||||
/// Whether the bytes start as a PNG of this size within the byte budget (not a decode: a
|
||||
/// consumer decodes with its own dimension and allocation limits).
|
||||
pub fn valid(&self, bytes: &[u8]) -> bool {
|
||||
bytes.len() <= self.max_bytes
|
||||
&& bytes.get(..8) == Some(b"\x89PNG\r\n\x1a\n")
|
||||
&& bytes.get(8..16) == Some(b"\0\0\0\rIHDR")
|
||||
&& bytes.get(16..20) == Some((self.size[0] as u32).to_be_bytes().as_slice())
|
||||
&& bytes.get(20..24) == Some((self.size[1] as u32).to_be_bytes().as_slice())
|
||||
}
|
||||
|
||||
/// Add the picture, keyed to `document` (the file's own part), when it is one and there is
|
||||
/// room for it within `budget`; otherwise leave it out, never failing the save.
|
||||
pub fn write(&self, writer: &mut Writer, png: &[u8], document: &[u8], budget: usize) -> Result<(), FormatError> {
|
||||
if !self.valid(png) {
|
||||
return Ok(());
|
||||
}
|
||||
let key =
|
||||
serde_json::to_vec(&serde_json::json!({"format": self.format, "app": self.app, "document": digest(document), "payload": digest(png)}))
|
||||
.map_err(invalid)?;
|
||||
let fits = |name: &str, len: usize| len <= (writer.limits.part_limit)(name);
|
||||
if !fits("cache/thumbnail.json", key.len())
|
||||
|| !fits("cache/thumbnail.png", png.len())
|
||||
|| !writer.room(2, key.len().saturating_add(png.len()), budget)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
writer.part("cache/thumbnail.json", &key, false)?;
|
||||
writer.part("cache/thumbnail.png", png, false)
|
||||
}
|
||||
|
||||
/// The picture, when this build wrote it for exactly the file's own part (`document.json`,
|
||||
/// read within `document_limit`).
|
||||
pub fn read(&self, reader: &mut Reader, document_limit: usize) -> Result<Vec<u8>, FormatError> {
|
||||
let key: Value = serde_json::from_slice(&reader.limited_part("cache/thumbnail.json", 4096)?).map_err(invalid)?;
|
||||
let document = reader.document(document_limit)?;
|
||||
if key.get("format").and_then(Value::as_str) != Some(self.format)
|
||||
|| key.get("app").and_then(Value::as_str) != Some(self.app)
|
||||
|| key.get("document").and_then(Value::as_str) != Some(digest(&document).as_str())
|
||||
{
|
||||
return Err(invalid("thumbnail key"));
|
||||
}
|
||||
let png = reader.limited_part("cache/thumbnail.png", self.max_bytes)?;
|
||||
if !self.valid(&png) || key.get("payload").and_then(Value::as_str) != Some(digest(&png).as_str()) {
|
||||
return Err(invalid("thumbnail payload"));
|
||||
}
|
||||
Ok(png)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "container_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,260 @@
|
||||
use super::*;
|
||||
use std::io::Cursor;
|
||||
|
||||
const LIMITS: Limits = Limits {
|
||||
max_entries: 8,
|
||||
max_expanded: || 1 << 20,
|
||||
part_limit: |name| if name.ends_with(".big") { 1 << 20 } else { 1024 },
|
||||
level: |_, _| 1,
|
||||
check_size: |length| if length <= 1 << 20 { Ok(()) } else { Err("too large".into()) },
|
||||
cancelled: None,
|
||||
};
|
||||
|
||||
/// Parts and blobs written are read back as they were; a blob is stored once, named by its hash,
|
||||
/// and checked against it.
|
||||
#[test]
|
||||
fn parts_and_blobs_read_back() {
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("document.json", br#"{"a":1}"#, true).unwrap();
|
||||
let first = writer.blob(b"payload", "bin", false).unwrap();
|
||||
let again = writer.blob(b"payload", "bin", true).unwrap();
|
||||
assert_eq!(first, again, "the same payload is one blob");
|
||||
assert_eq!(first, Value::String(format!("blobs/{}.bin", digest(b"payload"))));
|
||||
let bytes = writer.finish().unwrap();
|
||||
assert!(is_container(&bytes));
|
||||
let mut reader = Reader::new(Cursor::new(bytes), LIMITS).unwrap();
|
||||
assert_eq!(reader.part_names().len(), 2);
|
||||
assert!(reader.has_part("document.json") && !reader.has_part("other.json"));
|
||||
assert_eq!(&*reader.document(64).unwrap(), br#"{"a":1}"#);
|
||||
let reference = serde_json::json!({"blob": first});
|
||||
assert_eq!(reader.blob(&reference, "blob", "bin").unwrap(), b"payload");
|
||||
assert!(reader.blob(&reference, "blob", "png").is_err(), "another extension");
|
||||
assert_eq!(reader.hydrated, 7);
|
||||
reader.hydrated = (LIMITS.max_expanded)();
|
||||
assert!(reader.blob(&reference, "blob", "bin").unwrap_err().to_string().contains("resolved payloads too large"));
|
||||
}
|
||||
|
||||
/// A part past its limit, or more entries than the limit, are refused as they are written.
|
||||
#[test]
|
||||
fn budgets_are_kept_when_writing() {
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
assert_eq!(writer.part("small.json", &[0; 1025], false), Err(FormatError::Invalid("part too large".into())));
|
||||
assert!(writer.part("large.big", &[0; 1025], true).is_ok());
|
||||
for i in 0..7 {
|
||||
writer.part(&format!("{i}.json"), b"x", false).unwrap();
|
||||
}
|
||||
assert_eq!(writer.part("one-more.json", b"x", false), Err(FormatError::Invalid("package too large".into())));
|
||||
assert!(!writer.room(1, 0, 1 << 20), "no room for an entry past the limit");
|
||||
}
|
||||
|
||||
/// A thumbnail is kept only for the file it shows.
|
||||
#[test]
|
||||
fn a_thumbnail_is_keyed_to_its_file() {
|
||||
let picture = Thumbnail { format: "app-thumbnail/1", app: "1.0", size: [2, 1], max_bytes: 1024 };
|
||||
let mut png = b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR".to_vec();
|
||||
png.extend_from_slice(&2u32.to_be_bytes());
|
||||
png.extend_from_slice(&1u32.to_be_bytes());
|
||||
assert!(picture.valid(&png));
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("document.json", b"{}", false).unwrap();
|
||||
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
|
||||
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
|
||||
assert_eq!(picture.read(&mut reader, 64).unwrap(), png);
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("document.json", b"{\"changed\":1}", false).unwrap();
|
||||
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
|
||||
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
|
||||
assert!(picture.read(&mut reader, 64).is_err(), "stale");
|
||||
}
|
||||
|
||||
/// A refused part is not remembered as written: once the budget is raised it is written.
|
||||
#[test]
|
||||
fn a_refused_part_can_be_written_later() {
|
||||
static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(4);
|
||||
let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::SeqCst), ..LIMITS };
|
||||
let mut writer = Writer::new(limits);
|
||||
assert_eq!(writer.part("a.json", b"12345678", false), Err(FormatError::Invalid("package too large".into())));
|
||||
assert!(writer.blob(b"12345678", "bin", false).is_err());
|
||||
BUDGET.store(1 << 20, std::sync::atomic::Ordering::SeqCst);
|
||||
writer.part("a.json", b"12345678", false).unwrap();
|
||||
writer.blob(b"12345678", "bin", false).unwrap();
|
||||
let reader = Reader::new(Cursor::new(writer.finish().unwrap()), limits).unwrap();
|
||||
assert_eq!(reader.part_names().len(), 2, "both parts were written on the retry");
|
||||
}
|
||||
|
||||
/// A cancelled save writes no more parts, whether the part is small or written in pieces.
|
||||
#[test]
|
||||
fn cancellation_stops_the_save() {
|
||||
let cancelled = Limits { cancelled: Some(|| true), ..LIMITS };
|
||||
let mut writer = Writer::new(cancelled);
|
||||
assert_eq!(writer.part("a.json", b"a", false), Err(FormatError::Cancelled));
|
||||
// cancelled after the first piece of a long part: checks before the part, then per piece
|
||||
static CHECKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
let midway = Limits {
|
||||
cancelled: Some(|| CHECKS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) >= 2),
|
||||
max_expanded: || 1 << 22,
|
||||
part_limit: |_| 1 << 22,
|
||||
..LIMITS
|
||||
};
|
||||
let mut writer = Writer::new(midway);
|
||||
assert_eq!(writer.part("big.big", &vec![0; PIECE + 1], true), Err(FormatError::Cancelled));
|
||||
assert_eq!(CHECKS.load(std::sync::atomic::Ordering::SeqCst), 3);
|
||||
let going = Limits { cancelled: Some(|| false), max_expanded: || 1 << 22, part_limit: |_| 1 << 22, ..LIMITS };
|
||||
let mut writer = Writer::new(going);
|
||||
writer.part("big.big", &vec![7; PIECE + 1], true).unwrap();
|
||||
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), going).unwrap();
|
||||
assert_eq!(reader.limited_part("big.big", 1 << 22).unwrap().len(), PIECE + 1);
|
||||
}
|
||||
|
||||
/// Room counts what is already written, and the reader keeps every part's configured limit.
|
||||
#[test]
|
||||
fn room_counts_written_parts_and_reads_keep_part_limits() {
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("large.big", &[0; 500 * 1024], false).unwrap();
|
||||
assert!(!writer.room(1, 200 * 1024, 600 * 1024));
|
||||
assert!(writer.room(1, 1024, 1 << 20));
|
||||
// what is written already is past the budget: not even an empty part fits
|
||||
assert!(!writer.room(1, 0, 400 * 1024));
|
||||
assert_eq!(writer.space(1, 400 * 1024), 0);
|
||||
let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS };
|
||||
let mut writer = Writer::new(wide);
|
||||
writer.part("document.json", &[b' '; 2048], false).unwrap();
|
||||
let bytes = writer.finish().unwrap();
|
||||
assert!(Reader::new(Cursor::new(bytes.clone()), LIMITS).is_err(), "declared past its configured limit");
|
||||
let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap();
|
||||
assert!(reader.limited_part("document.json", 4096).is_ok());
|
||||
}
|
||||
|
||||
/// A thumbnail past a cache part's own limit is left out, never failing the save.
|
||||
#[test]
|
||||
fn a_thumbnail_past_its_part_limit_is_left_out() {
|
||||
let picture = Thumbnail { format: "app-thumbnail/1", app: "1.0", size: [2, 1], max_bytes: 2048 };
|
||||
let mut png = b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR".to_vec();
|
||||
png.extend_from_slice(&2u32.to_be_bytes());
|
||||
png.extend_from_slice(&1u32.to_be_bytes());
|
||||
png.resize(1500, 0);
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
|
||||
let reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
|
||||
assert!(!reader.has_part("cache/thumbnail.json") && !reader.has_part("cache/thumbnail.png"));
|
||||
}
|
||||
|
||||
/// A source that grows after its length was approved is refused.
|
||||
#[test]
|
||||
fn a_source_that_grows_while_read_is_refused() {
|
||||
struct Growing(Cursor<Vec<u8>>, bool);
|
||||
impl Read for Growing {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
if !self.1 {
|
||||
self.1 = true;
|
||||
let end = self.0.get_ref().len();
|
||||
self.0.get_mut().extend_from_slice(&[0; 16]);
|
||||
self.0.set_position(self.0.position().min(end as u64));
|
||||
}
|
||||
self.0.read(buf)
|
||||
}
|
||||
}
|
||||
impl Seek for Growing {
|
||||
fn seek(&mut self, to: SeekFrom) -> std::io::Result<u64> {
|
||||
self.0.seek(to)
|
||||
}
|
||||
}
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("document.json", b"{}", false).unwrap();
|
||||
let bytes = writer.finish().unwrap();
|
||||
assert!(Reader::new(Growing(Cursor::new(bytes), false), LIMITS).err().is_some_and(|e| e.to_string().contains("changed")));
|
||||
}
|
||||
|
||||
/// Compressed parts use the configured Deflate level.
|
||||
#[test]
|
||||
fn compressed_parts_use_the_configured_level() {
|
||||
// words in a varied order, so the levels compress them differently
|
||||
let words = ["alpha ", "beta ", "gamma ", "delta ", "epsilon ", "zeta ", "eta ", "theta "];
|
||||
let mut seed = 7u32;
|
||||
let text: Vec<u8> = (0..150)
|
||||
.flat_map(|_| {
|
||||
seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345);
|
||||
words[(seed >> 16) as usize % words.len()].bytes()
|
||||
})
|
||||
.take(1000)
|
||||
.collect();
|
||||
let size = |level: fn(&str, usize) -> i64| {
|
||||
let mut writer = Writer::new(Limits { level, ..LIMITS });
|
||||
writer.part("document.json", &text, true).unwrap();
|
||||
writer.finish().unwrap().len()
|
||||
};
|
||||
assert_ne!(size(|_, _| 1), size(|_, _| 9), "the level is used");
|
||||
assert_eq!(size(|_, _| 42), size(|_, _| 9), "past 9 is clamped");
|
||||
assert_eq!(size(|_, _| -3), size(|_, _| 1), "below 1 is clamped");
|
||||
}
|
||||
|
||||
/// An archive naming one part twice is refused.
|
||||
#[test]
|
||||
fn a_repeated_part_name_is_refused() {
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("document.json", b"{}", false).unwrap();
|
||||
writer.part("documenX.json", b"[]", false).unwrap();
|
||||
let mut bytes = writer.finish().unwrap();
|
||||
let (from, to) = (b"documenX.json", b"document.json");
|
||||
let mut at = 0;
|
||||
while let Some(i) = bytes[at..].windows(from.len()).position(|w| w == from) {
|
||||
bytes[at + i..at + i + to.len()].copy_from_slice(to);
|
||||
at += i + to.len();
|
||||
}
|
||||
assert!(Reader::new(Cursor::new(bytes), LIMITS).is_err());
|
||||
}
|
||||
|
||||
/// Cached documents and blobs keep the current part limits.
|
||||
#[test]
|
||||
fn cached_parts_keep_the_current_part_limits() {
|
||||
let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS };
|
||||
let mut writer = Writer::new(wide);
|
||||
writer.part("document.json", &[b' '; 512], false).unwrap();
|
||||
let blob = writer.blob(&[7; 512], "bin", false).unwrap();
|
||||
let bytes = writer.finish().unwrap();
|
||||
let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap();
|
||||
let reference = serde_json::json!({"blob": blob});
|
||||
assert!(reader.document(4096).is_ok());
|
||||
assert!(reader.blob(&reference, "blob", "bin").is_ok());
|
||||
reader.limits = Limits { part_limit: |_| 128, ..wide };
|
||||
assert!(reader.document(4096).is_err());
|
||||
assert!(reader.limited_blob(&reference, "blob", "bin", 4096).is_err());
|
||||
}
|
||||
|
||||
/// A raised expanded budget applies to the open reader, and what was read still counts.
|
||||
#[test]
|
||||
fn a_raised_expanded_budget_applies_at_once() {
|
||||
static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(600);
|
||||
let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::Relaxed), part_limit: |_| 1 << 20, ..LIMITS };
|
||||
let wide = Limits { max_expanded: || 1 << 20, ..limits };
|
||||
let mut writer = Writer::new(wide);
|
||||
writer.part("document.json", &[b' '; 256], false).unwrap();
|
||||
writer.part("other.big", &[1; 512], false).unwrap();
|
||||
let bytes = writer.finish().unwrap();
|
||||
BUDGET.store(1 << 20, std::sync::atomic::Ordering::Relaxed);
|
||||
let mut reader = Reader::new(Cursor::new(bytes), limits).unwrap();
|
||||
BUDGET.store(600, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(reader.limited_part("document.json", 4096).is_ok());
|
||||
assert!(reader.limited_part("other.big", 4096).is_err(), "256 read + 512 is past 600");
|
||||
BUDGET.store(1024, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(reader.limited_part("other.big", 4096).is_ok(), "768 fits the raised 1024");
|
||||
// a lowered budget does not forget what was read: 768 + 256 is past 1000 after a dip to 100
|
||||
BUDGET.store(100, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(reader.limited_part("document.json", 4096).is_err());
|
||||
BUDGET.store(1000, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(reader.limited_part("document.json", 4096).is_err(), "1024 is past 1000");
|
||||
BUDGET.store(1024, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(reader.limited_part("document.json", 4096).is_ok());
|
||||
}
|
||||
|
||||
/// An entry whose local header can't be read is refused at open, not skipped.
|
||||
#[test]
|
||||
fn an_unreadable_entry_is_refused() {
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("document.json", b"{}", false).unwrap();
|
||||
writer.part("other.json", b"[]", false).unwrap();
|
||||
let mut bytes = writer.finish().unwrap();
|
||||
let second = bytes.windows(4).enumerate().filter(|(_, w)| *w == b"PK\x03\x04").map(|(i, _)| i).nth(1).unwrap();
|
||||
bytes[second + 2] = 0;
|
||||
assert!(Reader::new(Cursor::new(bytes), LIMITS).is_err());
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
//! Versioned file formats: what an app needs to read every file it ever wrote, and to open a file
|
||||
//! a newer build of it wrote.
|
||||
//!
|
||||
//! A file starts with `"format": "<prefix>N"`. A [`Family`] is one kind of file (an app's designs,
|
||||
//! its drawings): its tag, its own numbers, and how a file of an older number is brought up to
|
||||
//! this build's, step by step (N → N+1 → … → current). The app's code only knows the current
|
||||
//! format; all translation lives in the steps:
|
||||
//! - the steps written out by hand for the first formats, one per format ([`Formats::hand`]);
|
||||
//! - then one folder per format, `v<N>/`, one file per change in it ([`Part`]), each with a step
|
||||
//! when files from before the change are rewritten as they open ([`Formats::folded`]);
|
||||
//! - then the changes no release has a number for yet, in `pending/` ([`Formats::pending`]):
|
||||
//! while there are any, the build writes the next number, with their names in the file's
|
||||
//! `format_pending`, so a later build runs just the steps a file is missing. A release moves
|
||||
//! them into the next `v<N>/`. [`build`] lists the folders for a build script.
|
||||
//!
|
||||
//! A file a newer build wrote says so by a number past this build's or a change name this build
|
||||
//! doesn't have ([`Family::newer`]); the app then reads what it can and keeps the rest
|
||||
//! ([`newer`]). A file is saved as JSON, or in a [`container`] with its binary payloads.
|
||||
//!
|
||||
//! Steps are JSON-level (the file's shape) and return the app's own error type; every error this
|
||||
//! crate makes is a [`FormatError`], which the app's error type takes `From`.
|
||||
|
||||
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
pub mod build;
|
||||
pub mod container;
|
||||
pub mod newer;
|
||||
|
||||
/// Why a file is refused: its text says what is wrong, as the user reads it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum FormatError {
|
||||
/// The file isn't one of the family's, or it is damaged.
|
||||
Invalid(String),
|
||||
/// The work was cancelled (a save the person stopped): nothing more was written.
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for FormatError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
FormatError::Invalid(why) => f.write_str(why),
|
||||
FormatError::Cancelled => f.write_str("cancelled"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for FormatError {}
|
||||
|
||||
/// One JSON-level upgrade step: rewrites a file in place, or says why it can't.
|
||||
pub type Step<E> = fn(&mut Value) -> Result<(), E>;
|
||||
|
||||
/// One change to a family's file format: a file in its `v<N>/` or `pending/` folder.
|
||||
pub struct Part<E> {
|
||||
/// The file's name, which a file written while the change was pending lists.
|
||||
pub name: &'static str,
|
||||
/// How a file from before the change is rewritten as it opens, if it is.
|
||||
pub step: Option<Step<E>>,
|
||||
}
|
||||
|
||||
/// Another spelling of a change name that a file of one format may carry (a build that named the
|
||||
/// change otherwise): read as the change it stands for, never written.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Alias {
|
||||
/// The format whose files may say it.
|
||||
pub format: u32,
|
||||
/// The spelling in the file.
|
||||
pub name: &'static str,
|
||||
/// The change it stands for.
|
||||
pub known_as: &'static str,
|
||||
}
|
||||
|
||||
/// A family's upgrade path: the steps written out by hand, the folded formats after them (each a
|
||||
/// list of changes) and the pending changes, which make the next format.
|
||||
pub struct Formats<'a, E> {
|
||||
/// Entry `i` turns format `i + 1` into `i + 2`.
|
||||
pub hand: &'a [Step<E>],
|
||||
/// Each format after the hand-written ones, oldest first, and its changes.
|
||||
pub folded: &'a [(u32, &'a [Part<E>])],
|
||||
/// The changes no release has a number for yet.
|
||||
pub pending: &'a [Part<E>],
|
||||
}
|
||||
|
||||
/// A kind of file whose format is versioned: its own format tag, its own numbers and its own
|
||||
/// changes. Reading a file's format, telling a file a newer build saved and the upgrade are the
|
||||
/// same for every family.
|
||||
pub struct Family<'a, E> {
|
||||
/// The app, as messages name it.
|
||||
pub app: &'static str,
|
||||
/// What a file of the family is called in messages.
|
||||
pub noun: &'static str,
|
||||
/// The tag before the number in a file's `format`.
|
||||
pub prefix: &'static str,
|
||||
/// The tag files saved under an older name carry: read, never written.
|
||||
pub legacy_prefix: Option<&'static str>,
|
||||
/// The format of a file that names none, if such files exist.
|
||||
pub untagged: Option<u32>,
|
||||
/// Other spellings of change names that files may carry.
|
||||
pub aliases: &'a [Alias],
|
||||
/// The upgrade path.
|
||||
pub formats: Formats<'a, E>,
|
||||
}
|
||||
|
||||
/// How a file shows a newer build saved it ([`Family::newer`]).
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Newer {
|
||||
/// The number the file says.
|
||||
pub format: u32,
|
||||
/// The number this build reads up to.
|
||||
pub reads_up_to: u32,
|
||||
/// The changes the file names that this build doesn't have.
|
||||
pub unknown_changes: Vec<String>,
|
||||
}
|
||||
|
||||
impl<E> Formats<'_, E> {
|
||||
/// The format this build writes: the last folded one, or the next while changes are pending.
|
||||
pub const fn current(&self) -> u32 {
|
||||
self.hand.len() as u32 + 1 + self.folded.len() as u32 + if self.pending.is_empty() { 0 } else { 1 }
|
||||
}
|
||||
|
||||
/// The changes format `version` is made of, if it is made of changes.
|
||||
fn parts(&self, version: u32) -> Option<&[Part<E>]> {
|
||||
let after = version.checked_sub(self.hand.len() as u32 + 2)? as usize;
|
||||
match self.folded.get(after) {
|
||||
Some((_, parts)) => Some(parts),
|
||||
None if after == self.folded.len() && !self.pending.is_empty() => Some(self.pending),
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a change of this name is anywhere in this build's history.
|
||||
fn knows_part_name(&self, name: &str) -> bool {
|
||||
self.folded.iter().flat_map(|(_, parts)| *parts).chain(self.pending).any(|part| part.name == name)
|
||||
}
|
||||
}
|
||||
|
||||
impl<E> Family<'_, E> {
|
||||
/// The format this build writes: the last folded one, or the next while changes are pending.
|
||||
pub const fn current(&self) -> u32 {
|
||||
self.formats.current()
|
||||
}
|
||||
|
||||
/// What a file of the family is called in messages.
|
||||
pub const fn noun(&self) -> &'static str {
|
||||
self.noun
|
||||
}
|
||||
|
||||
/// The names of the pending changes this build writes into a file (`format_pending`).
|
||||
pub fn pending_names(&self) -> Vec<String> {
|
||||
self.formats.pending.iter().map(|p| p.name.to_string()).collect()
|
||||
}
|
||||
|
||||
/// The format tag of format `n`.
|
||||
pub fn format_string(&self, n: u32) -> String {
|
||||
format!("{}{n}", self.prefix)
|
||||
}
|
||||
|
||||
/// The version in one of this family's format tags; `None` for any other tag, another
|
||||
/// family's included.
|
||||
pub fn version_of(&self, s: &str) -> Option<u32> {
|
||||
// each tag form parses on its own: a legacy prefix may start with the primary one
|
||||
// (`app/` and `app/design/`)
|
||||
let parse = |prefix: &str| s.strip_prefix(prefix).and_then(|n| n.parse().ok());
|
||||
parse(self.prefix).or_else(|| self.legacy_prefix.and_then(parse))
|
||||
}
|
||||
|
||||
/// The change a name a file of format `version` lists stands for: its alias's, or itself.
|
||||
/// An alias counts only while this build has the change it stands for, so recognising it
|
||||
/// never grants support to another reader.
|
||||
fn canonical_applied_name<'name>(&self, version: u32, name: &'name str) -> &'name str {
|
||||
match self.aliases.iter().find(|a| a.format == version && a.name == name) {
|
||||
Some(alias) if self.formats.knows_part_name(alias.known_as) => alias.known_as,
|
||||
_ => name,
|
||||
}
|
||||
}
|
||||
|
||||
/// What says a file of format `version` with the changes `applied` came from a newer build:
|
||||
/// a number past this build's, or a change this build doesn't have anywhere in its history
|
||||
/// (a branch's pending change may have landed at a later number). A list beside a format that
|
||||
/// isn't made of changes means nothing (a tool or a test set an older number on a newer file).
|
||||
/// `None` for a file this build fully knows.
|
||||
pub fn newer(&self, version: u32, applied: Option<&[String]>) -> Option<Newer> {
|
||||
let current = self.current();
|
||||
let listed = applied.filter(|_| version > current || self.formats.parts(version).is_some()).unwrap_or_default();
|
||||
let unknown_changes: Vec<String> =
|
||||
listed.iter().filter(|name| !self.formats.knows_part_name(self.canonical_applied_name(version, name))).cloned().collect();
|
||||
(version > current || !unknown_changes.is_empty()).then_some(Newer { format: version, reads_up_to: current, unknown_changes })
|
||||
}
|
||||
}
|
||||
|
||||
impl<E: From<FormatError>> Family<'_, E> {
|
||||
/// Read a file's format and take its `format_pending` out of its JSON, before anything else
|
||||
/// in it is read: a file that isn't this family's is refused here. One a newer build saved is
|
||||
/// not: [`Family::newer`] says so.
|
||||
pub fn take_version(&self, v: &mut Value) -> Result<(u32, Option<Vec<String>>), E> {
|
||||
let (app, noun) = (self.app, self.noun);
|
||||
let not_ours = |what: String| E::from(FormatError::Invalid(format!("not a {app} {noun} ({what})")));
|
||||
let Some(obj) = v.as_object() else { return Err(not_ours("expected a JSON object".into())) };
|
||||
let version = match obj.get("format") {
|
||||
None => self.untagged.ok_or_else(|| not_ours("no format".into()))?,
|
||||
Some(Value::String(f)) => self.version_of(f).ok_or_else(|| not_ours(format!("format `{f}`")))?,
|
||||
Some(_) => return Err(not_ours("format is not a string".into())),
|
||||
};
|
||||
let applied = take_names(v, "format_pending", noun).map_err(E::from)?;
|
||||
if version == 0 {
|
||||
return Err(not_ours("format 0".into()));
|
||||
}
|
||||
Ok((version, applied))
|
||||
}
|
||||
|
||||
/// Bring a file of format `version` up to this build's; `applied` is its `format_pending`
|
||||
/// (both from [`Family::take_version`]): the changes of its format it already has, when it
|
||||
/// was written while they were pending. The others' steps run first, and already-applied
|
||||
/// changes never run again at their later number. A file past this build's number has no
|
||||
/// steps to run.
|
||||
pub fn upgrade(&self, v: &mut Value, version: u32, applied: Option<&[String]>) -> Result<(), E> {
|
||||
let formats = &self.formats;
|
||||
if version == 0 {
|
||||
return Err(E::from(FormatError::Invalid(format!("not a {} {} (format 0)", self.app, self.noun))));
|
||||
}
|
||||
// A list beside a format that isn't made of changes is ignored, like any field an upgrade
|
||||
// drops.
|
||||
let applied = applied.filter(|_| formats.parts(version).is_some());
|
||||
let already_applied =
|
||||
|part: &Part<E>| applied.is_some_and(|names| names.iter().any(|name| self.canonical_applied_name(version, name) == part.name));
|
||||
if applied.is_some() {
|
||||
for part in formats.parts(version).unwrap_or_default().iter().filter(|part| !already_applied(part)) {
|
||||
part.step.map_or(Ok(()), |step| step(v))?;
|
||||
}
|
||||
}
|
||||
for from in version..self.current() {
|
||||
match formats.hand.get(from as usize - 1) {
|
||||
Some(step) => step(v)?,
|
||||
None => {
|
||||
for part in formats.parts(from + 1).unwrap_or_default().iter().filter(|part| !already_applied(part)) {
|
||||
part.step.map_or(Ok(()), |step| step(v))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Take a short list of names out of a file's JSON: its `format_pending` (the pending changes it
|
||||
/// has).
|
||||
fn take_names(v: &mut Value, key: &str, noun: &str) -> Result<Option<Vec<String>>, FormatError> {
|
||||
let Some(list) = v.as_object_mut().and_then(|o| o.remove(key)) else { return Ok(None) };
|
||||
let invalid = || FormatError::Invalid(format!("damaged {noun}: `{key}` is not a short list of names"));
|
||||
let list = list.as_array().filter(|l| l.len() <= 1000).ok_or_else(invalid)?;
|
||||
list.iter().map(|n| n.as_str().filter(|n| n.len() <= 128).map(str::to_string).ok_or_else(invalid)).collect::<Result<_, _>>().map(Some)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "lib_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,142 @@
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
/// The app's error, as an app has one.
|
||||
#[derive(Debug, PartialEq)]
|
||||
struct AppError(String);
|
||||
|
||||
impl From<FormatError> for AppError {
|
||||
fn from(e: FormatError) -> Self {
|
||||
AppError(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Each step appends its name to the file's `log`, so the order steps ran in can be read back.
|
||||
fn log(v: &mut Value, step: &str) {
|
||||
if let Some(list) = v.get_mut("log").and_then(Value::as_array_mut) {
|
||||
list.push(json!(step));
|
||||
}
|
||||
}
|
||||
|
||||
const HAND: [Step<AppError>; 2] = [
|
||||
|v| {
|
||||
log(v, "1to2");
|
||||
Ok(())
|
||||
},
|
||||
|v| {
|
||||
log(v, "2to3");
|
||||
Ok(())
|
||||
},
|
||||
];
|
||||
const V4: [Part<AppError>; 2] = [
|
||||
Part {
|
||||
name: "a",
|
||||
step: Some(|v| {
|
||||
log(v, "a");
|
||||
Ok(())
|
||||
}),
|
||||
},
|
||||
Part { name: "b", step: None },
|
||||
];
|
||||
const V5: [Part<AppError>; 1] = [Part {
|
||||
name: "c",
|
||||
step: Some(|v| {
|
||||
log(v, "c");
|
||||
Ok(())
|
||||
}),
|
||||
}];
|
||||
const FOLDED: [(u32, &[Part<AppError>]); 2] = [(4, &V4), (5, &V5)];
|
||||
const PENDING: [Part<AppError>; 2] = [
|
||||
Part {
|
||||
name: "d",
|
||||
step: Some(|v| {
|
||||
log(v, "d");
|
||||
Ok(())
|
||||
}),
|
||||
},
|
||||
Part {
|
||||
name: "e",
|
||||
step: Some(|v| {
|
||||
log(v, "e");
|
||||
Ok(())
|
||||
}),
|
||||
},
|
||||
];
|
||||
const ALIASES: [Alias; 1] = [Alias { format: 5, name: "old-c", known_as: "c" }];
|
||||
|
||||
/// Format 1 untagged; hand steps to 3; folded 4 and 5; pending changes make 6.
|
||||
const FAMILY: Family<'static, AppError> = Family {
|
||||
app: "App",
|
||||
noun: "design",
|
||||
prefix: "app/",
|
||||
legacy_prefix: Some("old/"),
|
||||
untagged: Some(1),
|
||||
aliases: &ALIASES,
|
||||
formats: Formats { hand: &HAND, folded: &FOLDED, pending: &PENDING },
|
||||
};
|
||||
|
||||
fn steps_run(version: u32, applied: Option<&[String]>) -> Vec<String> {
|
||||
let mut v = json!({"log": []});
|
||||
FAMILY.upgrade(&mut v, version, applied).unwrap();
|
||||
v["log"].as_array().unwrap().iter().map(|s| s.as_str().unwrap().to_string()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tags_and_numbers() {
|
||||
assert_eq!(FAMILY.current(), 6);
|
||||
assert_eq!(FAMILY.format_string(6), "app/6");
|
||||
assert_eq!(FAMILY.version_of("app/4"), Some(4));
|
||||
assert_eq!(FAMILY.version_of("old/3"), Some(3), "the legacy tag is read");
|
||||
assert_eq!(FAMILY.version_of("other/3"), None);
|
||||
assert_eq!(FAMILY.pending_names(), vec!["d".to_string(), "e".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_is_taken_apart_or_refused_with_the_apps_words() {
|
||||
let mut v = json!({"format": "app/6", "format_pending": ["d"], "x": 1});
|
||||
assert_eq!(FAMILY.take_version(&mut v), Ok((6, Some(vec!["d".to_string()]))));
|
||||
assert!(v.get("format_pending").is_none(), "taken out");
|
||||
assert_eq!(FAMILY.take_version(&mut json!({})), Ok((1, None)), "untagged");
|
||||
let refused = |mut v: Value| FAMILY.take_version(&mut v).unwrap_err().0;
|
||||
assert_eq!(refused(json!([])), "not a App design (expected a JSON object)");
|
||||
assert_eq!(refused(json!({"format": "other/2"})), "not a App design (format `other/2`)");
|
||||
assert_eq!(refused(json!({"format": 2})), "not a App design (format is not a string)");
|
||||
assert_eq!(refused(json!({"format": "app/0"})), "not a App design (format 0)");
|
||||
assert_eq!(refused(json!({"format": "app/2", "format_pending": [1]})), "damaged design: `format_pending` is not a short list of names");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_step_runs_once_in_order() {
|
||||
assert_eq!(steps_run(1, None), ["1to2", "2to3", "a", "c", "d", "e"]);
|
||||
assert_eq!(steps_run(4, None), ["c", "d", "e"]);
|
||||
assert_eq!(steps_run(6, None), Vec::<String>::new());
|
||||
// A file written while `d` was pending has it: only `e` runs.
|
||||
assert_eq!(steps_run(6, Some(&["d".to_string()])), ["e"]);
|
||||
// A file of format 5 written while `c` was still pending, without it: `c` runs first.
|
||||
assert_eq!(steps_run(5, Some(&[])), ["c", "d", "e"]);
|
||||
// Under its other spelling `c` counts as had.
|
||||
assert_eq!(steps_run(5, Some(&["old-c".to_string()])), ["d", "e"]);
|
||||
// A list beside a hand-written format means nothing.
|
||||
assert_eq!(steps_run(2, Some(&["d".to_string()])), ["2to3", "a", "c", "d", "e"]);
|
||||
let mut v = json!({});
|
||||
assert_eq!(FAMILY.upgrade(&mut v, 0, None), Err(AppError("not a App design (format 0)".into())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_newer_file_is_told_by_its_number_or_an_unknown_change() {
|
||||
assert_eq!(FAMILY.newer(6, None), None);
|
||||
assert_eq!(FAMILY.newer(6, Some(&["d".to_string()])), None);
|
||||
assert_eq!(FAMILY.newer(7, None), Some(Newer { format: 7, reads_up_to: 6, unknown_changes: Vec::new() }));
|
||||
assert_eq!(FAMILY.newer(6, Some(&["z".to_string()])), Some(Newer { format: 6, reads_up_to: 6, unknown_changes: vec!["z".to_string()] }));
|
||||
assert_eq!(FAMILY.newer(5, Some(&["old-c".to_string()])), None, "an alias of a known change");
|
||||
assert_eq!(FAMILY.newer(4, Some(&["old-c".to_string()])), Some(Newer { format: 4, reads_up_to: 6, unknown_changes: vec!["old-c".to_string()] }));
|
||||
assert_eq!(FAMILY.newer(2, Some(&["z".to_string()])), None, "a list beside a hand-written format means nothing");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_legacy_prefix_that_extends_the_primary_one_is_read() {
|
||||
let nested = Family { prefix: "app/", legacy_prefix: Some("app/design/"), ..FAMILY };
|
||||
assert_eq!(nested.version_of("app/design/3"), Some(3));
|
||||
assert_eq!(nested.version_of("app/4"), Some(4));
|
||||
assert_eq!(nested.version_of("app/design/x"), None);
|
||||
}
|
||||
@@ -0,0 +1,470 @@
|
||||
//! A file a newer build saved: it opens, with a warning, and saving it loses nothing.
|
||||
//!
|
||||
//! A newer format number, or a change the file names that this build doesn't have, says a newer
|
||||
//! build saved the file ([`crate::Family::newer`]). It may use nothing this build lacks, so the app
|
||||
//! reads it as usual. When part of it can't be read, that part stays out of the way and the rest
|
||||
//! opens: the app reads the elements of its lists it can ([`NewerFile::keep_element`]), then any
|
||||
//! other top-level section it can't read is left out whole ([`leave_out_unreadable`],
|
||||
//! [`read_around`]), and a field it doesn't know, which typed reading would drop, is noticed by
|
||||
//! comparing the file with what was read ([`NewerFile::compare`]).
|
||||
//!
|
||||
//! All of these are named in [`NewerFile::not_understood`] and go back into the file when it is
|
||||
//! saved ([`NewerFile::restore_into`]), as they came. What can't be put back because the file was
|
||||
//! changed around it is what `restore_into` returns.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::Newer;
|
||||
|
||||
/// The most sections left out one at a time to find the one a file can't be read with.
|
||||
const MAX_SECTION_TRIES: usize = 64;
|
||||
|
||||
/// What the file says and holds that this build doesn't read.
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct NewerFile {
|
||||
/// The app, as the notice names it.
|
||||
pub app: &'static str,
|
||||
/// The number the file says.
|
||||
pub format: u32,
|
||||
/// The number this build reads up to.
|
||||
pub reads_up_to: u32,
|
||||
/// The changes the file names that this build doesn't have.
|
||||
pub unknown_changes: Vec<String>,
|
||||
/// What this build couldn't read, by name.
|
||||
pub not_understood: Vec<String>,
|
||||
/// Fields known records lost in typed reading.
|
||||
kept: Option<Kept>,
|
||||
/// Top-level sections left out whole.
|
||||
sections: Vec<Section>,
|
||||
/// Elements left out of a top-level list: its key and the element.
|
||||
elements: Vec<(String, Value)>,
|
||||
/// The parts of a container this build has no use for: name and bytes. A blob only one of the
|
||||
/// kept pieces refers to is one of them.
|
||||
pub parts: Vec<(String, Arc<[u8]>)>,
|
||||
/// The file's `container.json`, when this build doesn't know what it says.
|
||||
pub container: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
struct Section {
|
||||
key: String,
|
||||
value: Value,
|
||||
/// This build's record of the section, without the file's: while it is unchanged, the file's
|
||||
/// goes back.
|
||||
empty: Option<Value>,
|
||||
}
|
||||
|
||||
/// The fields of a record that typed reading dropped.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
enum Kept {
|
||||
Field(Value),
|
||||
Fields(BTreeMap<String, Kept>),
|
||||
Items(Vec<(Anchor, Kept)>),
|
||||
}
|
||||
|
||||
/// Which element of a list some kept fields belong to.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
enum Anchor {
|
||||
Id(u64),
|
||||
Name(String),
|
||||
/// Its place, good only while the list keeps its length.
|
||||
Index {
|
||||
at: usize,
|
||||
of: usize,
|
||||
},
|
||||
}
|
||||
|
||||
impl NewerFile {
|
||||
/// A file of `app` that `newer` says a newer build saved, nothing left out of it yet.
|
||||
pub fn new(app: &'static str, newer: &Newer) -> NewerFile {
|
||||
NewerFile {
|
||||
app,
|
||||
format: newer.format,
|
||||
reads_up_to: newer.reads_up_to,
|
||||
unknown_changes: newer.unknown_changes.clone(),
|
||||
..NewerFile::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// What to tell the user, whatever else is said.
|
||||
pub fn notice(&self) -> String {
|
||||
let (app, reads, format) = (self.app, self.reads_up_to, self.format);
|
||||
match self.unknown_changes.first() {
|
||||
Some(change) if format <= reads => {
|
||||
format!("Saved by a newer {app} (format {format} with `{change}`, which this version doesn't know). If something's missing, update.")
|
||||
}
|
||||
_ => format!("Saved by a newer {app} (format {format}, this version reads up to {reads}). If something's missing, update."),
|
||||
}
|
||||
}
|
||||
|
||||
/// Every line to report: the notice, then what couldn't be read, by name.
|
||||
pub fn warnings(&self) -> Vec<String> {
|
||||
let unread = self.not_understood.iter().map(|what| format!("Not read: {what}. Saving keeps it."));
|
||||
std::iter::once(self.notice()).chain(unread).collect()
|
||||
}
|
||||
|
||||
/// What a reply says about it.
|
||||
pub fn report(&self) -> Value {
|
||||
json!({
|
||||
"format": self.format,
|
||||
"reads_up_to": self.reads_up_to,
|
||||
"unknown_changes": self.unknown_changes,
|
||||
"not_understood": self.not_understood,
|
||||
"notice": self.notice(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the next save needs a container to keep what the file holds: plain JSON has no room
|
||||
/// for parts beside the file's own, nor for the container's own `container.json`.
|
||||
pub fn needs_container(&self) -> bool {
|
||||
!self.parts.is_empty() || self.container.is_some()
|
||||
}
|
||||
|
||||
/// Whether anything kept back names this text (a blob's name in a container).
|
||||
pub fn mentions(&self, text: &str) -> bool {
|
||||
fn kept(k: &Kept, text: &str) -> bool {
|
||||
match k {
|
||||
Kept::Field(v) => value_mentions(v, text),
|
||||
Kept::Fields(m) => m.values().any(|x| kept(x, text)),
|
||||
Kept::Items(l) => l.iter().any(|(_, x)| kept(x, text)),
|
||||
}
|
||||
}
|
||||
self.kept.as_ref().is_some_and(|k| kept(k, text))
|
||||
|| self.sections.iter().any(|s| value_mentions(&s.value, text))
|
||||
|| self.elements.iter().any(|(_, v)| value_mentions(v, text))
|
||||
}
|
||||
|
||||
/// Say that a section the reader couldn't finish reading is left out. `empty` is what the
|
||||
/// app writes for the section while it holds nothing, if anything.
|
||||
pub fn keep_section(&mut self, key: &str, value: Value, empty: Option<Value>, why: &str) {
|
||||
self.not_understood.push(format!("section `{key}` ({why})"));
|
||||
self.sections.push(Section { key: key.into(), value, empty });
|
||||
}
|
||||
|
||||
/// Say that an element of the top-level list `key` is left out, as `note` names it; it goes
|
||||
/// back at the end of the list when the file is saved.
|
||||
pub fn keep_element(&mut self, key: &str, element: Value, note: String) {
|
||||
self.not_understood.push(note);
|
||||
self.elements.push((key.into(), element));
|
||||
}
|
||||
|
||||
/// Compare the file as it reads (`file`, after the same upgrade as what was read) with what was
|
||||
/// read of it (`known`, its typed record written out): what `known` lacks is kept and named.
|
||||
/// An object whose `type` is `placeholder_tag` (an element the app kept whole) is not
|
||||
/// compared; nor are these top-level fields: those in `container_only` (what a container adds
|
||||
/// beside the file's own), `format` and `format_pending`, and the sections kept whole (they go
|
||||
/// back whole).
|
||||
pub fn compare(&mut self, file: &Value, known: &Value, placeholder_tag: &str, container_only: &[&str]) {
|
||||
let mut paths = Vec::new();
|
||||
let mut top_only: Vec<&str> = container_only.to_vec();
|
||||
top_only.extend(["format", "format_pending"]);
|
||||
top_only.extend(self.sections.iter().map(|s| s.key.as_str()));
|
||||
// a list typed writing leaves out holds only elements kept on their own: not kept twice
|
||||
top_only.extend(self.elements.iter().map(|(key, _)| key.as_str()).filter(|key| known.get(*key).is_none()));
|
||||
let kept = vanished(file, known, "", &mut paths, placeholder_tag, &top_only);
|
||||
self.kept = kept;
|
||||
self.not_understood.extend(paths);
|
||||
for section in &mut self.sections {
|
||||
section.empty = known.get(§ion.key).cloned();
|
||||
}
|
||||
}
|
||||
|
||||
/// Write back what was kept of the file into `value`, the file as this build writes it.
|
||||
/// Returns what couldn't go back, because the file was changed around it.
|
||||
pub fn restore_into(&self, value: &mut Value) -> Vec<String> {
|
||||
let mut lost = Vec::new();
|
||||
if let Some(kept) = &self.kept {
|
||||
apply(kept, value, "", &mut lost);
|
||||
}
|
||||
if let Some(top) = value.as_object_mut() {
|
||||
for section in &self.sections {
|
||||
if top.get(§ion.key) == section.empty.as_ref() {
|
||||
top.insert(section.key.clone(), section.value.clone());
|
||||
} else {
|
||||
lost.push(format!("section `{}`, replaced by what was edited here", section.key));
|
||||
}
|
||||
}
|
||||
// each list's records indexed once, by the identity records are matched by
|
||||
let mut indexes: HashMap<&str, Positions> = HashMap::new();
|
||||
for (key, element) in &self.elements {
|
||||
let name = element.get("name");
|
||||
let anchor = anchor_of(element);
|
||||
match top.entry(key.clone()).or_insert_with(|| json!([])) {
|
||||
Value::Array(list) => {
|
||||
let index = indexes.entry(key.as_str()).or_insert_with(|| Positions::of(list));
|
||||
if anchor.as_ref().is_some_and(|a| index.find(list.len(), a).is_some()) {
|
||||
lost.push(format!(
|
||||
"`{key}` element {}, replaced by what was made here",
|
||||
name.and_then(Value::as_str).unwrap_or("unnamed")
|
||||
));
|
||||
} else {
|
||||
index.add(element, list.len());
|
||||
list.push(element.clone());
|
||||
}
|
||||
}
|
||||
_ => lost.push(format!("`{key}` element {}, replaced by what was made here", name.and_then(Value::as_str).unwrap_or("unnamed"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
lost
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a string anywhere in `v` is `text`; iterative, so no nesting depth overflows the stack.
|
||||
fn value_mentions(v: &Value, text: &str) -> bool {
|
||||
let mut todo = vec![v];
|
||||
while let Some(v) = todo.pop() {
|
||||
match v {
|
||||
Value::String(s) if s == text => return true,
|
||||
Value::Array(a) => todo.extend(a),
|
||||
Value::Object(o) => todo.extend(o.values()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// The first line of a reader's error, without the list of every kind it knows, and without the
|
||||
/// `"<noun> file: "` its reader starts it with.
|
||||
pub fn short<'a>(why: &'a str, noun: &str) -> &'a str {
|
||||
let why = why.split(", expected").next().unwrap_or(why);
|
||||
why.strip_prefix(&format!("{noun} file: ")).unwrap_or(why)
|
||||
}
|
||||
|
||||
/// Leave a section out and keep it. A section the file needs is left empty.
|
||||
fn move_out(top: &mut Map<String, Value>, key: &str, why: &str, noun: &str, file: &mut NewerFile) {
|
||||
let Some(value) = top.remove(key) else { return };
|
||||
file.not_understood.push(format!("section `{key}` ({})", short(why, noun)));
|
||||
file.sections.push(Section { key: key.into(), value, empty: None });
|
||||
}
|
||||
|
||||
/// Every top-level section of `top` that doesn't read by itself (`piece`, the app's reading of a
|
||||
/// file holding just that section) is left out and kept in `file`. `noun` is what a file of the
|
||||
/// app's is called, which its reader's errors start with.
|
||||
pub fn leave_out_unreadable<E: std::fmt::Display>(
|
||||
top: &mut Map<String, Value>,
|
||||
file: &mut NewerFile,
|
||||
noun: &str,
|
||||
piece: impl Fn(&str, &Value) -> Result<(), E>,
|
||||
) {
|
||||
let keys: Vec<String> = top.keys().cloned().collect();
|
||||
for key in &keys {
|
||||
let value = top.get(key).cloned().unwrap_or_default();
|
||||
if let Err(why) = piece(key, &value) {
|
||||
move_out(top, key, &why.to_string(), noun, file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read `top` with `attempt`, leaving out the sections it can't be read with: if it doesn't read,
|
||||
/// each section but the `core` ones (which the file can't be read without) is tried left out alone,
|
||||
/// and as a last resort every one of them is. What is left out is kept in `file`. The error is the
|
||||
/// first attempt's, when even the core alone doesn't read.
|
||||
pub fn read_around<D, E: std::fmt::Display>(
|
||||
top: &mut Map<String, Value>,
|
||||
file: &mut NewerFile,
|
||||
core: &[&str],
|
||||
noun: &str,
|
||||
attempt: impl Fn(&Map<String, Value>) -> Result<D, E>,
|
||||
) -> Result<D, String> {
|
||||
let first = match attempt(top) {
|
||||
Ok(doc) => return Ok(doc),
|
||||
Err(why) => why.to_string(),
|
||||
};
|
||||
// Sections that are fine by themselves but not together with the rest.
|
||||
let keys: Vec<String> = top.keys().filter(|k| !core.contains(&k.as_str())).cloned().collect();
|
||||
// Each try reads the whole file: a file of many sections goes straight to the last resort.
|
||||
for key in keys.iter().filter(|_| keys.len() <= MAX_SECTION_TRIES) {
|
||||
let mut without = top.clone();
|
||||
without.remove(key);
|
||||
if let Ok(doc) = attempt(&without) {
|
||||
move_out(top, key, &first, noun, file);
|
||||
return Ok(doc);
|
||||
}
|
||||
}
|
||||
let mut bare = top.clone();
|
||||
bare.retain(|key, _| core.contains(&key.as_str()));
|
||||
match attempt(&bare) {
|
||||
Ok(doc) => {
|
||||
for key in &keys {
|
||||
move_out(top, key, &first, noun, file);
|
||||
}
|
||||
Ok(doc)
|
||||
}
|
||||
Err(_) => Err(first),
|
||||
}
|
||||
}
|
||||
|
||||
/// A value a writer leaves out without losing anything: not worth telling anyone about.
|
||||
pub fn looks_default(v: &Value) -> bool {
|
||||
match v {
|
||||
Value::Null => true,
|
||||
Value::Bool(b) => !b,
|
||||
Value::Number(n) => n.as_f64() == Some(0.0),
|
||||
Value::String(s) => s.is_empty(),
|
||||
Value::Array(a) => a.is_empty(),
|
||||
Value::Object(o) => o.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// What `raw` has that `known`, what was read of it, doesn't. `paths` gets those fields' names.
|
||||
/// `top_only` are top-level keys left out of the comparison.
|
||||
fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec<String>, placeholder_tag: &str, top_only: &[&str]) -> Option<Kept> {
|
||||
match (raw, known) {
|
||||
(Value::Object(raw), Value::Object(known)) => {
|
||||
if known.get("type").and_then(Value::as_str) == Some(placeholder_tag) {
|
||||
return None;
|
||||
}
|
||||
let mut kept = BTreeMap::new();
|
||||
for (key, value) in raw {
|
||||
if path.is_empty() && top_only.contains(&key.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let inner = if path.is_empty() { key.clone() } else { format!("{path}.{key}") };
|
||||
match known.get(key) {
|
||||
None => {
|
||||
if !looks_default(value) {
|
||||
paths.push(format!("field `{inner}`"));
|
||||
}
|
||||
kept.insert(key.clone(), Kept::Field(value.clone()));
|
||||
}
|
||||
Some(known) => {
|
||||
if let Some(k) = vanished(value, known, &inner, paths, placeholder_tag, top_only) {
|
||||
kept.insert(key.clone(), k);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
(!kept.is_empty()).then_some(Kept::Fields(kept))
|
||||
}
|
||||
(Value::Array(raw), Value::Array(known)) => {
|
||||
let mut items = Vec::new();
|
||||
let positions = Positions::of(known);
|
||||
for (at, element) in raw.iter().enumerate() {
|
||||
// an identity typed reading kept: an id it dropped falls back to the name
|
||||
let anchor = surviving_anchor(element, &positions).unwrap_or(Anchor::Index { at, of: raw.len() });
|
||||
let Some(partner) = positions.find(known.len(), &anchor).and_then(|i| known.get(i)) else { continue };
|
||||
let label = match (element.get("name").and_then(Value::as_str), &anchor) {
|
||||
(Some(name), _) => format!("{path}[{name}]"),
|
||||
(None, Anchor::Id(id)) => format!("{path}[{id}]"),
|
||||
_ => format!("{path}[{at}]"),
|
||||
};
|
||||
if let Some(k) = vanished(element, partner, &label, paths, placeholder_tag, top_only) {
|
||||
items.push((anchor, k));
|
||||
}
|
||||
}
|
||||
(!items.is_empty()).then_some(Kept::Items(items))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn anchor_of(element: &Value) -> Option<Anchor> {
|
||||
if let Some(id) = element.get("id").and_then(Value::as_u64) {
|
||||
return Some(Anchor::Id(id));
|
||||
}
|
||||
element.get("name").and_then(Value::as_str).map(|name| Anchor::Name(name.to_string()))
|
||||
}
|
||||
|
||||
/// The identity of `element` that `known`'s records still carry: its id, or else its name. When
|
||||
/// typed reading kept neither, there is none, and the record is matched by its place.
|
||||
fn surviving_anchor(element: &Value, known: &Positions) -> Option<Anchor> {
|
||||
let id = element.get("id").and_then(Value::as_u64).map(Anchor::Id);
|
||||
let name = element.get("name").and_then(Value::as_str).map(|n| Anchor::Name(n.to_string()));
|
||||
[id, name].into_iter().flatten().find(|anchor| known.find(usize::MAX, anchor).is_some())
|
||||
}
|
||||
|
||||
/// Where each id and name first appears in a list, so matching records is not a scan per record.
|
||||
struct Positions {
|
||||
ids: HashMap<u64, usize>,
|
||||
names: HashMap<String, usize>,
|
||||
}
|
||||
|
||||
impl Positions {
|
||||
fn of(list: &[Value]) -> Positions {
|
||||
let (mut ids, mut names) = (HashMap::new(), HashMap::new());
|
||||
for (i, x) in list.iter().enumerate() {
|
||||
if let Some(id) = x.get("id").and_then(Value::as_u64) {
|
||||
ids.entry(id).or_insert(i);
|
||||
}
|
||||
if let Some(name) = x.get("name").and_then(Value::as_str) {
|
||||
names.entry(name.to_string()).or_insert(i);
|
||||
}
|
||||
}
|
||||
Positions { ids, names }
|
||||
}
|
||||
|
||||
/// Count `x`, now at `i`, in.
|
||||
fn add(&mut self, x: &Value, i: usize) {
|
||||
if let Some(id) = x.get("id").and_then(Value::as_u64) {
|
||||
self.ids.entry(id).or_insert(i);
|
||||
}
|
||||
if let Some(name) = x.get("name").and_then(Value::as_str) {
|
||||
self.names.entry(name.to_string()).or_insert(i);
|
||||
}
|
||||
}
|
||||
|
||||
/// The position of the record `anchor` names in a list of `len` records (an index anchor
|
||||
/// only while the list keeps its length).
|
||||
fn find(&self, len: usize, anchor: &Anchor) -> Option<usize> {
|
||||
match anchor {
|
||||
Anchor::Id(id) => self.ids.get(id).copied(),
|
||||
Anchor::Name(name) => self.names.get(name).copied(),
|
||||
Anchor::Index { at, of } => (len == *of && at < of).then_some(*at),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Put the kept fields back. An element that is gone was deleted here, its fields with it; one
|
||||
/// whose place can't be told any more, and a field whose record is gone, go into `lost`.
|
||||
fn apply(kept: &Kept, value: &mut Value, path: &str, lost: &mut Vec<String>) {
|
||||
match kept {
|
||||
Kept::Field(_) => {}
|
||||
Kept::Fields(fields) => {
|
||||
let Some(object) = value.as_object_mut() else {
|
||||
lost.push(format!("field `{path}`"));
|
||||
return;
|
||||
};
|
||||
for (key, sub) in fields {
|
||||
let inner = if path.is_empty() { key.clone() } else { format!("{path}.{key}") };
|
||||
match (sub, object.get_mut(key)) {
|
||||
(Kept::Field(v), None) => {
|
||||
object.insert(key.clone(), v.clone());
|
||||
}
|
||||
// the edited value stays; a retained one it replaced is reported
|
||||
(Kept::Field(v), Some(existing)) => {
|
||||
if existing != v {
|
||||
lost.push(format!("field `{inner}`"));
|
||||
}
|
||||
}
|
||||
(sub, Some(inner_value)) => apply(sub, inner_value, &inner, lost),
|
||||
// The list it was in is empty and not written: its elements were deleted.
|
||||
(Kept::Items(_), None) => {}
|
||||
(_, None) => lost.push(format!("field `{inner}`")),
|
||||
}
|
||||
}
|
||||
}
|
||||
Kept::Items(items) => {
|
||||
let Some(list) = value.as_array_mut() else {
|
||||
lost.push(format!("field `{path}`"));
|
||||
return;
|
||||
};
|
||||
let positions = Positions::of(list);
|
||||
let len = list.len();
|
||||
for (anchor, sub) in items {
|
||||
match (positions.find(len, anchor).and_then(|i| list.get_mut(i)), anchor) {
|
||||
(Some(element), _) => apply(sub, element, path, lost),
|
||||
(None, Anchor::Index { at, .. }) => lost.push(format!("element {at} of `{path}`")),
|
||||
(None, _) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "newer_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,218 @@
|
||||
use super::*;
|
||||
|
||||
fn newer() -> Newer {
|
||||
Newer { format: 9, reads_up_to: 8, unknown_changes: Vec::new() }
|
||||
}
|
||||
|
||||
/// What typed reading dropped is named and goes back on saving, where it was.
|
||||
#[test]
|
||||
fn dropped_fields_are_named_and_put_back() {
|
||||
let file = json!({"name": "A", "format": "app/9", "items": [{"id": 1, "size": 2, "colour": "red"}], "extra": {"x": 1}});
|
||||
let known = json!({"name": "A", "items": [{"id": 1, "size": 2}]});
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.compare(&file, &known, "placeholder", &[]);
|
||||
assert_eq!(said.not_understood, ["field `extra`", "field `items[1].colour`"]);
|
||||
let mut saved = json!({"name": "A", "items": [{"id": 1, "size": 3}]});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"name": "A", "items": [{"id": 1, "size": 3, "colour": "red"}], "extra": {"x": 1}}));
|
||||
assert_eq!(said.notice(), "Saved by a newer App (format 9, this version reads up to 8). If something's missing, update.");
|
||||
}
|
||||
|
||||
/// A section that can't be read is left out, kept, and goes back while it is unchanged here.
|
||||
#[test]
|
||||
fn unreadable_sections_are_left_out_and_kept() {
|
||||
let mut top = match json!({"name": "A", "good": 1, "bad": 2}) {
|
||||
Value::Object(m) => m,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
leave_out_unreadable(&mut top, &mut said, "design", |key: &str, _: &Value| if key == "bad" { Err("design file: no") } else { Ok(()) });
|
||||
assert!(!top.contains_key("bad"));
|
||||
assert_eq!(said.not_understood, ["section `bad` (no)"]);
|
||||
let read =
|
||||
read_around(
|
||||
&mut top,
|
||||
&mut said,
|
||||
&["name"],
|
||||
"design",
|
||||
|top: &Map<String, Value>| {
|
||||
if top.contains_key("good") { Err("design file: clash") } else { Ok(top.len()) }
|
||||
},
|
||||
);
|
||||
assert_eq!(read, Ok(1));
|
||||
assert_eq!(said.not_understood, ["section `bad` (no)", "section `good` (clash)"]);
|
||||
let mut saved = json!({"name": "A"});
|
||||
said.compare(&saved.clone(), &saved.clone(), "placeholder", &[]);
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"name": "A", "good": 1, "bad": 2}));
|
||||
}
|
||||
|
||||
/// Notices, warnings and reports name what was left out, and kept values can be searched.
|
||||
#[test]
|
||||
fn warnings_report_and_mentions_cover_everything_kept() {
|
||||
let mut said = NewerFile::new("App", &Newer { format: 8, reads_up_to: 8, unknown_changes: vec!["curves".into()] });
|
||||
assert_eq!(said.notice(), "Saved by a newer App (format 8 with `curves`, which this version doesn't know). If something's missing, update.");
|
||||
assert!(!said.needs_container());
|
||||
said.keep_section("plugins", json!({"tool": "ruler"}), Some(json!({})), "unknown");
|
||||
said.keep_element("shapes", json!({"name": "star", "points": ["tip"]}), "shape `star`".into());
|
||||
let file = json!({"shapes": [{"name": "a", "tags": ["deep"], "more": {"x": "nested"}}], "n": 1});
|
||||
said.compare(&file, &json!({"shapes": [{"name": "a"}], "n": 1}), "placeholder", &[]);
|
||||
assert_eq!(said.warnings()[0], said.notice());
|
||||
assert!(said.warnings().contains(&"Not read: section `plugins` (unknown). Saving keeps it.".to_string()));
|
||||
let report = said.report();
|
||||
assert_eq!(report["format"], 8);
|
||||
assert_eq!(report["unknown_changes"], json!(["curves"]));
|
||||
assert_eq!(report["notice"], json!(said.notice()));
|
||||
for text in ["ruler", "tip", "deep", "nested"] {
|
||||
assert!(said.mentions(text), "{text}");
|
||||
}
|
||||
assert!(!said.mentions("absent"));
|
||||
said.parts.push(("blob".into(), Arc::from(&b"x"[..])));
|
||||
assert!(said.needs_container());
|
||||
}
|
||||
|
||||
/// A kept section or element that was replaced here is reported lost, not written over.
|
||||
#[test]
|
||||
fn replaced_sections_and_elements_are_reported_lost() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.keep_section("plugins", json!({"tool": "ruler"}), None, "unknown");
|
||||
said.keep_element("shapes", json!({"name": "star"}), "shape `star`".into());
|
||||
said.keep_element("shapes", json!({"points": 3}), "an unnamed shape".into());
|
||||
let mut saved = json!({"plugins": {"edited": true}, "shapes": [{"name": "star"}]});
|
||||
let lost = said.restore_into(&mut saved);
|
||||
assert_eq!(lost, ["section `plugins`, replaced by what was edited here", "`shapes` element star, replaced by what was made here"]);
|
||||
assert_eq!(saved["shapes"], json!([{"name": "star"}, {"points": 3}]));
|
||||
let mut fresh = json!({});
|
||||
said.sections.clear();
|
||||
assert!(said.restore_into(&mut fresh).is_empty());
|
||||
assert_eq!(fresh["shapes"], json!([{"name": "star"}, {"points": 3}]));
|
||||
}
|
||||
|
||||
/// Retained container metadata alone needs a container to survive the save.
|
||||
#[test]
|
||||
fn container_metadata_alone_needs_a_container() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
assert!(!said.needs_container());
|
||||
said.container = Some(json!({"layout": 2}));
|
||||
assert!(said.needs_container());
|
||||
}
|
||||
|
||||
/// An edit that replaces a retained field keeps the edit and reports the replaced value.
|
||||
#[test]
|
||||
fn a_replaced_retained_field_is_reported() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.compare(&json!({"extra": 1}), &json!({}), "placeholder", &[]);
|
||||
let mut edited = json!({"extra": 2});
|
||||
assert_eq!(said.restore_into(&mut edited), ["field `extra`"]);
|
||||
assert_eq!(edited, json!({"extra": 2}));
|
||||
let mut same = json!({"extra": 1});
|
||||
assert!(said.restore_into(&mut same).is_empty());
|
||||
}
|
||||
|
||||
/// Records are matched by id and by name across long lists, and an index anchor only while the
|
||||
/// list keeps its length.
|
||||
#[test]
|
||||
fn records_match_by_id_name_or_unchanged_index() {
|
||||
let records: Vec<Value> = (0..2000).map(|i| json!({"id": i, "extra": i})).collect();
|
||||
let known: Vec<Value> = (0..2000).map(|i| json!({"id": i})).collect();
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.compare(&json!({"items": records}), &json!({"items": known}), "placeholder", &[]);
|
||||
let mut saved = json!({"items": (0..2000).rev().map(|i| json!({"id": i})).collect::<Vec<_>>()});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved["items"][0], json!({"id": 1999, "extra": 1999}));
|
||||
let mut by_name = NewerFile::new("App", &newer());
|
||||
by_name.compare(&json!({"l": [{"name": "a", "x": 1}, [1, 2]]}), &json!({"l": [{"name": "a"}, [1]]}), "placeholder", &[]);
|
||||
let mut renamed = json!({"l": [{"name": "a"}]});
|
||||
by_name.restore_into(&mut renamed);
|
||||
assert_eq!(renamed["l"][0], json!({"name": "a", "x": 1}));
|
||||
}
|
||||
|
||||
/// A section kept whole goes back whole, even when this build's record of it has fields.
|
||||
#[test]
|
||||
fn a_section_kept_whole_is_not_compared_field_by_field() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.keep_section("plugins", json!({"mode": "future", "payload": 1}), None, "unreadable");
|
||||
let file = json!({"name": "A", "plugins": {"mode": "future", "payload": 1}});
|
||||
said.compare(&file, &json!({"name": "A", "plugins": {"mode": "none"}}), "placeholder", &[]);
|
||||
let mut saved = json!({"name": "A", "plugins": {"mode": "none"}});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, file);
|
||||
}
|
||||
|
||||
/// A kept element goes back only while no record has its id or name.
|
||||
#[test]
|
||||
fn a_kept_element_keeps_record_identity() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.keep_element("items", json!({"id": 7, "kind": "future"}), "element 7".into());
|
||||
said.keep_element("items", json!({"kind": "anonymous"}), "unnamed".into());
|
||||
let mut clash = json!({"items": [{"id": 7, "kind": "made here"}]});
|
||||
assert_eq!(said.restore_into(&mut clash).len(), 1);
|
||||
assert_eq!(clash, json!({"items": [{"id": 7, "kind": "made here"}, {"kind": "anonymous"}]}));
|
||||
let mut free = json!({"items": []});
|
||||
assert!(said.restore_into(&mut free).is_empty());
|
||||
assert_eq!(free, json!({"items": [{"id": 7, "kind": "future"}, {"kind": "anonymous"}]}));
|
||||
}
|
||||
|
||||
/// Container-only fields are left out at the top level only.
|
||||
#[test]
|
||||
fn container_only_fields_are_top_level() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
let file = json!({"thumbnail": "t", "layer": {"thumbnail": "nested"}});
|
||||
said.compare(&file, &json!({"layer": {}}), "placeholder", &["thumbnail"]);
|
||||
assert_eq!(said.not_understood, ["field `layer.thumbnail`"]);
|
||||
let mut saved = json!({"layer": {}});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"layer": {"thumbnail": "nested"}}));
|
||||
}
|
||||
|
||||
/// An element kept on its own is not kept again with its list when typed writing leaves the list out.
|
||||
#[test]
|
||||
fn a_kept_element_is_not_restored_twice() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.keep_element("items", json!({"kind": "future"}), "unnamed".into());
|
||||
said.compare(&json!({"items": [{"kind": "future"}]}), &json!({}), "placeholder", &[]);
|
||||
let mut saved = json!({});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"items": [{"kind": "future"}]}));
|
||||
}
|
||||
|
||||
/// A record whose id typed reading drops is matched by its name, and its fields go back.
|
||||
#[test]
|
||||
fn a_dropped_id_falls_back_to_the_name() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.compare(&json!({"items": [{"id": 7, "name": "a", "extra": 1}]}), &json!({"items": [{"name": "a"}]}), "placeholder", &[]);
|
||||
let mut saved = json!({"items": [{"name": "a"}]});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"items": [{"name": "a", "id": 7, "extra": 1}]}));
|
||||
}
|
||||
|
||||
/// Kept elements sharing a list are checked against one index, the ones put back included.
|
||||
#[test]
|
||||
fn kept_elements_check_each_other() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.keep_element("items", json!({"id": 1, "v": "first"}), "1".into());
|
||||
said.keep_element("items", json!({"id": 1, "v": "again"}), "1 again".into());
|
||||
let mut saved = json!({"items": []});
|
||||
assert_eq!(said.restore_into(&mut saved).len(), 1);
|
||||
assert_eq!(saved, json!({"items": [{"id": 1, "v": "first"}]}));
|
||||
// deep nesting is searched without recursion
|
||||
let mut deep = json!("needle");
|
||||
for _ in 0..20_000 {
|
||||
deep = Value::Array(vec![deep]);
|
||||
}
|
||||
said.keep_section("deep", deep, None, "test");
|
||||
assert!(said.mentions("needle"));
|
||||
// serde_json drops deep values recursively; leak it rather than overflow the test thread
|
||||
std::mem::forget(said);
|
||||
}
|
||||
|
||||
/// A record whose whole identity typed reading drops is matched by its place while the list keeps
|
||||
/// its length, so an unchanged save keeps its fields.
|
||||
#[test]
|
||||
fn a_record_without_a_surviving_identity_is_matched_by_place() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.compare(&json!({"items": [{"id": 7, "extra": 1}]}), &json!({"items": [{}]}), "placeholder", &[]);
|
||||
let mut saved = json!({"items": [{}]});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"items": [{"id": 7, "extra": 1}]}));
|
||||
}
|
||||
@@ -19,6 +19,9 @@ heic-rs = { version = "=0.1.4", default-features = false, features = ["std"], op
|
||||
heic-rs = { version = "=0.1.4", default-features = false, features = ["std", "parallel"], optional = true }
|
||||
rayon = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = { workspace = true }
|
||||
|
||||
[features]
|
||||
default = ["decode"]
|
||||
# The decoder (probe, decode, decode_thumbnail, read_metadata): heic-rs for the container and HEVC.
|
||||
|
||||
+17
-4
@@ -36,11 +36,24 @@ Errors are `Error::Unsupported` (image sequences, overlays, 4:2:2/4:4:4, …), `
|
||||
`Error::Malformed`. It never panics: every call into heic-rs runs under `catch_unwind`, and a panic
|
||||
inside it (PhotoCraft's fuzzing found two in 0.1.1) becomes `Error::Malformed`.
|
||||
|
||||
Features: `decode` (default) is the decoder above. `metadata` adds `craft_heif::items::read`: the
|
||||
first `Exif` item and XMP packet from the box structure alone (`iinf`, `iloc`, `idat`; at most 4096
|
||||
boxes, 1024 items, 1024 extents per item, 32 MiB per item), without heic-rs. A build with
|
||||
Features: `decode` (default) is the decoder above. `metadata` adds `craft_heif::items`, which reads
|
||||
the `Exif` and XMP items from the box structure alone (`pitm`, `iinf`, `iref`, `iloc`, `idat`),
|
||||
without heic-rs:
|
||||
|
||||
- `items::read` returns the photo's own EXIF and XMP: the items `cdsc` ties to the primary image or
|
||||
to an image it is made from (`dimg`: grid tiles, `iden`, `iovl`, `tmap`), else those tied to no
|
||||
image. A thumbnail's or depth map's metadata is not the photo's.
|
||||
- `items::scan` returns every EXIF and XMP item, whichever image it describes, and says when it
|
||||
can't be sure it saw them all: `Scan::complete` is false for an incomplete or damaged item
|
||||
catalog, and an item whose bytes can't be read comes back as `Unreadable`. An app that must
|
||||
prove a file has no location (before an export that removes it) refuses when unsure.
|
||||
- `items::exif_tiff` finds the TIFF stream in an `Exif` item, with or without a correct offset
|
||||
field or an `Exif\0\0` prefix.
|
||||
|
||||
The reader walks at most 65,536 boxes and walks `iloc` once. It gathers at most 16 MiB per item,
|
||||
and a scan gathers at most 64 MiB in all. A build with
|
||||
`default-features = false, features = ["metadata"]` has no heic-rs dependency, and its
|
||||
`read_metadata` is that reader. With both features, a test checks that the two readers agree.
|
||||
`read_metadata` is `items::read`. With both features, a test checks that the two readers agree.
|
||||
|
||||
Feature `testdata` exposes `craft_heif::testdata`: lossless synthetic HEIF files (PCM-coded
|
||||
32 × 32 HEVC pictures, grids, every property above) for the tests of the crates above.
|
||||
|
||||
+666
-414
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -13,7 +13,10 @@ workspace = true
|
||||
[features]
|
||||
corpus = []
|
||||
default = []
|
||||
# Opt in to threaded AVIF encoding without enabling AVIF (or assembly) by itself.
|
||||
threads = ["ravif?/threading", "rav1e?/threading"]
|
||||
sgi = []
|
||||
sunraster = []
|
||||
avif = ["image/avif", "dep:ravif", "dep:rav1e", "dep:avif-serialize"]
|
||||
heif = ["dep:craft-heif"]
|
||||
# AVIF decode: craft-video's clean-room pure-Rust AV1 decoder. Native targets only (on wasm32 a
|
||||
|
||||
@@ -43,7 +43,7 @@ craft-image-codecs = { git = "https://github.com/storytold/craft-libs", rev = "<
|
||||
| TIFF | yes | yes | U8, U16, F32 | all six (CMYK, CMYK+A included) | yes | yes (tag 34675) | no | yes (tag 700) | yes | yes (Description, Make, Model, Software, DateTime, Artist, Copyright) | no | `tiff` |
|
||||
| WebP | yes | yes | U8 | RGB, RGBA | yes | yes | yes | yes | no | no | yes (VP8, when `webp_lossless` is off) | `image-webp` / built-in VP8 |
|
||||
| GIF | yes | yes | U8 | RGBA | 1-bit | no | no | no | no | no | yes (256-colour palette) | `image` |
|
||||
| BMP | yes | yes | U8 | RGB, RGBA | yes | no | no | no | no | no | no | `image` |
|
||||
| BMP | yes | yes | U8 | RGB, RGBA | yes | no | no | no | yes (header pixels per metre) | no | no | `image` |
|
||||
| TGA | yes | yes | U8 | Gray, GrayA, RGB, RGBA | yes | no | no | no | no | no | no | `image` (read) / built in (RLE write, one scanline per packet) |
|
||||
| ICO | yes | yes | U8 | RGBA (at most 256×256) | yes | no | no | no | no | no | no | `image` |
|
||||
| DDS | yes | yes | U8 | Gray, GrayA, RGB, RGBA | yes | no | no | no | no | no | yes (BC1–BC3, when `EncodeOptions::dds` asks for one) | built in |
|
||||
@@ -57,6 +57,7 @@ craft-image-codecs = { git = "https://github.com/storytold/craft-libs", rev = "<
|
||||
| Cineon | yes | yes | U16 (10-bit code values read and written exactly through U16) | Gray, RGB | no | no | no | no | no | no | no | built in |
|
||||
| DICOM | yes | **no** | U8, U16 | Gray, RGB | no | no | no | no | no | no | n/a | built in |
|
||||
| Amiga IFF (ILBM/PBM) | yes | yes (24-bit ILBM) | U8 | RGB, RGBA | yes (1-bit mask plane on write) | no | no | no | no | no | no | built in |
|
||||
| Sun Raster (`.ras`) | with feature `sunraster` (incl. RLE and colour maps) | with feature `sunraster` (uncompressed) | U8 | Gray, RGB, RGBA | yes | no | no | no | no | no | no | `sunraster` (off by default) |
|
||||
| SGI (`.sgi`, `.rgb`, `.bw`) | with feature `sgi` | with feature `sgi` (RLE) | U8, U16 | Gray, Gray+A, RGB, RGBA | yes | no | no | no | no | no | no | `sgi` (off by default) |
|
||||
| JPEG-LS (`.jls`) | yes (any NEAR; one scan per component) | yes (lossless) | U8, U16 | Gray, RGB | no | no | no | no | no | no | no | built in |
|
||||
| AVIF | with feature `avif-decode` (native only) | only with feature `avif` | U8, U16 (10/12-bit decodes to U16; PQ/HLG to linear F32) | RGB, RGBA | yes (auxiliary alpha) | read (`colr` prof/rICC) | read | read | no | no | yes | `craft-video` AV1 decoder; `image` + `ravif` to write |
|
||||
@@ -78,7 +79,8 @@ the same.
|
||||
Palette BMP and PNG pixels decode as RGB(A) in `Image`; `bmp_palette` and `png_palette` expose the
|
||||
embedded table (and a PNG's transparent entry) so `photocraft-io` can open them in Indexed Color mode.
|
||||
`bmp_palette` returns the colours the header declares, not the decoder's 256-entry padding.
|
||||
`encode_bmp_indexed` writes the inverse: indices plus a table as an 8-bit palette BMP.
|
||||
`encode_bmp_indexed` writes the inverse: indices plus a table as an 8-bit palette BMP;
|
||||
`encode_bmp_indexed_with_dpi` also writes a resolution in the header.
|
||||
|
||||
## Documented asymmetries and limitations
|
||||
|
||||
@@ -235,6 +237,11 @@ embedded table (and a PNG's transparent entry) so `photocraft-io` can open them
|
||||
the EXIF and XMP resolution are rewritten to the image's (`export_exif`, `export_xmp`): the
|
||||
RATIONAL values in place, ResolutionUnit set to 2, an entry that can't be rewritten removed
|
||||
from IFD0 without moving any other data. IFD1 (the thumbnail's) is left as it is.
|
||||
* **Resolution without decoding** (module `resolution_probe`): `file_resolution(bytes)` reads
|
||||
the pixels per inch a JPEG, PNG, WebP, BMP or classic TIFF records from its headers alone, for
|
||||
placing a picture at its physical size before (or without) decoding it. JPEG, PNG, WebP and
|
||||
TIFF answer as `decode` would; BMP reads the info header's pixels per metre. Values outside
|
||||
1–100 000 ppi count as none.
|
||||
* **JPEG**
|
||||
* 8-bit only. Neither decoder backend supports 12-bit.
|
||||
* A file cut off inside its image data decodes leniently (a baseline JPEG's missing rows come
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
pub(crate) mod avif;
|
||||
#[cfg(all(feature = "avif-decode", not(target_arch = "wasm32")))]
|
||||
pub(crate) mod avif_decode;
|
||||
pub(crate) mod bmp;
|
||||
pub(crate) mod dds;
|
||||
pub(crate) mod dds_bc;
|
||||
pub(crate) mod deep_exr;
|
||||
@@ -22,6 +23,8 @@ pub(crate) mod png;
|
||||
pub(crate) mod pnm;
|
||||
#[cfg(feature = "sgi")]
|
||||
pub(crate) mod sgi;
|
||||
#[cfg(feature = "sunraster")]
|
||||
pub(crate) mod sunraster;
|
||||
pub(crate) mod tga;
|
||||
pub(crate) mod tiff;
|
||||
pub(crate) mod tiff_exif;
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
//! A BMP's resolution: `biXPelsPerMeter` and `biYPelsPerMeter`, the pixels per metre of the
|
||||
//! BITMAPINFOHEADER and every later header (V4, V5, OS/2 2.x), at file offsets 38 and 42. The
|
||||
//! 12-byte BITMAPCOREHEADER of OS/2 1.x has no such fields. The `image` crate reads and writes
|
||||
//! the pixels only, and writes 0 here, which readers take as "no resolution".
|
||||
|
||||
use super::png::dpi_of_ppm;
|
||||
|
||||
/// Bytes before the fields: the 14-byte BITMAPFILEHEADER, then the DIB header's size, width,
|
||||
/// height, planes, bit count, compression and image size.
|
||||
const X_PPM: usize = 38;
|
||||
const Y_PPM: usize = 42;
|
||||
/// The shortest DIB header that carries the resolution (BITMAPINFOHEADER).
|
||||
const INFO_HEADER: u32 = 40;
|
||||
const METERS_PER_INCH: f64 = 0.0254;
|
||||
|
||||
/// The little-endian 4 bytes at `at`, or `None` past the end of `bytes`.
|
||||
fn le4(bytes: &[u8], at: usize) -> Option<[u8; 4]> {
|
||||
bytes.get(at..at.checked_add(4)?)?.try_into().ok()
|
||||
}
|
||||
|
||||
/// Whether `bytes` start a BMP whose header has the resolution fields.
|
||||
fn has_resolution_fields(bytes: &[u8]) -> bool {
|
||||
bytes.starts_with(b"BM") && le4(bytes, Y_PPM).is_some() && le4(bytes, 14).map(u32::from_le_bytes).is_some_and(|dib| dib >= INFO_HEADER)
|
||||
}
|
||||
|
||||
/// The resolution in pixels per inch a BMP records, or `None` when its header has no fields for it
|
||||
/// or either is 0 or negative (most writers leave them 0). A whole number of ppi stored rounded to
|
||||
/// whole pixels per metre (300 ppi = 11811) reads back whole, as a PNG's `pHYs` does.
|
||||
pub(crate) fn resolution(bytes: &[u8]) -> Option<(f32, f32)> {
|
||||
if !has_resolution_fields(bytes) {
|
||||
return None;
|
||||
}
|
||||
let field = |at: usize| le4(bytes, at).map(i32::from_le_bytes).and_then(|v| u32::try_from(v).ok());
|
||||
let (x, y) = (field(X_PPM)?, field(Y_PPM)?);
|
||||
(x > 0 && y > 0).then(|| (dpi_of_ppm(x), dpi_of_ppm(y)))
|
||||
}
|
||||
|
||||
/// Pixels per metre for `ppi`, rounded as Photoshop and the PNG `pHYs` writer do (300 ppi = 11811),
|
||||
/// or `None` for a resolution that has no positive whole number of pixels per metre.
|
||||
fn ppm_of_dpi(ppi: f32) -> Option<i32> {
|
||||
let ppm = (f64::from(ppi) / METERS_PER_INCH).round();
|
||||
(ppm.is_finite() && ppm >= 1.0).then(|| ppm.min(f64::from(i32::MAX)) as i32)
|
||||
}
|
||||
|
||||
/// Writes `dpi` into the resolution fields of the BMP in `bytes`. A header without the fields, or
|
||||
/// a resolution that cannot be stored, leaves the bytes as they are.
|
||||
pub(crate) fn set_resolution(bytes: &mut [u8], dpi: Option<(f32, f32)>) {
|
||||
let Some((x, y)) = dpi.and_then(|(x, y)| Some((ppm_of_dpi(x)?, ppm_of_dpi(y)?))) else { return };
|
||||
if !has_resolution_fields(bytes) {
|
||||
return;
|
||||
}
|
||||
for (at, ppm) in [(X_PPM, x), (Y_PPM, y)] {
|
||||
if let Some(field) = bytes.get_mut(at..at + 4) {
|
||||
field.copy_from_slice(&ppm.to_le_bytes());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A 46-byte BMP start: file header, a DIB header of `dib` bytes, 11811 px/m each way.
|
||||
fn start(dib: u32) -> Vec<u8> {
|
||||
let mut b = vec![0u8; Y_PPM + 4];
|
||||
b[..2].copy_from_slice(b"BM");
|
||||
b[14..18].copy_from_slice(&dib.to_le_bytes());
|
||||
b[X_PPM..X_PPM + 4].copy_from_slice(&11811i32.to_le_bytes());
|
||||
b[Y_PPM..Y_PPM + 4].copy_from_slice(&11811i32.to_le_bytes());
|
||||
b
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_headers_with_the_fields_are_read_or_written() {
|
||||
assert_eq!(resolution(&start(40)), Some((300.0, 300.0)));
|
||||
assert_eq!(resolution(&start(124)), Some((300.0, 300.0)), "BITMAPV5HEADER");
|
||||
// OS/2 1.x's 12-byte core header has no resolution: those bytes are pixels or palette.
|
||||
let mut core = start(12);
|
||||
assert_eq!(resolution(&core), None);
|
||||
set_resolution(&mut core, Some((72.0, 72.0)));
|
||||
assert_eq!(core, start(12), "left as it is");
|
||||
// Too short, or not a BMP: nothing read, nothing written, no panic.
|
||||
let mut short = start(40)[..Y_PPM + 3].to_vec();
|
||||
assert_eq!(resolution(&short), None);
|
||||
set_resolution(&mut short, Some((72.0, 72.0)));
|
||||
assert_eq!(short, start(40)[..Y_PPM + 3]);
|
||||
let mut other = start(40);
|
||||
other[0] = b'X';
|
||||
assert_eq!(resolution(&other), None);
|
||||
set_resolution(&mut other, Some((72.0, 72.0)));
|
||||
assert_eq!(&other[X_PPM..X_PPM + 4], &11811i32.to_le_bytes(), "not a BMP: left as it is");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolutions_without_a_whole_pixel_per_metre_are_not_written() {
|
||||
for bad in [0.0, -300.0, 0.01, f32::NAN, f32::INFINITY] {
|
||||
let mut b = start(40);
|
||||
set_resolution(&mut b, Some((bad, 300.0)));
|
||||
assert_eq!(b, start(40), "{bad}");
|
||||
}
|
||||
// A huge one is capped at the field's maximum rather than wrapping negative.
|
||||
let mut b = start(40);
|
||||
set_resolution(&mut b, Some((1e12, 300.0)));
|
||||
assert_eq!(&b[X_PPM..X_PPM + 4], &i32::MAX.to_le_bytes());
|
||||
}
|
||||
}
|
||||
@@ -304,19 +304,14 @@ pub(crate) fn decode_deep(meta: &MetaData, bytes: &[u8], limits: &Limits) -> Res
|
||||
let bytes_per_sample: usize = exr_types.iter().map(|t| sample_bytes(*t)).sum();
|
||||
let multipart = meta.requirements.has_multiple_layers;
|
||||
|
||||
// The offset tables follow the headers: `chunk_count` u64 per header, in header order.
|
||||
// Our part's table starts after the tables of all earlier parts.
|
||||
let mut table_pos = header_end(bytes)?;
|
||||
for h_ in meta.headers.iter().take(header_index) {
|
||||
table_pos = table_pos.saturating_add((h_.chunk_count.min(1 << 24) as u64).saturating_mul(8));
|
||||
}
|
||||
// A chunk offset before the end of the last offset table points into table data.
|
||||
let (table_pos, tables_end) = offset_tables(meta, bytes, header_index)?;
|
||||
let mut cur = Cursor::new(bytes);
|
||||
seek_to(&mut cur, table_pos)?;
|
||||
let mut offsets = Vec::with_capacity(header.chunk_count.min(1 << 20));
|
||||
for _ in 0..header.chunk_count {
|
||||
offsets.push(u64(&mut cur)?);
|
||||
}
|
||||
let tables_end = cur.position();
|
||||
|
||||
// Block geometry.
|
||||
let (tiles_x, tile_w, tile_h, slots) = match &header.blocks {
|
||||
@@ -565,6 +560,26 @@ fn header_end(bytes: &[u8]) -> Result<u64, CodecError> {
|
||||
Ok(cur.position())
|
||||
}
|
||||
|
||||
/// Where part `header_index`'s offset table starts and where the last part's table ends. The
|
||||
/// tables follow the headers, `chunk_count` u64 per header in header order, and the chunks
|
||||
/// follow the last table. Sizes that overflow, or tables that run past the end of the file,
|
||||
/// are errors.
|
||||
fn offset_tables(meta: &MetaData, bytes: &[u8], header_index: usize) -> Result<(u64, u64), CodecError> {
|
||||
let mut table_pos = header_end(bytes)?;
|
||||
let mut tables_end = table_pos;
|
||||
for (i, h) in meta.headers.iter().enumerate() {
|
||||
let len = u64::try_from(h.chunk_count).ok().and_then(|n| n.checked_mul(8));
|
||||
tables_end = len.and_then(|len| tables_end.checked_add(len)).ok_or_else(|| err("the offset tables are too large"))?;
|
||||
if i < header_index {
|
||||
table_pos = tables_end;
|
||||
}
|
||||
}
|
||||
if tables_end > bytes.len() as u64 {
|
||||
return Err(err(format!("the offset tables end at byte {tables_end}, past the end of the file")));
|
||||
}
|
||||
Ok((table_pos, tables_end))
|
||||
}
|
||||
|
||||
/// Composite the deep samples into a flat image: per pixel, samples sorted by Z (stable; the
|
||||
/// file is assumed messy), front-to-back `over` with premultiplied colour, un-premultiplied
|
||||
/// into straight alpha at the end. Without any alpha channel the front-most sample wins,
|
||||
|
||||
@@ -198,6 +198,15 @@ fn data_end(b: &[u8]) -> (DataEnd, u64) {
|
||||
data_end_mode(b, false)
|
||||
}
|
||||
|
||||
/// Whether JPEG `b` holds too little scan data for a `w` × `h` frame to be worth decoding: no scan
|
||||
/// data at all, or under one byte for every [`MAX_PIXELS_PER_BYTE`] pixels (counted on the scan
|
||||
/// data the marker walk crosses, so APP padding doesn't count). `zune-jpeg` allocates the whole
|
||||
/// output before it reads a scan, so a few bytes could otherwise ask for a frame of any size.
|
||||
pub(crate) fn scan_too_short(b: &[u8], w: u32, h: u32) -> bool {
|
||||
let (end, scan_bytes) = data_end(b);
|
||||
end == DataEnd::Empty || u64::from(w) * u64::from(h) / MAX_PIXELS_PER_BYTE > scan_bytes
|
||||
}
|
||||
|
||||
/// Whether the marker walk actually reaches the first EOI, with malformed segment lengths refused.
|
||||
pub(crate) fn reaches_eoi(b: &[u8]) -> bool {
|
||||
data_end_mode(b, true).0 == DataEnd::Complete
|
||||
@@ -322,6 +331,40 @@ fn without_extended_xmp(b: &[u8]) -> Option<Vec<u8>> {
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Whether the first frame header of `b` samples its first component below the largest
|
||||
/// horizontal or vertical sampling factor of the frame. zune-jpeg 0.5 sizes its upsampler's
|
||||
/// scratch row from the first component and asserts on some such frames (jpeg-encoder writes
|
||||
/// CMYK that way: C, M and Y at 1×1, K larger). `false` when no frame header can be read.
|
||||
fn first_component_subsampled(b: &[u8]) -> bool {
|
||||
if b.get(..2) != Some(&[0xFF, 0xD8]) {
|
||||
return false;
|
||||
}
|
||||
let mut i = 2;
|
||||
while let (Some(&0xFF), Some(&m)) = (b.get(i), b.get(i + 1)) {
|
||||
match m {
|
||||
0xFF => i += 1,
|
||||
0x01 | 0xD0..=0xD8 => i += 2,
|
||||
0xD9 | 0xDA => return false,
|
||||
_ => {
|
||||
let Some(len) = b.get(i + 2..i + 4).map(|l| usize::from(u16::from_be_bytes([l[0], l[1]]))) else { return false };
|
||||
if matches!(m, 0xC0..=0xCF) && !matches!(m, 0xC4 | 0xC8 | 0xCC) {
|
||||
let Some(seg) = b.get(i + 4..i + 2 + len.max(2)) else { return false };
|
||||
let n = usize::from(seg.get(5).copied().unwrap_or(0));
|
||||
let factors: Vec<(u8, u8)> = (0..n).filter_map(|c| seg.get(6 + c * 3 + 1)).map(|f| (f >> 4, f & 0x0F)).collect();
|
||||
let (Some(&(h0, v0)), Some(hmax), Some(vmax)) =
|
||||
(factors.first(), factors.iter().map(|f| f.0).max(), factors.iter().map(|f| f.1).max())
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
return h0 < hmax || v0 < vmax;
|
||||
}
|
||||
i += 2 + len;
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Whether `craft-jpeg` (rather than `zune-jpeg`) decodes the file: arithmetic coding
|
||||
/// (SOF9–SOF11), lossless (SOF3), or a DCT frame whose precision isn't 8 bits (#4934, #5003).
|
||||
/// This includes 12-bit progressive Huffman (SOF2), which `craft-jpeg` supports.
|
||||
@@ -389,6 +432,30 @@ fn decode_craft_jpeg(bytes: &[u8], limits: &Limits, adobe_transform: Option<u8>,
|
||||
/// truncated or not, are far above this.
|
||||
const MAX_PIXELS_PER_BYTE: u64 = 2048;
|
||||
|
||||
/// Decode with craft-jpeg: the files [`uses_craft_jpeg`] routes away from `zune-jpeg`, and the
|
||||
/// ones `zune-jpeg` panics on.
|
||||
fn decode_whole(bytes: &[u8], limits: &Limits, meta: JpegMeta, end: DataEnd, scan_bytes: u64) -> Result<Image, CodecError> {
|
||||
// craft-jpeg sizes its buffers from the frame header before it reads a scan.
|
||||
if scan_bytes == 0 {
|
||||
return Err(err("the file has no scan data"));
|
||||
}
|
||||
if let Some((w, h, _)) = meta.frame
|
||||
&& end == DataEnd::Truncated
|
||||
// Only truncated scans: arithmetic coding (SOF9–SOF11) can spend well under a bit on
|
||||
// a flat block, so a complete scan may be shorter than this bound.
|
||||
&& u64::from(w) * u64::from(h) / MAX_PIXELS_PER_BYTE > scan_bytes
|
||||
{
|
||||
return Err(err("the truncated scan is too short for the frame it declares"));
|
||||
}
|
||||
let (mut img, damaged) = decode_craft_jpeg(bytes, limits, meta.adobe_transform, meta.adobe_marker)?;
|
||||
img.icc = meta.icc;
|
||||
img.meta = Metadata { exif: meta.exif, xmp: meta.xmp, dpi: meta.dpi, photoshop_resources: meta.photoshop, ..Default::default() };
|
||||
if end == DataEnd::Truncated || damaged {
|
||||
img.warnings.push(DecodeWarning::Truncated { format: F });
|
||||
}
|
||||
Ok(img)
|
||||
}
|
||||
|
||||
/// The stored size and EXIF block [`decode`] would find, from the marker segments before the
|
||||
/// first scan: the first frame header's size, checked against `limits`, and the APP1 EXIF block.
|
||||
pub(crate) fn probe(bytes: &[u8], limits: &Limits) -> Result<(u32, u32, Option<Vec<u8>>), CodecError> {
|
||||
@@ -407,26 +474,9 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits) -> Result<Image, CodecError>
|
||||
if let Some((w, h, nc)) = meta.frame {
|
||||
limits.check_bytes(w, h, u64::from(nc.max(1)))?;
|
||||
}
|
||||
if uses_craft_jpeg(bytes) {
|
||||
// craft-jpeg sizes its buffers from the frame header before it reads a scan.
|
||||
if scan_bytes == 0 {
|
||||
return Err(err("the file has no scan data"));
|
||||
}
|
||||
if let Some((w, h, _)) = meta.frame
|
||||
&& end == DataEnd::Truncated
|
||||
// Only truncated scans: arithmetic coding (SOF9–SOF11) can spend well under a bit on
|
||||
// a flat block, so a complete scan may be shorter than this bound.
|
||||
&& u64::from(w) * u64::from(h) / MAX_PIXELS_PER_BYTE > scan_bytes
|
||||
{
|
||||
return Err(err("the truncated scan is too short for the frame it declares"));
|
||||
}
|
||||
let (mut img, damaged) = decode_craft_jpeg(bytes, limits, meta.adobe_transform, meta.adobe_marker)?;
|
||||
img.icc = meta.icc;
|
||||
img.meta = Metadata { exif: meta.exif, xmp: meta.xmp, dpi: meta.dpi, photoshop_resources: meta.photoshop, ..Default::default() };
|
||||
if end == DataEnd::Truncated || damaged {
|
||||
img.warnings.push(DecodeWarning::Truncated { format: F });
|
||||
}
|
||||
return Ok(img);
|
||||
// Where a panic aborts (wasm32), the frames zune-jpeg may panic on can't be caught below.
|
||||
if uses_craft_jpeg(bytes) || (cfg!(panic = "abort") && first_component_subsampled(bytes)) {
|
||||
return decode_whole(bytes, limits, meta, end, scan_bytes);
|
||||
}
|
||||
let options = DecoderOptions::default().set_strict_mode(false).set_max_width(65535).set_max_height(65535);
|
||||
let stripped = without_extended_xmp(bytes);
|
||||
@@ -452,7 +502,12 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits) -> Result<Image, CodecError>
|
||||
return Err(err(format!("{what} is too short for the frame it declares")));
|
||||
}
|
||||
dec.set_options(dec.options().jpeg_set_out_colorspace(out_cs));
|
||||
let mut px = dec.decode().map_err(err)?;
|
||||
// zune-jpeg 0.5 asserts on some subsampled four-component frames whose first component isn't
|
||||
// the largest (its upsampler's scratch row is sized from that component): craft-jpeg reads them.
|
||||
let Ok(decoded) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| dec.decode())) else {
|
||||
return decode_whole(bytes, limits, meta, end, scan_bytes);
|
||||
};
|
||||
let mut px = decoded.map_err(err)?;
|
||||
let expected = w as usize * h as usize * layout.channels();
|
||||
if px.len() < expected {
|
||||
return Err(err("short pixel buffer"));
|
||||
@@ -1018,6 +1073,25 @@ mod tests {
|
||||
(v * 65535.0 / 4095.0).round() as u16
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_component_subsampling() {
|
||||
let sof = |comps: &[(u8, u8)]| {
|
||||
let mut s = vec![0xFF, 0xD8, 0xFF, 0xE0, 0, 4, 0, 0, 0xFF, 0xC2, 0, (8 + 3 * comps.len()) as u8, 8, 0, 8, 0, 8, comps.len() as u8];
|
||||
for (i, &(h, v)) in comps.iter().enumerate() {
|
||||
s.extend([i as u8 + 1, (h << 4) | v, 0]);
|
||||
}
|
||||
s
|
||||
};
|
||||
assert!(first_component_subsampled(&sof(&[(1, 1), (1, 1), (1, 1), (2, 2)])));
|
||||
assert!(first_component_subsampled(&sof(&[(1, 2), (1, 2), (2, 1)])));
|
||||
assert!(!first_component_subsampled(&sof(&[(2, 2), (1, 1), (1, 1)])));
|
||||
assert!(!first_component_subsampled(&sof(&[(1, 1); 4])));
|
||||
assert!(!first_component_subsampled(&sof(&[])));
|
||||
assert!(!first_component_subsampled(&[0xFF, 0xD8, 0xFF, 0xDA, 0, 2]));
|
||||
assert!(!first_component_subsampled(&[0xFF, 0xD8, 0xFF, 0xC0, 0, 20, 8]));
|
||||
assert!(!first_component_subsampled(b"not a jpeg"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routing() {
|
||||
assert!(!uses_craft_jpeg(&[0xFF, 0xD8, 0xFF, 0xC0, 0, 11, 8, 0, 8, 0, 8, 1, 1, 0x11, 0]));
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
//! Sun Raster files (`.ras`, `.sun`): 1-bit, 8-bit (gray or colour-mapped), 24-bit RGB and 32-bit
|
||||
//! RGBA, uncompressed or byte-encoded (RLE) on read; uncompressed on write.
|
||||
//!
|
||||
//! Follows the published rasterfile format: a 32-byte big-endian header, an optional colour map,
|
||||
//! then rows padded to an even number of bytes. In the "standard" type (1) 24- and 32-bit pixels
|
||||
//! are stored blue-first; in the "RGB" type (3) red-first. Other types (TIFF, IFF, experimental)
|
||||
//! are reported as unsupported.
|
||||
|
||||
use crate::error::CodecError;
|
||||
use crate::fidelity::Plan;
|
||||
use crate::format::Format;
|
||||
use crate::image::{ChannelLayout, Image, SampleType};
|
||||
use crate::options::Limits;
|
||||
|
||||
const F: Format = Format::SunRaster;
|
||||
const HEADER: usize = 32;
|
||||
const MAGIC: u32 = 0x59a6_6a95;
|
||||
|
||||
fn bad(reason: impl std::fmt::Display) -> CodecError {
|
||||
CodecError::malformed(F, reason)
|
||||
}
|
||||
|
||||
fn be32(b: &[u8], i: usize) -> u32 {
|
||||
u32::from_be_bytes([b[i], b[i + 1], b[i + 2], b[i + 3]])
|
||||
}
|
||||
|
||||
/// Whether `b` starts with a Sun rasterfile header.
|
||||
pub(crate) fn looks_like_sunraster(b: &[u8]) -> bool {
|
||||
b.len() >= HEADER && be32(b, 0) == MAGIC && matches!(be32(b, 12), 1 | 8 | 24 | 32)
|
||||
}
|
||||
|
||||
/// Expand byte-encoded data (`0x80` escapes a run) until `want` bytes are produced.
|
||||
fn unrle(src: &[u8], want: usize) -> Result<Vec<u8>, CodecError> {
|
||||
let mut out = Vec::new();
|
||||
out.try_reserve_exact(want).map_err(|_| bad("pixel buffer is too large"))?;
|
||||
let mut i = 0;
|
||||
while out.len() < want {
|
||||
let b = *src.get(i).ok_or_else(|| bad("truncated RLE data"))?;
|
||||
i += 1;
|
||||
if b != 0x80 {
|
||||
out.push(b);
|
||||
continue;
|
||||
}
|
||||
let n = *src.get(i).ok_or_else(|| bad("truncated RLE data"))?;
|
||||
i += 1;
|
||||
if n == 0 {
|
||||
out.push(0x80);
|
||||
continue;
|
||||
}
|
||||
let v = *src.get(i).ok_or_else(|| bad("truncated RLE data"))?;
|
||||
i += 1;
|
||||
let count = usize::from(n) + 1;
|
||||
if count > want - out.len() {
|
||||
return Err(bad("RLE run passes the end of the image"));
|
||||
}
|
||||
out.resize(out.len() + count, v);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub(crate) fn decode(b: &[u8], limits: &Limits) -> Result<Image, CodecError> {
|
||||
if !looks_like_sunraster(b) {
|
||||
return Err(bad("invalid Sun raster header"));
|
||||
}
|
||||
let (w, h, depth, kind, map_type, map_len) = (be32(b, 4), be32(b, 8), be32(b, 12) as usize, be32(b, 20), be32(b, 24), be32(b, 28) as usize);
|
||||
if !matches!(kind, 0..=3) {
|
||||
return Err(CodecError::unsupported(F, format!("Sun raster type {kind} is not supported")));
|
||||
}
|
||||
if map_type > 2 {
|
||||
return Err(CodecError::unsupported(F, format!("Sun raster map type {map_type} is not supported")));
|
||||
}
|
||||
let layout = match depth {
|
||||
1 | 8 if map_type != 1 => ChannelLayout::Gray,
|
||||
1 | 8 | 24 => ChannelLayout::Rgb,
|
||||
_ => ChannelLayout::Rgba,
|
||||
};
|
||||
limits.check(w, h, layout, SampleType::U8)?;
|
||||
let (wu, hu) = (w as usize, h as usize);
|
||||
let stride = wu.checked_mul(depth).and_then(|n| n.checked_add(15)).ok_or_else(|| bad("image size overflow"))? / 16 * 2;
|
||||
let want = stride.checked_mul(hu).ok_or_else(|| bad("image size overflow"))?;
|
||||
let data_at = HEADER.checked_add(map_len).ok_or_else(|| bad("colour map size overflow"))?;
|
||||
let map = b.get(HEADER..data_at).ok_or_else(|| bad("truncated colour map"))?;
|
||||
let rest = &b[data_at..];
|
||||
let raw;
|
||||
let rows: &[u8] = if kind == 2 {
|
||||
raw = unrle(rest, want)?;
|
||||
&raw
|
||||
} else {
|
||||
rest.get(..want).ok_or_else(|| bad("the file is too short for the declared image"))?
|
||||
};
|
||||
let channels = layout.channels();
|
||||
let mut out = Vec::new();
|
||||
out.try_reserve_exact(wu * hu * channels).map_err(|_| bad("pixel buffer is too large"))?;
|
||||
let entries = map_len / 3;
|
||||
for y in 0..hu {
|
||||
let row = &rows[y * stride..(y + 1) * stride];
|
||||
for x in 0..wu {
|
||||
match depth {
|
||||
1 => {
|
||||
let bit = (row[x / 8] >> (7 - x % 8)) & 1;
|
||||
if map_type == 1 {
|
||||
// The bit is an index into the (two-entry or longer) colour map.
|
||||
let i = usize::from(bit);
|
||||
if i >= entries {
|
||||
return Err(bad("pixel index is outside the colour map"));
|
||||
}
|
||||
out.extend_from_slice(&[map[i], map[entries + i], map[2 * entries + i]]);
|
||||
} else {
|
||||
// A set bit is black.
|
||||
out.push(if bit == 1 { 0 } else { 255 });
|
||||
}
|
||||
}
|
||||
8 if map_type == 1 => {
|
||||
let i = usize::from(row[x]);
|
||||
if i >= entries {
|
||||
return Err(bad("pixel index is outside the colour map"));
|
||||
}
|
||||
out.extend_from_slice(&[map[i], map[entries + i], map[2 * entries + i]]);
|
||||
}
|
||||
8 => out.push(row[x]),
|
||||
24 => {
|
||||
let p = &row[3 * x..3 * x + 3];
|
||||
out.extend_from_slice(&if kind == 3 { [p[0], p[1], p[2]] } else { [p[2], p[1], p[0]] });
|
||||
}
|
||||
_ => {
|
||||
let p = &row[4 * x..4 * x + 4];
|
||||
// In the standard types the first byte (X in XBGR) is unused, not alpha.
|
||||
out.extend_from_slice(&if kind == 3 { [p[1], p[2], p[3], p[0]] } else { [p[3], p[2], p[1], 255] });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Image::from_u8(w, h, layout, out)
|
||||
}
|
||||
|
||||
/// A buffer of `cap` bytes, or an error instead of an abort when the allocation cannot be made.
|
||||
fn reserve(cap: usize) -> std::result::Result<Vec<u8>, std::collections::TryReserveError> {
|
||||
let mut v = Vec::new();
|
||||
v.try_reserve_exact(cap)?;
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// Writes an uncompressed rasterfile: 8-bit gray, 24-bit RGB or 32-bit RGBA, in the RGB-ordered type.
|
||||
pub(crate) fn encode(src: &Image, plan: Plan) -> Result<Vec<u8>, CodecError> {
|
||||
let enc = |msg: String| CodecError::encode(F, msg);
|
||||
let img = src.converted(plan.layout, plan.sample);
|
||||
if img.sample_type() != SampleType::U8 {
|
||||
return Err(enc(format!("Sun raster stores 8 bits per channel, not {:?}", img.sample_type())));
|
||||
}
|
||||
let depth: usize = match img.layout() {
|
||||
ChannelLayout::Gray => 8,
|
||||
ChannelLayout::Rgb => 24,
|
||||
ChannelLayout::Rgba => 32,
|
||||
l => return Err(enc(format!("no Sun raster layout for {l:?}"))),
|
||||
};
|
||||
let (w, h) = img.dimensions();
|
||||
let (wu, hu) = (w as usize, h as usize);
|
||||
let bytes = depth / 8;
|
||||
let stride = (wu * bytes).next_multiple_of(2);
|
||||
let len = stride.checked_mul(hu).and_then(|n| u32::try_from(n).ok()).ok_or_else(|| enc("image too large".into()))?;
|
||||
let data = img.data();
|
||||
if data.len() != wu * hu * bytes {
|
||||
return Err(enc("the image has the wrong number of samples".into()));
|
||||
}
|
||||
let mut out = reserve(HEADER.checked_add(len as usize).ok_or_else(|| enc("image too large".into()))?)
|
||||
.map_err(|_| enc("the image is too large to encode".into()))?;
|
||||
for v in [MAGIC, w, h, depth as u32, len, 3, 0, 0] {
|
||||
out.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
for y in 0..hu {
|
||||
let row = &data[y * wu * bytes..(y + 1) * wu * bytes];
|
||||
if depth == 32 {
|
||||
// Stored alpha first.
|
||||
for p in row.as_chunks::<4>().0 {
|
||||
out.extend_from_slice(&[p[3], p[0], p[1], p[2]]);
|
||||
}
|
||||
} else {
|
||||
out.extend_from_slice(row);
|
||||
}
|
||||
out.resize(HEADER + (y + 1) * stride, 0);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn pattern(w: u32, h: u32, layout: ChannelLayout) -> Image {
|
||||
let n = w as usize * h as usize * layout.channels();
|
||||
Image::from_u8(w, h, layout, (0..n).map(|i| ((i * 13 + i / 3) % 253) as u8).collect()).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trips_gray_rgb_and_rgba_with_odd_widths() {
|
||||
for layout in [ChannelLayout::Gray, ChannelLayout::Rgb, ChannelLayout::Rgba] {
|
||||
let img = pattern(13, 7, layout);
|
||||
let bytes = encode(&img, Plan { layout, sample: SampleType::U8 }).unwrap();
|
||||
assert!(looks_like_sunraster(&bytes));
|
||||
let back = decode(&bytes, &Limits::default()).unwrap();
|
||||
assert_eq!((back.layout(), back.data()), (layout, img.data()));
|
||||
}
|
||||
}
|
||||
|
||||
fn file(depth: u32, kind: u32, map: &[u8], map_type: u32, w: u32, h: u32, data: &[u8]) -> Vec<u8> {
|
||||
let mut f = Vec::new();
|
||||
for v in [MAGIC, w, h, depth, data.len() as u32, kind, map_type, map.len() as u32] {
|
||||
f.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
f.extend_from_slice(map);
|
||||
f.extend_from_slice(data);
|
||||
f
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_type_is_blue_first_and_rle_expands() {
|
||||
let f = file(24, 1, &[], 0, 1, 1, &[1, 2, 3, 0]);
|
||||
assert_eq!(decode(&f, &Limits::default()).unwrap().data(), &[3, 2, 1]);
|
||||
// 0x80 0x00 is a literal 0x80; 0x80 n v is n + 1 copies of v. The 2x2 image has stride 2.
|
||||
let f = file(8, 2, &[], 0, 2, 2, &[0x80, 0x00, 0x80, 0x00, 0x80, 0x01, 7]);
|
||||
assert_eq!(decode(&f, &Limits::default()).unwrap().data(), &[0x80, 0x80, 7, 7]);
|
||||
// A run that passes the end of the image is an error.
|
||||
assert!(decode(&file(8, 2, &[], 0, 2, 2, &[0x80, 0x09, 7]), &Limits::default()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn colour_mapped_and_one_bit_files() {
|
||||
// Two entries: red plane, green plane, blue plane.
|
||||
let map = [10, 20, 30, 40, 50, 60];
|
||||
let f = file(8, 1, &map, 1, 2, 1, &[0, 1]);
|
||||
let img = decode(&f, &Limits::default()).unwrap();
|
||||
assert_eq!((img.layout(), img.data()), (ChannelLayout::Rgb, &[10, 30, 50, 20, 40, 60][..]));
|
||||
let f = file(8, 1, &map, 1, 1, 1, &[5, 0]);
|
||||
assert!(decode(&f, &Limits::default()).is_err());
|
||||
let f = file(1, 1, &[], 0, 3, 1, &[0b1010_0000, 0]);
|
||||
assert_eq!(decode(&f, &Limits::default()).unwrap().data(), &[0, 255, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_bit_files_with_a_colour_map_read_as_colour() {
|
||||
// Two entries: red plane, green plane, blue plane; bit 0 is entry 0.
|
||||
let map = [10, 20, 30, 40, 50, 60];
|
||||
let f = file(1, 1, &map, 1, 3, 1, &[0b0100_0000, 0]);
|
||||
let img = decode(&f, &Limits::default()).unwrap();
|
||||
assert_eq!((img.layout(), img.data()), (ChannelLayout::Rgb, &[10, 30, 50, 20, 40, 60, 10, 30, 50][..]));
|
||||
// A one-entry map has no entry for a set bit.
|
||||
assert!(decode(&file(1, 1, &[1, 2, 3], 1, 1, 1, &[0b1000_0000, 0]), &Limits::default()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_unused_first_byte_of_standard_32_bit_pixels_is_not_alpha() {
|
||||
let img = decode(&file(32, 1, &[], 0, 1, 1, &[0, 30, 20, 10]), &Limits::default()).unwrap();
|
||||
assert_eq!((img.layout(), img.data()), (ChannelLayout::Rgba, &[10, 20, 30, 255][..]));
|
||||
// In the RGB-ordered type the first byte is the alpha the writer stores there.
|
||||
let img = decode(&file(32, 3, &[], 0, 1, 1, &[7, 10, 20, 30]), &Limits::default()).unwrap();
|
||||
assert_eq!(img.data(), &[10, 20, 30, 7]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_output_buffer_too_large_to_allocate_is_an_error() {
|
||||
assert!(reserve(usize::MAX).is_err());
|
||||
assert!(reserve(16).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_input_is_an_error_not_a_panic() {
|
||||
let img = pattern(9, 5, ChannelLayout::Rgb);
|
||||
let good = encode(&img, Plan { layout: ChannelLayout::Rgb, sample: SampleType::U8 }).unwrap();
|
||||
for len in 0..good.len() {
|
||||
let _ = decode(&good[..len], &Limits::default());
|
||||
}
|
||||
let mut huge = good.clone();
|
||||
huge[4..8].copy_from_slice(&u32::MAX.to_be_bytes());
|
||||
huge[8..12].copy_from_slice(&u32::MAX.to_be_bytes());
|
||||
assert!(decode(&huge, &Limits::default()).is_err());
|
||||
let mut bad_map = good;
|
||||
bad_map[28..32].copy_from_slice(&u32::MAX.to_be_bytes());
|
||||
assert!(decode(&bad_map, &Limits::default()).is_err());
|
||||
assert!(matches!(decode(&file(24, 4, &[], 0, 1, 1, &[0; 4]), &Limits::default()), Err(CodecError::Unsupported { .. })));
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,18 @@ const TAG_DNG_VERSION: u16 = 50706;
|
||||
/// first IFD is often a reduced preview, so this runs before decoding.
|
||||
/// Reads only bounds-checked header bytes; anything malformed is "not raw".
|
||||
fn camera_raw(b: &[u8]) -> bool {
|
||||
camera_raw_mode(b, false)
|
||||
}
|
||||
|
||||
/// The older LightCraft RAW/TIFF classification policy, sharing this bounded directory walk.
|
||||
pub(crate) fn legacy_camera_raw(b: &[u8]) -> bool {
|
||||
camera_raw_mode(b, true)
|
||||
}
|
||||
|
||||
fn camera_raw_mode(b: &[u8], legacy: bool) -> bool {
|
||||
if legacy && b.get(8..10) == Some(b"CR") {
|
||||
return true;
|
||||
}
|
||||
let le = match b.get(0..4) {
|
||||
Some(b"II*\0") => true,
|
||||
Some(b"MM\0*") => false,
|
||||
@@ -92,8 +104,48 @@ fn camera_raw(b: &[u8]) -> bool {
|
||||
let value = |e: usize| match u16_at(e.saturating_add(2))? {
|
||||
3 => u16_at(e.saturating_add(8)).map(u32::from),
|
||||
4 | 13 => u32_at(e.saturating_add(8)),
|
||||
1 | 7 if legacy => b.get(e.saturating_add(8)).copied().map(u32::from),
|
||||
_ => None,
|
||||
};
|
||||
// The compatibility policy recognizes Sony lossless YCbCr only when IFD0 names Sony.
|
||||
let is_sony = if legacy {
|
||||
u32_at(4)
|
||||
.and_then(|v| usize::try_from(v).ok())
|
||||
.and_then(|ifd| {
|
||||
let n = usize::from(u16_at(ifd)?);
|
||||
if n > 4096 {
|
||||
return None;
|
||||
}
|
||||
for i in 0..n {
|
||||
let e = ifd.checked_add(2)?.checked_add(i.checked_mul(12)?)?;
|
||||
let tag = u16_at(e)?;
|
||||
let ty = u16_at(e + 2)?;
|
||||
let count = u32_at(e + 4)? as usize;
|
||||
if tag == 0x010f {
|
||||
let matches = (|| {
|
||||
let size = match ty {
|
||||
1 | 2 | 6 | 7 => 1usize,
|
||||
3 | 8 => 2,
|
||||
4 | 9 | 11 | 13 => 4,
|
||||
5 | 10 | 12 => 8,
|
||||
_ => return None,
|
||||
};
|
||||
let len = size.checked_mul(count)?;
|
||||
let start = if len <= 4 { e + 8 } else { u32_at(e + 8)? as usize };
|
||||
Some(b.get(start..start.checked_add(len)?)?.get(..4)?.eq_ignore_ascii_case(b"SONY"))
|
||||
})()
|
||||
.unwrap_or(false);
|
||||
if matches {
|
||||
return Some(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
} else {
|
||||
false
|
||||
};
|
||||
let mut queue: Vec<u32> = u32_at(4).into_iter().collect();
|
||||
let mut visited = 0;
|
||||
while let Some(ifd) = queue.pop() {
|
||||
@@ -104,6 +156,44 @@ fn camera_raw(b: &[u8]) -> bool {
|
||||
}
|
||||
let ifd = ifd as usize;
|
||||
let Some(n) = u16_at(ifd) else { continue };
|
||||
if legacy
|
||||
&& (n > 4096
|
||||
|| (n > 0
|
||||
&& ifd
|
||||
.checked_add(2)
|
||||
.and_then(|p| usize::from(n - 1).checked_mul(12).and_then(|v| p.checked_add(v)))
|
||||
.and_then(|p| p.checked_add(8))
|
||||
.is_none_or(|end| end > b.len())))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if is_sony {
|
||||
let mut cfa = false;
|
||||
let (mut compression, mut photo, mut samples) = (None, None, None);
|
||||
let (mut saw_compression, mut saw_photo, mut saw_samples) = (false, false, false);
|
||||
for i in 0..usize::from(n) {
|
||||
let e = ifd.saturating_add(2 + 12 * i);
|
||||
match u16_at(e) {
|
||||
Some(0x828e) => cfa = true,
|
||||
Some(0x0103) if !saw_compression => {
|
||||
saw_compression = true;
|
||||
compression = value(e);
|
||||
}
|
||||
Some(0x0106) if !saw_photo => {
|
||||
saw_photo = true;
|
||||
photo = value(e);
|
||||
}
|
||||
Some(0x0115) if !saw_samples => {
|
||||
saw_samples = true;
|
||||
samples = value(e);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if cfa && compression == Some(7) && photo == Some(6) && samples == Some(3) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
for i in 0..usize::from(n) {
|
||||
let e = ifd.saturating_add(2 + 12 * i);
|
||||
let (Some(tag), Some(count)) = (u16_at(e), u32_at(e.saturating_add(4))) else {
|
||||
@@ -111,6 +201,7 @@ fn camera_raw(b: &[u8]) -> bool {
|
||||
};
|
||||
match tag {
|
||||
TAG_DNG_VERSION => return true,
|
||||
0x0103 if legacy && matches!(value(e), Some(32767 | 32769 | 32770 | 34316 | 34713 | 65000 | 65535)) => return true,
|
||||
TAG_PHOTOMETRIC if matches!(value(e), Some(PHOTOMETRIC_CFA | PHOTOMETRIC_LINEAR_RAW)) => {
|
||||
return true;
|
||||
}
|
||||
@@ -125,7 +216,7 @@ fn camera_raw(b: &[u8]) -> bool {
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if let Some(next) = u32_at(ifd.saturating_add(2 + 12 * usize::from(n))).filter(|&o| o != 0) {
|
||||
if !legacy && let Some(next) = u32_at(ifd.saturating_add(2 + 12 * usize::from(n))).filter(|&o| o != 0) {
|
||||
queue.push(next);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,6 +45,8 @@ pub enum Format {
|
||||
Iff,
|
||||
/// JPEG 2000 (JP2 container or raw J2K codestream): read-only; see [`ASYMMETRIC_EXCEPTIONS`].
|
||||
Jp2,
|
||||
/// Sun Raster (`.ras`): 1-, 8-, 24- and 32-bit, uncompressed or RLE.
|
||||
SunRaster,
|
||||
/// SGI image files (`.sgi`, `.rgb`, `.bw`): 8- and 16-bit gray, gray + alpha, RGB and RGBA.
|
||||
Sgi,
|
||||
/// JPEG-LS (`.jls`): lossless and near-lossless gray and RGB.
|
||||
@@ -120,7 +122,7 @@ const TIFF_LAYOUTS: &[ChannelLayout] = &[L::Gray, L::GrayA, L::Rgb, L::Rgba, L::
|
||||
|
||||
impl Format {
|
||||
/// Every format known to the crate (enabled or not).
|
||||
pub const ALL: [Format; 26] = [
|
||||
pub const ALL: [Format; 27] = [
|
||||
Format::Png,
|
||||
Format::Jpeg,
|
||||
Format::Jp2,
|
||||
@@ -145,6 +147,7 @@ impl Format {
|
||||
Format::Cineon,
|
||||
Format::Dicom,
|
||||
Format::Iff,
|
||||
Format::SunRaster,
|
||||
Format::Sgi,
|
||||
Format::JpegLs,
|
||||
];
|
||||
@@ -185,6 +188,7 @@ impl Format {
|
||||
Format::Dicom => "DICOM",
|
||||
Format::Iff => "Amiga IFF",
|
||||
Format::Jp2 => "JPEG 2000",
|
||||
Format::SunRaster => "Sun Raster",
|
||||
Format::Sgi => "SGI",
|
||||
Format::JpegLs => "JPEG-LS",
|
||||
}
|
||||
@@ -218,6 +222,7 @@ impl Format {
|
||||
Format::Dicom => &["dcm", "dicom", "dc3"],
|
||||
Format::Iff => &["iff", "lbm", "ilbm"],
|
||||
Format::Jp2 => &["jp2", "j2k", "j2c", "jpf", "jpx"],
|
||||
Format::SunRaster => &["ras", "sun"],
|
||||
Format::Sgi => &["sgi", "rgb", "rgba", "bw"],
|
||||
Format::JpegLs => &["jls"],
|
||||
}
|
||||
@@ -249,6 +254,7 @@ impl Format {
|
||||
Format::Dicom => "application/dicom",
|
||||
Format::Iff => "image/x-iff",
|
||||
Format::Jp2 => "image/jp2",
|
||||
Format::SunRaster => "image/x-sun-raster",
|
||||
Format::Sgi => "image/x-sgi",
|
||||
Format::JpegLs => "image/jls",
|
||||
}
|
||||
@@ -302,7 +308,8 @@ pub fn caps(format: Format) -> FormatCaps {
|
||||
// Resolution travels in EXIF (WebP has no field of its own).
|
||||
Format::WebP => FormatCaps { icc: true, exif: true, xmp: true, dpi: true, animation: true, ..base },
|
||||
Format::Gif => FormatCaps { layouts: &[L::Rgba], animation: true, lossy: true, ..base },
|
||||
Format::Bmp => base,
|
||||
// Resolution in the header's pixels per metre (BITMAPINFOHEADER and later).
|
||||
Format::Bmp => FormatCaps { dpi: true, ..base },
|
||||
Format::Tga => FormatCaps { layouts: RGB_GRAY, ..base },
|
||||
Format::Ico => FormatCaps { layouts: &[L::Rgba], ..base },
|
||||
Format::Pnm => FormatCaps { depths: &[S::U8, S::U16, S::F32], layouts: ALL_LAYOUTS, ..base },
|
||||
@@ -327,6 +334,13 @@ pub fn caps(format: Format) -> FormatCaps {
|
||||
Format::Cineon => FormatCaps { depths: &[S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base },
|
||||
Format::Dicom => FormatCaps { read: true, write: false, depths: &[S::U8, S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base },
|
||||
Format::Iff => FormatCaps { depths: &[S::U8], layouts: &[L::Rgb, L::Rgba], alpha: true, ..base },
|
||||
Format::SunRaster => FormatCaps {
|
||||
read: cfg!(feature = "sunraster"),
|
||||
write: cfg!(feature = "sunraster"),
|
||||
layouts: &[L::Gray, L::Rgb, L::Rgba],
|
||||
alpha: true,
|
||||
..base
|
||||
},
|
||||
Format::Sgi => {
|
||||
FormatCaps { read: cfg!(feature = "sgi"), write: cfg!(feature = "sgi"), depths: &[S::U8, S::U16], layouts: RGB_GRAY, alpha: true, ..base }
|
||||
}
|
||||
@@ -412,6 +426,10 @@ pub fn detect(bytes: &[u8]) -> Option<Format> {
|
||||
if b.starts_with(b"/* XPM */") || b.starts_with(b"! XPM2\n") || b.starts_with(b"! XPM2\r\n") {
|
||||
return Some(Format::Xpm);
|
||||
}
|
||||
#[cfg(feature = "sunraster")]
|
||||
if crate::codecs::sunraster::looks_like_sunraster(b) {
|
||||
return Some(Format::SunRaster);
|
||||
}
|
||||
#[cfg(feature = "sgi")]
|
||||
if crate::codecs::sgi::looks_like_sgi(b) {
|
||||
return Some(Format::Sgi);
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
//! A JPEG's headers and four-component samples read the way `zune-jpeg` and libjpeg-style readers
|
||||
//! give them, for apps that keep a CMYK JPEG's inks: the component count and ICC profile from the
|
||||
//! headers alone, and the ink amounts of a CMYK or YCCK JPEG.
|
||||
//!
|
||||
//! [`crate::decode`] reads the same files differently, on purpose: it keeps the samples of a
|
||||
//! four-component JPEG without Adobe's marker as they are, rounds its YCCK conversion, and hands
|
||||
//! arithmetic-coded, lossless and 12-bit files to its whole-coefficient decoder. The readers here
|
||||
//! follow the older convention instead, so a document that kept a CMYK JPEG's inks that way reads
|
||||
//! the same: every four-component file is taken to hold Adobe's inverted samples, YCCK is turned
|
||||
//! into CMYK as libjpeg does (truncating), and only the 8-bit Huffman-coded frames `zune-jpeg`
|
||||
//! decodes are read.
|
||||
|
||||
use zune_core::bytestream::ZCursor;
|
||||
use zune_core::colorspace::ColorSpace;
|
||||
use zune_core::options::DecoderOptions;
|
||||
use zune_jpeg::JpegDecoder;
|
||||
|
||||
use crate::Format;
|
||||
use crate::error::CodecError;
|
||||
use crate::image::{ChannelLayout, Image, SampleType};
|
||||
use crate::options::Limits;
|
||||
|
||||
/// A decoder of `bytes` whose headers have been read, with the default options (16384 pixels a
|
||||
/// side at most); `None` when they aren't a JPEG or its headers can't be read.
|
||||
fn headers(bytes: &[u8]) -> Option<JpegDecoder<ZCursor<&[u8]>>> {
|
||||
if !bytes.starts_with(&[0xFF, 0xD8]) {
|
||||
return None;
|
||||
}
|
||||
let mut d = JpegDecoder::new_with_options(ZCursor::new(bytes), DecoderOptions::default());
|
||||
d.decode_headers().ok()?;
|
||||
Some(d)
|
||||
}
|
||||
|
||||
/// The number of colour components the frame header of JPEG `bytes` declares, whatever it is;
|
||||
/// `None` when they aren't a JPEG whose headers can be read.
|
||||
pub fn jpeg_components(bytes: &[u8]) -> Option<usize> {
|
||||
Some(usize::from(headers(bytes)?.info()?.components))
|
||||
}
|
||||
|
||||
/// The width and height the frame header of JPEG `bytes` declares, for any colour type; `None`
|
||||
/// when they aren't a JPEG whose headers can be read (which includes sides past 16384 pixels).
|
||||
pub fn jpeg_dimensions(bytes: &[u8]) -> Option<(u32, u32)> {
|
||||
let (w, h) = headers(bytes)?.dimensions()?;
|
||||
Some((u32::try_from(w).ok()?, u32::try_from(h).ok()?))
|
||||
}
|
||||
|
||||
/// The ICC profile of JPEG `bytes`, put together from all its numbered `ICC_PROFILE` APP2
|
||||
/// segments; `None` without one, when it is empty, and when the headers can't be read.
|
||||
pub fn jpeg_icc_profile(bytes: &[u8]) -> Option<Vec<u8>> {
|
||||
headers(bytes)?.icc_profile().filter(|icc| !icc.is_empty())
|
||||
}
|
||||
|
||||
/// The ink amounts of a four-component (CMYK or YCCK) JPEG: C, M, Y and K bytes a pixel, 0 for
|
||||
/// no ink, as a [`ChannelLayout::Cmyk`] image. The samples are taken to be Adobe's inverted ones
|
||||
/// whether or not the file has Adobe's marker; YCCK is first turned into CMYK as libjpeg does
|
||||
/// (the complement of the YCbCr colour's RGB, each value truncated).
|
||||
///
|
||||
/// Errors: bytes that aren't a JPEG with readable headers ([`CodecError::InvalidImage`]); one of
|
||||
/// other than four components, or a frame `zune-jpeg` doesn't decode, such as an arithmetic-coded
|
||||
/// or 12-bit one ([`CodecError::Unsupported`]); a size past `limits`; scan data too short for the
|
||||
/// frame (under a byte for every 2048 pixels, refused before anything is allocated); and data that
|
||||
/// doesn't decode,
|
||||
/// including frames `zune-jpeg` panics on, which are caught.
|
||||
pub fn decode_jpeg_inks(bytes: &[u8], limits: &Limits) -> Result<Image, CodecError> {
|
||||
let bad = |m: &str| CodecError::InvalidImage(format!("JPEG inks: {m}"));
|
||||
let mut d = headers(bytes).ok_or_else(|| bad("not a JPEG with readable headers"))?;
|
||||
let space = d.input_colorspace().ok_or_else(|| bad("no colour space"))?;
|
||||
let components = d.info().map_or(0, |i| i.components);
|
||||
if components != 4 || !matches!(space, ColorSpace::CMYK | ColorSpace::YCCK) {
|
||||
return Err(CodecError::unsupported(Format::Jpeg, format!("inks of a {components}-component {space:?} image")));
|
||||
}
|
||||
let (w, h) = d.dimensions().ok_or_else(|| bad("no size"))?;
|
||||
let (w, h) = (u32::try_from(w).map_err(|_| bad("width"))?, u32::try_from(h).map_err(|_| bad("height"))?);
|
||||
limits.check(w, h, ChannelLayout::Cmyk, SampleType::U8)?;
|
||||
if crate::codecs::jpeg::scan_too_short(bytes, w, h) {
|
||||
return Err(bad("the scan data is too short for the frame it declares"));
|
||||
}
|
||||
d.set_options(DecoderOptions::default().jpeg_set_out_colorspace(space));
|
||||
// zune-jpeg 0.5 asserts on some subsampled four-component frames whose first component isn't
|
||||
// the largest (a scratch row too short for its upsampler): that is an error here, not a panic.
|
||||
let decoded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| d.decode()));
|
||||
let mut data = decoded
|
||||
.map_err(|_| CodecError::unsupported(Format::Jpeg, "inks: zune-jpeg failed on this file"))?
|
||||
.map_err(|e| CodecError::unsupported(Format::Jpeg, format!("inks: {e}")))?;
|
||||
for p in data.as_chunks_mut::<4>().0 {
|
||||
if space == ColorSpace::YCCK {
|
||||
let [y, cb, cr] = [p[0], p[1], p[2]].map(f32::from);
|
||||
// libjpeg's YCCK → CMYK: the complement of the YCbCr colour's RGB.
|
||||
p[0] = (434.456 - y - 1.402 * cr).clamp(0.0, 255.0) as u8;
|
||||
p[1] = (119.541 - y + 0.344 * cb + 0.714 * cr).clamp(0.0, 255.0) as u8;
|
||||
p[2] = (481.816 - y - 1.772 * cb).clamp(0.0, 255.0) as u8;
|
||||
}
|
||||
for v in p.iter_mut() {
|
||||
*v = 255 - *v;
|
||||
}
|
||||
}
|
||||
Image::from_u8(w, h, ChannelLayout::Cmyk, data)
|
||||
}
|
||||
@@ -26,15 +26,19 @@ mod fidelity;
|
||||
mod format;
|
||||
mod image;
|
||||
mod inspection;
|
||||
pub mod jpeg_inks;
|
||||
pub mod jpeg_sef;
|
||||
mod lab;
|
||||
mod options;
|
||||
pub mod orientation;
|
||||
pub mod png_probe;
|
||||
pub mod raw_metadata;
|
||||
pub mod resolution;
|
||||
pub mod resolution_probe;
|
||||
pub mod source_settings;
|
||||
pub mod tiff_probe;
|
||||
pub mod web;
|
||||
pub mod webp_probe;
|
||||
pub use crate::allocation::{probe_allocation_size, probe_stored_size};
|
||||
pub use crate::cicp::Cicp;
|
||||
pub use crate::cicp::ContentLightLevel;
|
||||
@@ -64,12 +68,15 @@ pub use crate::options::{
|
||||
};
|
||||
pub use crate::orientation::{OrientSteps, exif_orientation, upright_exif, upright_xmp};
|
||||
pub use crate::resolution::{exif_resolution, export_exif, export_xmp, photoshop_resolution, xmp_resolution};
|
||||
pub use crate::resolution_probe::file_resolution;
|
||||
pub use crate::source_settings::{WebpEncoding, jpeg_quality, webp_encoding};
|
||||
pub use half::f16;
|
||||
|
||||
#[cfg(feature = "sgi")]
|
||||
use crate::codecs::sgi;
|
||||
use crate::codecs::{dds, dicom, dpx, exr, fits, heif, iff, jp2, jpeg, jpegls, jxl, pcx, png, pnm, tiff, via_image, webp, xpm};
|
||||
#[cfg(feature = "sunraster")]
|
||||
use crate::codecs::sunraster;
|
||||
use crate::codecs::{bmp, dds, dicom, dpx, exr, fits, heif, iff, jp2, jpeg, jpegls, jxl, pcx, png, pnm, tiff, via_image, webp, xpm};
|
||||
|
||||
/// Detect the format and decode with default [`Limits`].
|
||||
pub fn decode(bytes: &[u8]) -> Result<Image, CodecError> {
|
||||
@@ -108,6 +115,16 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res
|
||||
Format::Cineon => dpx::decode_cineon(bytes, l),
|
||||
Format::Dicom => dicom::decode(bytes, l),
|
||||
Format::Iff => iff::decode(bytes, l),
|
||||
Format::SunRaster => {
|
||||
#[cfg(feature = "sunraster")]
|
||||
{
|
||||
sunraster::decode(bytes, l)
|
||||
}
|
||||
#[cfg(not(feature = "sunraster"))]
|
||||
{
|
||||
Err(CodecError::unsupported(format, "Sun Raster support requires the sunraster feature"))
|
||||
}
|
||||
}
|
||||
Format::Sgi => {
|
||||
#[cfg(feature = "sgi")]
|
||||
{
|
||||
@@ -127,7 +144,11 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res
|
||||
Format::Avif => codecs::avif_decode::decode(bytes, l, opts.keep_orientation),
|
||||
#[cfg(not(all(feature = "avif-decode", not(target_arch = "wasm32"))))]
|
||||
Format::Avif => via_image::decode(format, bytes, l),
|
||||
Format::Gif | Format::Bmp | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr => via_image::decode(format, bytes, l),
|
||||
Format::Bmp => via_image::decode(format, bytes, l).map(|mut img| {
|
||||
img.meta.dpi = bmp::resolution(bytes);
|
||||
img
|
||||
}),
|
||||
Format::Gif | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr => via_image::decode(format, bytes, l),
|
||||
}?;
|
||||
finish_format(format, bytes, img, opts)
|
||||
}
|
||||
@@ -257,6 +278,12 @@ pub fn probe_size(bytes: &[u8]) -> Result<(u32, u32), CodecError> {
|
||||
Ok(if (5..=8).contains(&o) { (h, w) } else { (w, h) })
|
||||
}
|
||||
|
||||
/// Classify classic TIFF camera RAWs using LightCraft's historical bounded heuristic.
|
||||
/// This compatibility policy differs from the decoder's own RAW rejection policy.
|
||||
pub fn legacy_tiff_raw_hint(bytes: &[u8]) -> bool {
|
||||
tiff::legacy_camera_raw(bytes)
|
||||
}
|
||||
|
||||
/// A JPEG's frame as its header declares it, read without decoding pixels: the stored size, the
|
||||
/// layout [`decode`] returns, whether [`decode`] hands the file to its whole-coefficient
|
||||
/// decoder (arithmetic-coded, lossless or 12-bit frames), whose working set is several times the
|
||||
@@ -304,10 +331,26 @@ pub fn bmp_palette(bytes: &[u8]) -> Result<Option<Vec<[u8; 3]>>, CodecError> {
|
||||
|
||||
/// Encodes `indices` (row-major, one byte per pixel) as an 8-bit palette BMP with the colour
|
||||
/// table `palette` (1 to 256 entries). Used for Indexed Color documents; BMP has no transparency.
|
||||
/// The header's resolution is left unset; [`encode_bmp_indexed_with_dpi`] writes one.
|
||||
pub fn encode_bmp_indexed(width: u32, height: u32, indices: &[u8], palette: &[[u8; 3]]) -> Result<Vec<u8>, CodecError> {
|
||||
via_image::encode_bmp_indexed(width, height, indices, palette)
|
||||
}
|
||||
|
||||
/// [`encode_bmp_indexed`] with the resolution `dpi` (pixels per inch, horizontal then vertical)
|
||||
/// in the header's pixels per metre, as [`encode`] writes [`Metadata::dpi`] for a BMP. `None`, or
|
||||
/// a resolution with no positive whole number of pixels per metre, leaves the fields unset.
|
||||
pub fn encode_bmp_indexed_with_dpi(
|
||||
width: u32,
|
||||
height: u32,
|
||||
indices: &[u8],
|
||||
palette: &[[u8; 3]],
|
||||
dpi: Option<(f32, f32)>,
|
||||
) -> Result<Vec<u8>, CodecError> {
|
||||
let mut bytes = via_image::encode_bmp_indexed(width, height, indices, palette)?;
|
||||
bmp::set_resolution(&mut bytes, dpi);
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
/// Lists the pages (image directories) of a TIFF or BigTIFF file without decoding pixels: the
|
||||
/// main IFD chain and its SubIFDs, each with its size, kind (page, reduced-resolution copy or
|
||||
/// mask) and storage. [`TiffInfo::default_page`] is the page [`decode`] opens.
|
||||
@@ -500,6 +543,16 @@ pub fn encode(image: &Image, format: Format, opts: &EncodeOptions) -> Result<Vec
|
||||
Format::Cineon => dpx::encode_cineon(image, plan),
|
||||
Format::Dicom => Err(CodecError::unsupported(format, "DICOM is import-only; save as TIFF or PNG instead")),
|
||||
Format::Iff => iff::encode(image, plan),
|
||||
Format::SunRaster => {
|
||||
#[cfg(feature = "sunraster")]
|
||||
{
|
||||
sunraster::encode(image, plan)
|
||||
}
|
||||
#[cfg(not(feature = "sunraster"))]
|
||||
{
|
||||
Err(CodecError::unsupported(format, "Sun Raster support requires the sunraster feature"))
|
||||
}
|
||||
}
|
||||
Format::Sgi => {
|
||||
#[cfg(feature = "sgi")]
|
||||
{
|
||||
@@ -515,8 +568,12 @@ pub fn encode(image: &Image, format: Format, opts: &EncodeOptions) -> Result<Vec
|
||||
Format::Heif => Err(CodecError::unsupported(format, "encoding is not available in this build")),
|
||||
Format::Jp2 => Err(CodecError::unsupported(format, "JPEG 2000 is import-only; save as PNG or TIFF instead")),
|
||||
Format::Jxl => jxl::encode(image, plan),
|
||||
Format::Gif | Format::Bmp | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr | Format::Avif => {
|
||||
via_image::encode(format, image, plan, opts)
|
||||
}
|
||||
// The `image` crate leaves a BMP's resolution fields 0; Photoshop fills them in. They are
|
||||
// part of the header, not metadata, so they are written whatever `embed_metadata` says.
|
||||
Format::Bmp => via_image::encode(format, image, plan, opts).map(|mut bytes| {
|
||||
bmp::set_resolution(&mut bytes, image.meta.dpi);
|
||||
bytes
|
||||
}),
|
||||
Format::Gif | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr | Format::Avif => via_image::encode(format, image, plan, opts),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,7 +88,7 @@ pub fn header_with_max_pixels(b: &[u8], max_pixels: u64) -> Result<(u32, u32, u1
|
||||
if !after_idat {
|
||||
return Err(err("truncated image data"));
|
||||
}
|
||||
Ok((w, h, exif.map_or(1, crate::exif_orientation)))
|
||||
Ok((w, h, exif.map_or(1, craft_metadata::compat_orientation::orientation)))
|
||||
}
|
||||
|
||||
/// Container color declarations preceding image data. Raw values leave color policy to callers.
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
//! Explicit verbatim EXIF and JPEG APP metadata helpers for callers that preserve stored orientation.
|
||||
//! Normal encoder APIs still normalize metadata for upright pixels; callers opt in to these helpers.
|
||||
use crate::{CodecError, Format};
|
||||
|
||||
/// Borrowed metadata used to construct JPEG APP segments.
|
||||
#[derive(Clone, Copy, Default)]
|
||||
pub struct JpegMetadata<'a> {
|
||||
/// TIFF-structured EXIF bytes.
|
||||
pub exif: Option<&'a [u8]>,
|
||||
/// XMP packet.
|
||||
pub xmp: Option<&'a str>,
|
||||
/// ICC profile bytes.
|
||||
pub icc: Option<&'a [u8]>,
|
||||
/// Pixels per inch, if nonzero.
|
||||
pub ppi: Option<u16>,
|
||||
}
|
||||
|
||||
fn error(format: Format, message: &str) -> CodecError {
|
||||
CodecError::encode(format, message)
|
||||
}
|
||||
|
||||
/// Build JFIF, EXIF, XMP and ICC APP payloads in that order, preserving bytes and segment limits.
|
||||
pub fn jpeg_segments(meta: JpegMetadata<'_>) -> Result<Vec<(u8, Vec<u8>)>, CodecError> {
|
||||
let mut segments = Vec::new();
|
||||
let density = meta.ppi.filter(|p| *p > 0).map_or([0, 0, 1, 0, 1], |p| {
|
||||
let [hi, lo] = p.to_be_bytes();
|
||||
[1, hi, lo, hi, lo]
|
||||
});
|
||||
segments.push((0xe0, [&b"JFIF\0\x01\x02"[..], &density, &[0, 0]].concat()));
|
||||
if let Some(exif) = meta.exif {
|
||||
if exif.len() > 65_527 {
|
||||
return Err(error(Format::Jpeg, "EXIF larger than one APP1 segment (64 KiB)"));
|
||||
}
|
||||
segments.push((0xe1, [&b"Exif\0\0"[..], exif].concat()));
|
||||
}
|
||||
if let Some(xmp) = meta.xmp {
|
||||
segments.push((0xe1, jpeg_xmp(xmp)?));
|
||||
}
|
||||
if let Some(icc) = meta.icc {
|
||||
let chunks: Vec<_> = icc.chunks(65_519).collect();
|
||||
if chunks.len() > 255 {
|
||||
return Err(error(Format::Jpeg, "ICC profile too large for JPEG"));
|
||||
}
|
||||
for (i, chunk) in chunks.iter().enumerate() {
|
||||
segments.push((0xe2, [&b"ICC_PROFILE\0"[..], &[i as u8 + 1, chunks.len() as u8], chunk].concat()));
|
||||
}
|
||||
}
|
||||
Ok(segments)
|
||||
}
|
||||
|
||||
/// Build a single standard XMP APP1 payload, rejecting extended-XMP-sized packets.
|
||||
pub fn jpeg_xmp(xmp: &str) -> Result<Vec<u8>, CodecError> {
|
||||
let payload = [&b"http://ns.adobe.com/xap/1.0/\0"[..], xmp.as_bytes()].concat();
|
||||
if payload.len() > 65_533 {
|
||||
return Err(error(Format::Jpeg, "XMP larger than one APP1 segment (extended XMP not supported)"));
|
||||
}
|
||||
Ok(payload)
|
||||
}
|
||||
|
||||
/// Restore bytes in the first existing JPEG EXIF payload when it has room. No segment is inserted.
|
||||
///
|
||||
/// The EXIF APP1 is found by walking the marker segments before the first scan, so an `Exif\0\0`
|
||||
/// inside another segment (a comment, say) is never taken for it, and the copy must fit inside
|
||||
/// the APP1's own declared length, not merely inside the file.
|
||||
pub fn replace_jpeg_exif(jpeg: &mut [u8], exif: &[u8]) {
|
||||
if let Some(payload) = jpeg_exif_payload(jpeg)
|
||||
&& let Some(dst) = jpeg.get_mut(payload).and_then(|p| p.get_mut(..exif.len()))
|
||||
{
|
||||
dst.copy_from_slice(exif);
|
||||
}
|
||||
}
|
||||
|
||||
/// The byte range of the TIFF data in the first EXIF APP1 (after `Exif\0\0`), up to the end of
|
||||
/// that segment; `None` when the header segments end, are damaged, or hold no EXIF APP1.
|
||||
fn jpeg_exif_payload(jpeg: &[u8]) -> Option<std::ops::Range<usize>> {
|
||||
if jpeg.get(..2) != Some(&[0xff, 0xd8][..]) {
|
||||
return None;
|
||||
}
|
||||
let mut p = 2usize;
|
||||
loop {
|
||||
// fill bytes (0xff) may precede a marker
|
||||
while jpeg.get(p) == Some(&0xff) && jpeg.get(p + 1) == Some(&0xff) {
|
||||
p += 1;
|
||||
}
|
||||
let (&0xff, &marker) = (jpeg.get(p)?, jpeg.get(p + 1)?) else { return None };
|
||||
match marker {
|
||||
// TEM and RSTn carry no length
|
||||
0x01 | 0xd0..=0xd7 => p += 2,
|
||||
// start of scan, end of image: no header segment follows
|
||||
0xda | 0xd9 | 0x00 => return None,
|
||||
_ => {
|
||||
let len = usize::from(u16::from_be_bytes([*jpeg.get(p + 2)?, *jpeg.get(p + 3)?]));
|
||||
let end = p.checked_add(2)?.checked_add(len).filter(|&e| len >= 2 && e <= jpeg.len())?;
|
||||
if marker == 0xe1 && jpeg.get(p + 4..end)?.starts_with(b"Exif\0\0") {
|
||||
return Some(p + 10..end);
|
||||
}
|
||||
p = end;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Replace the existing PNG eXIf chunk, preserving the other chunks and recomputing its CRC.
|
||||
/// No chunk is inserted when one is absent.
|
||||
pub fn replace_png_exif(png: &mut Vec<u8>, exif: &[u8]) -> Result<(), CodecError> {
|
||||
let len = u32::try_from(exif.len()).map_err(|_| error(Format::Png, "EXIF too large"))?;
|
||||
let mut p = 8usize;
|
||||
while let Some(bytes) = p.checked_add(4).and_then(|end| png.get(p..end)) {
|
||||
let size = u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as usize;
|
||||
let Some(end) = p.checked_add(12).and_then(|v| v.checked_add(size)).filter(|end| *end <= png.len()) else { break };
|
||||
if png.get(p + 4..p + 8) == Some(b"eXIf") {
|
||||
let mut chunk = Vec::new();
|
||||
chunk.extend_from_slice(&len.to_be_bytes());
|
||||
chunk.extend_from_slice(b"eXIf");
|
||||
chunk.extend_from_slice(exif);
|
||||
let crc = crc32fast::hash(&chunk[4..]);
|
||||
chunk.extend_from_slice(&crc.to_be_bytes());
|
||||
png.splice(p..end, chunk);
|
||||
break;
|
||||
}
|
||||
p = end;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Replace the existing WebP EXIF chunk, including padding and the RIFF size. No chunk is inserted
|
||||
/// when one is absent. The input must be an encoded WebP as emitted by this crate.
|
||||
pub fn replace_webp_exif(webp: &mut Vec<u8>, exif: &[u8]) -> Result<(), CodecError> {
|
||||
let len = u32::try_from(exif.len()).map_err(|_| error(Format::WebP, "EXIF too large"))?;
|
||||
let mut p = 12usize;
|
||||
while let Some(h) = webp.get(p..p.saturating_add(8)) {
|
||||
let size = u32::from_le_bytes([h[4], h[5], h[6], h[7]]) as usize;
|
||||
let end = p.saturating_add(8).saturating_add(size).saturating_add(size & 1);
|
||||
if &h[..4] == b"EXIF" && end <= webp.len() {
|
||||
let new_len = webp
|
||||
.len()
|
||||
.checked_sub(end - p)
|
||||
.and_then(|v| v.checked_add(8))
|
||||
.and_then(|v| v.checked_add(exif.len()))
|
||||
.and_then(|v| v.checked_add(exif.len() & 1))
|
||||
.ok_or_else(|| error(Format::WebP, "RIFF too large"))?;
|
||||
let riff_len = u32::try_from(new_len.saturating_sub(8)).map_err(|_| error(Format::WebP, "RIFF too large"))?;
|
||||
let mut chunk = Vec::new();
|
||||
chunk.extend_from_slice(b"EXIF");
|
||||
chunk.extend_from_slice(&len.to_le_bytes());
|
||||
chunk.extend_from_slice(exif);
|
||||
if exif.len() & 1 == 1 {
|
||||
chunk.push(0);
|
||||
}
|
||||
webp.splice(p..end, chunk);
|
||||
webp[4..8].copy_from_slice(&riff_len.to_le_bytes());
|
||||
break;
|
||||
}
|
||||
p = end;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
const EXIF: &[u8] = b"II\x2a\0\x08\0\0\0\x01\0\x12\x01\x03\0\x01\0\0\0\x06\0\0\0\0\0\0\0";
|
||||
#[test]
|
||||
fn replacements_match_previous_code_on_mutated_containers() {
|
||||
fn old_png(png: &mut Vec<u8>, exif: &[u8]) {
|
||||
let mut p = 8usize;
|
||||
while let Some(b) = png.get(p..p + 4) {
|
||||
let size = u32::from_be_bytes([b[0], b[1], b[2], b[3]]) as usize;
|
||||
let end = p.saturating_add(12).saturating_add(size);
|
||||
if end > png.len() {
|
||||
break;
|
||||
}
|
||||
if png.get(p + 4..p + 8) == Some(b"eXIf") {
|
||||
let mut chunk = (exif.len() as u32).to_be_bytes().to_vec();
|
||||
chunk.extend_from_slice(b"eXIf");
|
||||
chunk.extend_from_slice(exif);
|
||||
let mut crc = 0xffff_ffffu32;
|
||||
for &v in &chunk[4..] {
|
||||
crc ^= u32::from(v);
|
||||
for _ in 0..8 {
|
||||
crc = if crc & 1 != 0 { (crc >> 1) ^ 0xedb8_8320 } else { crc >> 1 };
|
||||
}
|
||||
}
|
||||
chunk.extend_from_slice(&(!crc).to_be_bytes());
|
||||
png.splice(p..end, chunk);
|
||||
return;
|
||||
}
|
||||
p = end;
|
||||
}
|
||||
}
|
||||
fn old_webp(webp: &mut Vec<u8>, exif: &[u8]) {
|
||||
let mut p = 12usize;
|
||||
while let Some(h) = webp.get(p..p.saturating_add(8)) {
|
||||
let size = u32::from_le_bytes([h[4], h[5], h[6], h[7]]) as usize;
|
||||
let end = p.saturating_add(8).saturating_add(size).saturating_add(size & 1);
|
||||
if &h[..4] == b"EXIF" && end <= webp.len() {
|
||||
let mut chunk = b"EXIF".to_vec();
|
||||
chunk.extend_from_slice(&(exif.len() as u32).to_le_bytes());
|
||||
chunk.extend_from_slice(exif);
|
||||
if exif.len() & 1 == 1 {
|
||||
chunk.push(0);
|
||||
}
|
||||
webp.splice(p..end, chunk);
|
||||
let size = (webp.len().saturating_sub(8)) as u32;
|
||||
webp[4..8].copy_from_slice(&size.to_le_bytes());
|
||||
return;
|
||||
}
|
||||
p = end;
|
||||
}
|
||||
}
|
||||
let mut image = crate::Image::from_u8(2, 2, crate::ChannelLayout::Rgb, vec![80; 12]).unwrap();
|
||||
image.meta.exif = Some(EXIF.to_vec());
|
||||
for format in [Format::Jpeg, Format::Png, Format::WebP] {
|
||||
let encoded = crate::encode(&image, format, &crate::EncodeOptions::default()).unwrap();
|
||||
for at in 0..encoded.len() {
|
||||
for value in [0, 255] {
|
||||
let mut original = encoded.clone();
|
||||
original[at] = value;
|
||||
let mut shared = original.clone();
|
||||
if format == Format::Jpeg {
|
||||
// the old byte search; the segment walk writes where it did or, when the
|
||||
// mutation broke the segment structure or the APP1 length, not at all
|
||||
let untouched = original.clone();
|
||||
if let Some(pos) = original.windows(6).position(|w| w == b"Exif\0\0")
|
||||
&& let Some(dst) = original.get_mut(pos + 6..pos + 6 + EXIF.len())
|
||||
{
|
||||
dst.copy_from_slice(EXIF);
|
||||
}
|
||||
replace_jpeg_exif(&mut shared, EXIF);
|
||||
assert!(shared == original || shared == untouched, "Jpeg: byte {at} value {value}");
|
||||
if at == 0 && value == 255 {
|
||||
// SOI kept: a real replacement
|
||||
assert_eq!(shared, original);
|
||||
}
|
||||
continue;
|
||||
} else if format == Format::Png {
|
||||
old_png(&mut original, EXIF);
|
||||
replace_png_exif(&mut shared, EXIF).unwrap();
|
||||
} else {
|
||||
old_webp(&mut original, EXIF);
|
||||
replace_webp_exif(&mut shared, EXIF).unwrap();
|
||||
}
|
||||
assert_eq!(shared, original, "{format:?}: byte {at} value {value}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_replacements_preserve_raw_orientation_in_real_encodings() {
|
||||
let mut image = crate::Image::from_u8(2, 2, crate::ChannelLayout::Rgb, vec![80; 12]).unwrap();
|
||||
image.meta.exif = Some(EXIF.to_vec());
|
||||
for format in [Format::Jpeg, Format::Png, Format::WebP] {
|
||||
let mut bytes = crate::encode(&image, format, &crate::EncodeOptions::default()).unwrap();
|
||||
let normal = crate::decode_as_with(format, &bytes, &crate::DecodeOptions { keep_orientation: true, ..Default::default() }).unwrap();
|
||||
assert_eq!(normal.meta.exif.as_deref().map(crate::exif_orientation), Some(1), "default {format:?}");
|
||||
match format {
|
||||
Format::Jpeg => replace_jpeg_exif(&mut bytes, EXIF),
|
||||
Format::Png => replace_png_exif(&mut bytes, EXIF).unwrap(),
|
||||
Format::WebP => replace_webp_exif(&mut bytes, EXIF).unwrap(),
|
||||
_ => {}
|
||||
}
|
||||
let decoded = crate::decode_as_with(format, &bytes, &crate::DecodeOptions { keep_orientation: true, ..Default::default() }).unwrap();
|
||||
assert_eq!(decoded.meta.exif.as_deref().map(crate::exif_orientation), Some(6), "{format:?}");
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn jpeg_exif_is_found_by_segment_not_by_signature() {
|
||||
fn segment(marker: u8, payload: &[u8]) -> Vec<u8> {
|
||||
[&[0xff, marker][..], &((payload.len() + 2) as u16).to_be_bytes(), payload].concat()
|
||||
}
|
||||
let tail = [0xff, 0xda, 0, 2, 0xff, 0xd9];
|
||||
let exif_app1 = segment(0xe1, &[&b"Exif\0\0"[..], &[0; 26]].concat());
|
||||
// a comment carrying the signature comes first: only the APP1 is rewritten
|
||||
let comment = segment(0xfe, &[&b"Exif\0\0"[..], &[7; 40]].concat());
|
||||
let mut jpeg = [&[0xff, 0xd8, 0xff][..], &comment, &exif_app1, &tail].concat();
|
||||
let before = jpeg.clone();
|
||||
replace_jpeg_exif(&mut jpeg, &[9; 26]);
|
||||
let app1_data = 3 + comment.len() + 10;
|
||||
assert_eq!(&jpeg[..app1_data], &before[..app1_data]);
|
||||
assert_eq!(&jpeg[app1_data..app1_data + 26], &[9; 26]);
|
||||
assert_eq!(&jpeg[app1_data + 26..], &before[app1_data + 26..]);
|
||||
// no EXIF APP1, only the comment: nothing changes
|
||||
let mut only_comment = [&[0xff, 0xd8][..], &comment, &tail].concat();
|
||||
let before = only_comment.clone();
|
||||
replace_jpeg_exif(&mut only_comment, &[9; 26]);
|
||||
assert_eq!(only_comment, before);
|
||||
// a replacement larger than the APP1 is refused even though the file continues
|
||||
let mut small = [&[0xff, 0xd8][..], &exif_app1, &segment(0xe2, &[1; 64]), &tail].concat();
|
||||
let before = small.clone();
|
||||
replace_jpeg_exif(&mut small, &[9; 28]);
|
||||
assert_eq!(small, before);
|
||||
// markers without a length, damaged lengths, a scan before any APP1, and a non-JPEG
|
||||
let mut tem = [&[0xff, 0xd8, 0xff, 0x01][..], &exif_app1, &tail].concat();
|
||||
replace_jpeg_exif(&mut tem, &[9; 26]);
|
||||
assert_eq!(&tem[14..40], &[9; 26]);
|
||||
for broken in [vec![0xff, 0xd8, 0xff, 0xe1, 0, 1], vec![0xff, 0xd8, 0xff, 0xe1, 0xff, 0xff, 0], tail.to_vec(), vec![0xff, 0xd8, 0xff, 0xda]] {
|
||||
let mut b = broken.clone();
|
||||
replace_jpeg_exif(&mut b, &[9; 2]);
|
||||
assert_eq!(b, broken);
|
||||
}
|
||||
let mut no_marker = [&[0xff, 0xd8, 0x12][..], &exif_app1].concat();
|
||||
let before = no_marker.clone();
|
||||
replace_jpeg_exif(&mut no_marker, &[9; 26]);
|
||||
assert_eq!(no_marker, before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jpeg_segments_keep_order_and_enforce_limits() {
|
||||
let parts = jpeg_segments(JpegMetadata { exif: Some(EXIF), xmp: Some("<x/>"), icc: Some(&[1, 2]), ppi: Some(300) }).unwrap();
|
||||
assert_eq!(parts.iter().map(|v| v.0).collect::<Vec<_>>(), [0xe0, 0xe1, 0xe1, 0xe2]);
|
||||
assert!(jpeg_segments(JpegMetadata { exif: Some(&vec![0; 65_528]), ..Default::default() }).is_err());
|
||||
assert!(jpeg_xmp(&"x".repeat(65_533)).is_err());
|
||||
let profile = vec![42; 65_520];
|
||||
let chunks = jpeg_segments(JpegMetadata { icc: Some(&profile), ..Default::default() }).unwrap();
|
||||
assert_eq!(chunks.len(), 3);
|
||||
assert_eq!(&chunks[1].1[12..14], &[1, 2]);
|
||||
assert_eq!(&chunks[2].1[12..14], &[2, 2]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
//! The resolution a raster file records, read from its headers without decoding any pixels.
|
||||
//!
|
||||
//! A layout app placing a picture wants its physical size before (or without) decoding it: a
|
||||
//! 600-pixel photo saved at 600 ppi places one inch wide. [`file_resolution`] answers from the
|
||||
//! container alone:
|
||||
//!
|
||||
//! - JPEG: the resolution [`crate::decode`] reports, from the same marker scan (stopping at the
|
||||
//! start of scan): Photoshop ResolutionInfo, then XMP, then EXIF, then the JFIF density.
|
||||
//! - PNG: the `pHYs` chunk when its unit is the metre (read as the decoder reads it, so a stored
|
||||
//! 11811 is 300 ppi), else the `eXIf` resolution; the chunk scan stops at `IDAT` or `IEND`.
|
||||
//! - WebP: the IFD0 resolution of the `EXIF` chunk.
|
||||
//! - BMP: the pixels-per-metre fields of a `BITMAPINFOHEADER` or later header.
|
||||
//! - Classic TIFF: the IFD0 resolution tags.
|
||||
//!
|
||||
//! EXIF and TIFF resolution tags are read by [`crate::resolution::exif_resolution`]. Every scan
|
||||
//! is bounded by the input length and a chunk count, and anything malformed, absent,
|
||||
//! non-positive or past [`MAX_PPI`] gives `None`, so the caller keeps its own default.
|
||||
|
||||
use crate::Limits;
|
||||
use crate::codecs::{jpeg, png, webp};
|
||||
use crate::resolution::exif_resolution;
|
||||
|
||||
/// Resolutions above this many pixels per inch are treated as corrupt.
|
||||
pub const MAX_PPI: f32 = 100_000.0;
|
||||
|
||||
/// PNG chunks looked at before giving up.
|
||||
const MAX_CHUNKS: usize = 4096;
|
||||
|
||||
const PNG_SIGNATURE: &[u8] = b"\x89PNG\r\n\x1a\n";
|
||||
|
||||
/// Pixels per inch (horizontal, vertical) that `bytes` records in its headers, or `None` when
|
||||
/// the format is not one listed in the [module documentation](self) or carries no usable value.
|
||||
pub fn file_resolution(bytes: &[u8]) -> Option<(f32, f32)> {
|
||||
let ppi = if bytes.starts_with(b"\xFF\xD8\xFF") {
|
||||
jpeg::scan_metadata(bytes).dpi
|
||||
} else if bytes.starts_with(PNG_SIGNATURE) {
|
||||
png_phys(bytes).or_else(|| png_exif(bytes))
|
||||
} else if bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WEBP".as_slice()) {
|
||||
webp_exif(bytes)
|
||||
} else if bytes.starts_with(b"BM") {
|
||||
bmp(bytes)
|
||||
} else if is_classic_tiff(bytes) {
|
||||
exif_resolution(bytes)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
ppi.filter(|&p| usable(p))
|
||||
}
|
||||
|
||||
/// Whether both axes are finite and between 1 and [`MAX_PPI`].
|
||||
pub fn usable((x, y): (f32, f32)) -> bool {
|
||||
let ok = |v: f32| v.is_finite() && (1.0..=MAX_PPI).contains(&v);
|
||||
ok(x) && ok(y)
|
||||
}
|
||||
|
||||
fn is_classic_tiff(bytes: &[u8]) -> bool {
|
||||
bytes.starts_with(b"II*\0") || bytes.starts_with(b"MM\0*")
|
||||
}
|
||||
|
||||
fn be_u32(b: &[u8], at: usize) -> Option<u32> {
|
||||
Some(u32::from_be_bytes(b.get(at..at.checked_add(4)?)?.try_into().ok()?))
|
||||
}
|
||||
|
||||
fn le_u32(b: &[u8], at: usize) -> Option<u32> {
|
||||
Some(u32::from_le_bytes(b.get(at..at.checked_add(4)?)?.try_into().ok()?))
|
||||
}
|
||||
|
||||
fn le_i32(b: &[u8], at: usize) -> Option<i32> {
|
||||
Some(i32::from_le_bytes(b.get(at..at.checked_add(4)?)?.try_into().ok()?))
|
||||
}
|
||||
|
||||
fn png_phys(bytes: &[u8]) -> Option<(f32, f32)> {
|
||||
let mut pos = PNG_SIGNATURE.len();
|
||||
for _ in 0..MAX_CHUNKS {
|
||||
let len = usize::try_from(be_u32(bytes, pos)?).ok()?;
|
||||
let data = pos.checked_add(8)?;
|
||||
match bytes.get(pos.checked_add(4)?..data)? {
|
||||
// A `pHYs` chunk holds exactly nine bytes; the whole chunk, CRC included, must be there.
|
||||
b"pHYs" if len == 9 => return bytes.get(data..data.checked_add(13)?).and_then(|chunk| phys(chunk.get(..9)?)),
|
||||
b"pHYs" => return None,
|
||||
b"IDAT" | b"IEND" => return None,
|
||||
_ => pos = data.checked_add(len)?.checked_add(4)?,
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// `pHYs`: pixels per unit in x and y, then the unit (1 = metre; 0 = aspect ratio only).
|
||||
fn phys(data: &[u8]) -> Option<(f32, f32)> {
|
||||
let (x, y) = (be_u32(data, 0)?, be_u32(data, 4)?);
|
||||
// As the decoder does: a zero count, or a unit other than the metre, records no resolution,
|
||||
// so the `eXIf` resolution can still answer.
|
||||
if *data.get(8)? != 1 || x == 0 || y == 0 {
|
||||
return None;
|
||||
}
|
||||
Some((png::dpi_of_ppm(x), png::dpi_of_ppm(y)))
|
||||
}
|
||||
|
||||
/// The resolution of a PNG's `eXIf` chunk, read by the decoder's own header probe.
|
||||
fn png_exif(bytes: &[u8]) -> Option<(f32, f32)> {
|
||||
let (_, _, exif) = png::probe(bytes, &Limits::default()).ok()?;
|
||||
exif_resolution(&exif?)
|
||||
}
|
||||
|
||||
/// The resolution of a WebP's `EXIF` chunk, read by the decoder's own container probe.
|
||||
fn webp_exif(bytes: &[u8]) -> Option<(f32, f32)> {
|
||||
let (_, _, exif) = webp::probe(bytes, &Limits::default()).ok()?;
|
||||
exif_resolution(&exif?)
|
||||
}
|
||||
|
||||
/// BMP: the info header size at 14 (40 or more for a header with resolution fields), then the
|
||||
/// signed pixels-per-metre values at 38 and 42.
|
||||
fn bmp(bytes: &[u8]) -> Option<(f32, f32)> {
|
||||
if le_u32(bytes, 14)? < 40 {
|
||||
return None;
|
||||
}
|
||||
let (x, y) = (le_i32(bytes, 38)?, le_i32(bytes, 42)?);
|
||||
if x <= 0 || y <= 0 {
|
||||
return None;
|
||||
}
|
||||
Some((png::dpi_of_ppm(x.unsigned_abs()), png::dpi_of_ppm(y.unsigned_abs())))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "resolution_probe_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,266 @@
|
||||
#![allow(clippy::unwrap_used)]
|
||||
|
||||
use super::{MAX_PPI, file_resolution, usable};
|
||||
use crate::{ChannelLayout, EncodeOptions, Format, Image, SampleType};
|
||||
|
||||
/// A little- or big-endian classic TIFF whose IFD0 holds XResolution and YResolution
|
||||
/// (`x`/1, `y`/1) and, when given, ResolutionUnit.
|
||||
fn tiff(le: bool, x: u32, y: u32, unit: Option<u16>) -> Vec<u8> {
|
||||
let u16b = |v: u16| if le { v.to_le_bytes() } else { v.to_be_bytes() };
|
||||
let u32b = |v: u32| if le { v.to_le_bytes() } else { v.to_be_bytes() };
|
||||
let entries: u16 = if unit.is_some() { 3 } else { 2 };
|
||||
let rationals = 8 + 2 + 12 * u32::from(entries) + 4;
|
||||
let mut t = Vec::new();
|
||||
t.extend_from_slice(if le { b"II*\0" } else { b"MM\0*" });
|
||||
t.extend_from_slice(&u32b(8));
|
||||
t.extend_from_slice(&u16b(entries));
|
||||
for (tag, offset) in [(0x011Au16, rationals), (0x011B, rationals + 8)] {
|
||||
t.extend_from_slice(&u16b(tag));
|
||||
t.extend_from_slice(&u16b(5));
|
||||
t.extend_from_slice(&u32b(1));
|
||||
t.extend_from_slice(&u32b(offset));
|
||||
}
|
||||
if let Some(unit) = unit {
|
||||
t.extend_from_slice(&u16b(0x0128));
|
||||
t.extend_from_slice(&u16b(3));
|
||||
t.extend_from_slice(&u32b(1));
|
||||
t.extend_from_slice(&u16b(unit));
|
||||
t.extend_from_slice(&[0, 0]);
|
||||
}
|
||||
t.extend_from_slice(&u32b(0));
|
||||
for v in [x, 1, y, 1] {
|
||||
t.extend_from_slice(&u32b(v));
|
||||
}
|
||||
t
|
||||
}
|
||||
|
||||
fn segment(marker: u8, payload: &[u8]) -> Vec<u8> {
|
||||
let mut s = vec![0xFF, marker];
|
||||
s.extend_from_slice(&u16::try_from(payload.len() + 2).unwrap().to_be_bytes());
|
||||
s.extend_from_slice(payload);
|
||||
s
|
||||
}
|
||||
|
||||
fn jfif(units: u8, x: u16, y: u16) -> Vec<u8> {
|
||||
let mut p = b"JFIF\0\x01\x02".to_vec();
|
||||
p.push(units);
|
||||
p.extend_from_slice(&x.to_be_bytes());
|
||||
p.extend_from_slice(&y.to_be_bytes());
|
||||
p.extend_from_slice(&[0, 0]);
|
||||
segment(0xE0, &p)
|
||||
}
|
||||
|
||||
fn exif_app1(tiff: &[u8]) -> Vec<u8> {
|
||||
let mut p = b"Exif\0\0".to_vec();
|
||||
p.extend_from_slice(tiff);
|
||||
segment(0xE1, &p)
|
||||
}
|
||||
|
||||
fn jpeg(segments: &[Vec<u8>]) -> Vec<u8> {
|
||||
let mut j = vec![0xFF, 0xD8];
|
||||
for s in segments {
|
||||
j.extend_from_slice(s);
|
||||
}
|
||||
// Start of scan, then entropy data holding bytes that look like an APP0 marker.
|
||||
j.extend_from_slice(&segment(0xDA, &[1, 1, 0, 0, 63, 0]));
|
||||
j.extend_from_slice(&jfif(1, 9, 9));
|
||||
j.extend_from_slice(&[0xFF, 0xD9]);
|
||||
j
|
||||
}
|
||||
|
||||
fn png_chunk(kind: &[u8; 4], data: &[u8]) -> Vec<u8> {
|
||||
let mut c = u32::try_from(data.len()).unwrap().to_be_bytes().to_vec();
|
||||
c.extend_from_slice(kind);
|
||||
c.extend_from_slice(data);
|
||||
c.extend_from_slice(&[0, 0, 0, 0]); // the CRC is not checked by a header probe
|
||||
c
|
||||
}
|
||||
|
||||
fn png(chunks: &[Vec<u8>]) -> Vec<u8> {
|
||||
let mut p = b"\x89PNG\r\n\x1a\n".to_vec();
|
||||
p.extend_from_slice(&png_chunk(b"IHDR", &[0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0, 0, 0]));
|
||||
for c in chunks {
|
||||
p.extend_from_slice(c);
|
||||
}
|
||||
p.extend_from_slice(&png_chunk(b"IEND", &[]));
|
||||
p
|
||||
}
|
||||
|
||||
fn phys(x: u32, y: u32, unit: u8) -> Vec<u8> {
|
||||
let mut d = x.to_be_bytes().to_vec();
|
||||
d.extend_from_slice(&y.to_be_bytes());
|
||||
d.push(unit);
|
||||
png_chunk(b"pHYs", &d)
|
||||
}
|
||||
|
||||
fn bmp(header_size: u32, x: i32, y: i32) -> Vec<u8> {
|
||||
let mut b = vec![0u8; 54];
|
||||
b[..2].copy_from_slice(b"BM");
|
||||
b[14..18].copy_from_slice(&header_size.to_le_bytes());
|
||||
b[38..42].copy_from_slice(&x.to_le_bytes());
|
||||
b[42..46].copy_from_slice(&y.to_le_bytes());
|
||||
b
|
||||
}
|
||||
|
||||
fn close(got: Option<(f32, f32)>, want: (f32, f32)) {
|
||||
let (x, y) = got.unwrap();
|
||||
assert!((x - want.0).abs() < 0.05 && (y - want.1).abs() < 0.05, "{x} {y} vs {want:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jpeg_jfif_density_in_inches_and_centimetres() {
|
||||
assert_eq!(file_resolution(&jpeg(&[jfif(1, 300, 150)])), Some((300.0, 150.0)));
|
||||
close(file_resolution(&jpeg(&[jfif(2, 100, 100)])), (254.0, 254.0));
|
||||
// Units 0 is an aspect ratio, not a density.
|
||||
assert_eq!(file_resolution(&jpeg(&[jfif(0, 1, 1)])), None);
|
||||
// Nothing in the header: the APP0 look-alike after the start of scan is not read.
|
||||
assert_eq!(file_resolution(&jpeg(&[])), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jpeg_exif_wins_over_jfif_and_non_exif_app1_is_skipped() {
|
||||
let xmp = segment(0xE1, b"http://ns.adobe.com/xap/1.0/\0<x:xmpmeta/>");
|
||||
let exif = exif_app1(&tiff(false, 600, 600, Some(2)));
|
||||
assert_eq!(file_resolution(&jpeg(&[jfif(1, 72, 72), xmp.clone(), exif])), Some((600.0, 600.0)));
|
||||
// An unusable EXIF value falls back to JFIF.
|
||||
let zero = exif_app1(&tiff(true, 0, 0, None));
|
||||
assert_eq!(file_resolution(&jpeg(&[jfif(1, 96, 96), xmp, zero])), Some((96.0, 96.0)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jpeg_fill_bytes_and_standalone_markers_are_stepped_over() {
|
||||
let mut j = vec![0xFF, 0xD8, 0xFF, 0xFF, 0xFF, 0x01, 0xFF, 0xD0];
|
||||
j.extend_from_slice(&jfif(1, 200, 200));
|
||||
assert_eq!(file_resolution(&j), Some((200.0, 200.0)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn png_phys_in_metres_only_before_image_data() {
|
||||
assert_eq!(file_resolution(&png(&[phys(11_811, 11_811, 1)])), Some((300.0, 300.0)));
|
||||
assert_eq!(file_resolution(&png(&[phys(2, 1, 0)])), None);
|
||||
assert_eq!(file_resolution(&png(&[png_chunk(b"IDAT", &[0; 4]), phys(11_811, 11_811, 1)])), None);
|
||||
assert_eq!(file_resolution(&png(&[png_chunk(b"tEXt", b"a\0b"), phys(3_780, 3_780, 1)])).map(|p| p.0.round()), Some(96.0));
|
||||
assert_eq!(file_resolution(&png(&[])), None);
|
||||
}
|
||||
|
||||
/// What the probe reads is what a full decode reports, for each format the encoder writes with
|
||||
/// a resolution.
|
||||
#[test]
|
||||
fn agrees_with_decode_for_encoded_files() {
|
||||
let mut img = Image::from_raw(2, 1, ChannelLayout::Rgb, SampleType::U8, vec![10, 20, 30, 40, 50, 60]).unwrap();
|
||||
img.meta.dpi = Some((300.0, 150.0));
|
||||
for format in [Format::Jpeg, Format::Png, Format::WebP, Format::Tiff] {
|
||||
let bytes = crate::encode(&img, format, &EncodeOptions::default()).unwrap();
|
||||
let decoded = crate::decode_as(format, &bytes).unwrap().meta.dpi;
|
||||
assert_eq!(file_resolution(&bytes), decoded, "{format:?}");
|
||||
assert_eq!(decoded, Some((300.0, 150.0)), "{format:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn webp_without_exif_has_none() {
|
||||
let img = Image::from_raw(1, 1, ChannelLayout::Rgb, SampleType::U8, vec![1, 2, 3]).unwrap();
|
||||
let bytes = crate::encode(&img, Format::WebP, &EncodeOptions::default()).unwrap();
|
||||
assert_eq!(file_resolution(&bytes), None);
|
||||
// A RIFF/WEBP container the decoder's probe refuses gives nothing either.
|
||||
assert_eq!(file_resolution(b"RIFF\x04\0\0\0WEBP"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bmp_pixels_per_metre() {
|
||||
close(file_resolution(&bmp(40, 3_780, 7_559)), (96.0, 192.0));
|
||||
assert_eq!(file_resolution(&bmp(12, 3_780, 3_780)), None);
|
||||
assert_eq!(file_resolution(&bmp(40, -3_780, 3_780)), None);
|
||||
assert_eq!(file_resolution(&bmp(40, 0, 0)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiff_tags_in_both_byte_orders_and_units() {
|
||||
assert_eq!(file_resolution(&tiff(true, 300, 300, None)), Some((300.0, 300.0)));
|
||||
close(file_resolution(&tiff(false, 100, 50, Some(3))), (254.0, 127.0));
|
||||
assert_eq!(file_resolution(&tiff(true, 300, 300, Some(1))), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_formats_and_absurd_values_give_none() {
|
||||
assert_eq!(file_resolution(b"GIF89a\x01\0\x01\0"), None);
|
||||
assert_eq!(file_resolution(b""), None);
|
||||
assert_eq!(file_resolution(&tiff(true, 1_000_000, 1_000_000, None)), None);
|
||||
assert!(usable((1.0, MAX_PPI)));
|
||||
assert!(!usable((0.5, 72.0)));
|
||||
assert!(!usable((f32::NAN, 72.0)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_and_corrupted_headers_never_panic() {
|
||||
let samples = [
|
||||
jpeg(&[jfif(1, 300, 300), exif_app1(&tiff(false, 600, 600, Some(2)))]),
|
||||
png(&[phys(11_811, 11_811, 1)]),
|
||||
bmp(40, 3_780, 3_780),
|
||||
tiff(false, 300, 300, Some(2)),
|
||||
];
|
||||
for s in &samples {
|
||||
for cut in 0..=s.len() {
|
||||
let _ = file_resolution(&s[..cut]);
|
||||
}
|
||||
for at in 0..s.len() {
|
||||
for v in [0u8, 0x7F, 0xFF] {
|
||||
let mut c = s.clone();
|
||||
c[at] = v;
|
||||
let _ = file_resolution(&c);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn png_phys_must_be_a_whole_nine_byte_chunk() {
|
||||
// A pHYs that declares no data doesn't borrow the bytes after it.
|
||||
let mut empty = png_chunk(b"pHYs", &[]);
|
||||
empty.extend_from_slice(&phys(11_811, 11_811, 1)[8..]);
|
||||
assert_eq!(file_resolution(&png(&[empty])), None);
|
||||
// Cut off before its CRC.
|
||||
let whole = png(&[phys(11_811, 11_811, 1)]);
|
||||
let crc_at = whole.len() - 12 - 4;
|
||||
assert_eq!(file_resolution(&whole[..crc_at]), None);
|
||||
assert_eq!(file_resolution(&whole[..crc_at + 4]), Some((300.0, 300.0)));
|
||||
}
|
||||
|
||||
/// A PNG chunk with its real CRC, for the decoder's own probe, which checks it.
|
||||
fn crc_chunk(kind: &[u8; 4], data: &[u8]) -> Vec<u8> {
|
||||
let mut c = png_chunk(kind, data);
|
||||
let body = c.get(4..c.len() - 4).unwrap().to_vec();
|
||||
let mut crc = !0u32;
|
||||
for b in body {
|
||||
crc ^= u32::from(b);
|
||||
for _ in 0..8 {
|
||||
crc = (crc >> 1) ^ (0xEDB8_8320 & (crc & 1).wrapping_neg());
|
||||
}
|
||||
}
|
||||
let n = c.len();
|
||||
c[n - 4..].copy_from_slice(&(!crc).to_be_bytes());
|
||||
c
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn png_zero_phys_leaves_the_exif_resolution() {
|
||||
let exif = crc_chunk(b"eXIf", &tiff(false, 600, 600, Some(2)));
|
||||
let zero = |x: u32, y: u32| {
|
||||
let mut d = x.to_be_bytes().to_vec();
|
||||
d.extend_from_slice(&y.to_be_bytes());
|
||||
d.push(1);
|
||||
crc_chunk(b"pHYs", &d)
|
||||
};
|
||||
let png = |chunks: &[Vec<u8>]| {
|
||||
let mut p = b"\x89PNG\r\n\x1a\n".to_vec();
|
||||
p.extend_from_slice(&crc_chunk(b"IHDR", &[0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0, 0, 0]));
|
||||
chunks.iter().for_each(|c| p.extend_from_slice(c));
|
||||
// The decoder's probe reads up to the image data, so there has to be some.
|
||||
p.extend_from_slice(&crc_chunk(b"IDAT", &[0x78, 0x9c, 0x63, 0, 0, 0, 1, 0, 1]));
|
||||
p.extend_from_slice(&crc_chunk(b"IEND", &[]));
|
||||
p
|
||||
};
|
||||
assert_eq!(file_resolution(&png(std::slice::from_ref(&exif))), Some((600.0, 600.0)));
|
||||
assert_eq!(file_resolution(&png(&[zero(0, 0), exif.clone()])), Some((600.0, 600.0)));
|
||||
assert_eq!(file_resolution(&png(&[zero(11_811, 0), exif])), Some((600.0, 600.0)));
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
//! Header-only WebP inspection with image-chunk completeness checks.
|
||||
|
||||
use crate::{CodecError, Format};
|
||||
|
||||
/// Stored width, height and EXIF orientation without decoding pixels. The image payload must
|
||||
/// fit in the input, but its compressed contents are not validated. Uses the default decode limits.
|
||||
pub fn header(bytes: &[u8]) -> Result<(u32, u32, u16), CodecError> {
|
||||
let err = |s: &str| CodecError::malformed(Format::WebP, s);
|
||||
let mut p = 12usize;
|
||||
let mut has_image = false;
|
||||
let mut exif = None;
|
||||
while let Some(h) = bytes.get(p..p.saturating_add(8)) {
|
||||
let size = u32::from_le_bytes([h[4], h[5], h[6], h[7]]) as usize;
|
||||
let end = p.saturating_add(8).saturating_add(size);
|
||||
if matches!(&h[..4], b"VP8 " | b"VP8L" | b"ANMF") {
|
||||
if end > bytes.len() {
|
||||
return Err(err("truncated image data"));
|
||||
}
|
||||
has_image = true;
|
||||
} else if &h[..4] == b"EXIF" && end <= bytes.len() && exif.is_none() {
|
||||
exif = bytes.get(p + 8..end);
|
||||
}
|
||||
p = end.saturating_add(size & 1);
|
||||
}
|
||||
if !has_image {
|
||||
return Err(err("no image data"));
|
||||
}
|
||||
let (w, h) = crate::probe_size(bytes)?;
|
||||
let orientation = exif.map_or(1, craft_metadata::compat_orientation::orientation);
|
||||
let (w, h) = if (5..=8).contains(&orientation) { (h, w) } else { (w, h) };
|
||||
Ok((w, h, orientation))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn preserves_stored_dimensions_with_exif_orientation() {
|
||||
let mut image = crate::Image::from_u8(2, 3, crate::ChannelLayout::Rgb, vec![42; 18]).unwrap();
|
||||
image.meta.exif = Some(b"II\x2a\x00\x08\x00\x00\x00\x01\x00\x12\x01\x03\x00\x01\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00".to_vec());
|
||||
let mut bytes = crate::encode(&image, Format::WebP, &crate::EncodeOptions::default()).unwrap();
|
||||
// The writer correctly makes exported pixels upright; simulate a camera file carrying orientation 6.
|
||||
let exif = bytes.windows(4).position(|v| v == b"EXIF").unwrap();
|
||||
let tiff = exif + 8;
|
||||
bytes[tiff + 18] = 6;
|
||||
assert_eq!(header(&bytes).unwrap(), (2, 3, 6));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_missing_and_truncated_image_chunks() {
|
||||
let mut bytes = b"RIFF\x00\x00\x00\x00WEBP".to_vec();
|
||||
assert!(header(&bytes).is_err());
|
||||
bytes.extend_from_slice(b"VP8 ");
|
||||
bytes.extend_from_slice(&10u32.to_le_bytes());
|
||||
bytes.push(0);
|
||||
assert!(header(&bytes).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
//! A BMP keeps its resolution in the header's `biXPelsPerMeter` and `biYPelsPerMeter`, which
|
||||
//! Photoshop writes and reads (storytold/photocraft#13630). BMP exports left both 0, so a 300 ppi
|
||||
//! document saved as BMP opened at 72 ppi elsewhere, and an opened BMP's resolution was ignored.
|
||||
|
||||
use crate::common::test_image;
|
||||
use craft_image_codecs::*;
|
||||
|
||||
/// The DIB header's size and its pixels per metre, horizontal then vertical (file offsets 14, 38
|
||||
/// and 42, little-endian).
|
||||
fn header(bytes: &[u8]) -> (u32, i32, i32) {
|
||||
let at = |i: usize| [bytes[i], bytes[i + 1], bytes[i + 2], bytes[i + 3]];
|
||||
(u32::from_le_bytes(at(14)), i32::from_le_bytes(at(38)), i32::from_le_bytes(at(42)))
|
||||
}
|
||||
|
||||
/// A test image of `layout` at resolution `dpi`.
|
||||
fn image(layout: ChannelLayout, dpi: Option<(f32, f32)>) -> Image {
|
||||
let mut img = test_image(layout, SampleType::U8);
|
||||
img.meta.dpi = dpi;
|
||||
img
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bmp_export_writes_the_resolution_in_pixels_per_metre_and_reads_it_back() {
|
||||
// RGB is written with a BITMAPINFOHEADER, RGBA with a BITMAPV4HEADER: same field offsets.
|
||||
for (layout, dib) in [(ChannelLayout::Rgb, 40), (ChannelLayout::Rgba, 108)] {
|
||||
// The fields are part of the header, not metadata: embed_metadata off keeps them.
|
||||
for embed_metadata in [true, false] {
|
||||
let opts = EncodeOptions { embed_metadata, ..Default::default() };
|
||||
let img = image(layout, Some((300.0, 150.0)));
|
||||
let bytes = encode(&img, Format::Bmp, &opts).unwrap();
|
||||
// 300 / 0.0254 = 11811.02 and 150 / 0.0254 = 5905.51, rounded.
|
||||
assert_eq!(header(&bytes), (dib, 11811, 5906), "{layout:?} metadata {embed_metadata}");
|
||||
assert_eq!(decode(&bytes).unwrap().meta.dpi, Some((300.0, 150.0)), "{layout:?} metadata {embed_metadata}");
|
||||
assert!(!fidelity_warnings_with(&img, Format::Bmp, &opts).contains(&FidelityWarning::DpiDropped), "{layout:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bmp_without_a_resolution_leaves_the_fields_unset() {
|
||||
let bytes = encode(&image(ChannelLayout::Rgb, None), Format::Bmp, &EncodeOptions::default()).unwrap();
|
||||
assert_eq!(header(&bytes), (40, 0, 0));
|
||||
assert_eq!(decode(&bytes).unwrap().meta.dpi, None, "0 means no resolution: the caller's default applies");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bmp_opens_at_the_resolution_its_header_records() {
|
||||
let bytes = encode(&image(ChannelLayout::Rgb, None), Format::Bmp, &EncodeOptions::default()).unwrap();
|
||||
let with = |x: i32, y: i32| {
|
||||
let mut b = bytes.clone();
|
||||
b[38..42].copy_from_slice(&x.to_le_bytes());
|
||||
b[42..46].copy_from_slice(&y.to_le_bytes());
|
||||
decode(&b).unwrap().meta.dpi
|
||||
};
|
||||
// What other writers store for 96 and 72 ppi reads back whole.
|
||||
assert_eq!(with(3780, 3780), Some((96.0, 96.0)));
|
||||
assert_eq!(with(2835, 2835), Some((72.0, 72.0)));
|
||||
// 40000 px/m is exactly 1016 ppi; a value no whole ppi rounds to stays as it is (71.98).
|
||||
let (x, y) = with(40000, 2834).unwrap();
|
||||
assert_eq!(x, 1016.0);
|
||||
assert!((y - 71.9836).abs() < 1e-3, "{y}");
|
||||
// An axis that is 0 or negative leaves the resolution unknown.
|
||||
assert_eq!(with(11811, 0), None);
|
||||
assert_eq!(with(-11811, 11811), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn indexed_bmp_writes_the_resolution_it_is_given() {
|
||||
let (idx, pal) = ([0u8, 1, 1, 0], [[0, 0, 0], [255, 0, 0]]);
|
||||
let bytes = encode_bmp_indexed_with_dpi(2, 2, &idx, &pal, Some((300.0, 300.0))).unwrap();
|
||||
assert_eq!(header(&bytes), (40, 11811, 11811));
|
||||
assert_eq!(decode(&bytes).unwrap().meta.dpi, Some((300.0, 300.0)));
|
||||
let unset = encode_bmp_indexed_with_dpi(2, 2, &idx, &pal, None).unwrap();
|
||||
assert_eq!(header(&unset), (40, 0, 0));
|
||||
// The existing function keeps writing no resolution, so its callers' files don't change.
|
||||
assert_eq!(encode_bmp_indexed(2, 2, &idx, &pal).unwrap(), unset);
|
||||
}
|
||||
@@ -799,6 +799,106 @@ fn inconsistent_count_table_errors() {
|
||||
assert!(matches!(decode_deep_exr(&bytes, &Limits::none()), Err(CodecError::Malformed { .. })));
|
||||
}
|
||||
|
||||
/// A two-part file made from the single-part deep file `deep` and a 1×1 flat scan-line part,
|
||||
/// the deep part first when `deep_first`: each header gains `name` and `chunkCount`, the
|
||||
/// version field the multipart bit, and each chunk its part number. The offset tables follow
|
||||
/// the headers in header order, then the chunks. Returns the file, where the first table
|
||||
/// starts, where the second table starts and where the last table ends.
|
||||
fn two_part(deep: &[u8], deep_first: bool) -> (Vec<u8>, usize, usize, usize) {
|
||||
let end = header_end_of(deep);
|
||||
let first = u64::from_le_bytes(deep[end..end + 8].try_into().expect("8")) as usize;
|
||||
let offsets: Vec<usize> =
|
||||
(0..(first - end) / 8).map(|i| u64::from_le_bytes(deep[end + 8 * i..end + 8 * i + 8].try_into().expect("8")) as usize).collect();
|
||||
let deep_chunks: Vec<&[u8]> = offsets.iter().enumerate().map(|(i, &o)| &deep[o..offsets.get(i + 1).copied().unwrap_or(deep.len())]).collect();
|
||||
|
||||
let mut deep_header = deep[8..end - 1].to_vec();
|
||||
attr(&mut deep_header, "name", "string", b"deep");
|
||||
attr(&mut deep_header, "chunkCount", "int", &(deep_chunks.len() as i32).to_le_bytes());
|
||||
let mut flat_header = Vec::new();
|
||||
let mut chlist = b"R\0".to_vec();
|
||||
chlist.extend_from_slice(&1i32.to_le_bytes()); // half
|
||||
chlist.extend_from_slice(&[0u8; 4]); // pLinear, reserved
|
||||
chlist.extend_from_slice(&1i32.to_le_bytes());
|
||||
chlist.extend_from_slice(&1i32.to_le_bytes());
|
||||
chlist.push(0);
|
||||
attr(&mut flat_header, "channels", "chlist", &chlist);
|
||||
attr(&mut flat_header, "compression", "compression", &[0]);
|
||||
let dw: Vec<u8> = [0i32; 4].iter().flat_map(|v| v.to_le_bytes()).collect();
|
||||
attr(&mut flat_header, "dataWindow", "box2i", &dw);
|
||||
attr(&mut flat_header, "displayWindow", "box2i", &dw);
|
||||
attr(&mut flat_header, "lineOrder", "lineOrder", &[0]);
|
||||
attr(&mut flat_header, "pixelAspectRatio", "float", &1f32.to_le_bytes());
|
||||
attr(&mut flat_header, "screenWindowCenter", "v2f", &[0f32.to_le_bytes(), 0f32.to_le_bytes()].concat());
|
||||
attr(&mut flat_header, "screenWindowWidth", "float", &1f32.to_le_bytes());
|
||||
attr(&mut flat_header, "name", "string", b"flat");
|
||||
attr(&mut flat_header, "type", "string", b"scanlineimage");
|
||||
attr(&mut flat_header, "chunkCount", "int", &1i32.to_le_bytes());
|
||||
// The flat part's one chunk: y, the data size, one half sample.
|
||||
let mut flat_chunk = 0i32.to_le_bytes().to_vec();
|
||||
flat_chunk.extend_from_slice(&2i32.to_le_bytes());
|
||||
flat_chunk.extend_from_slice(&f16::from_f32(0.5).to_le_bytes());
|
||||
let flat_chunks: Vec<&[u8]> = vec![&flat_chunk];
|
||||
|
||||
let parts: [(&[u8], &[&[u8]]); 2] = if deep_first {
|
||||
[(&deep_header, &deep_chunks), (&flat_header, &flat_chunks)]
|
||||
} else {
|
||||
[(&flat_header, &flat_chunks), (&deep_header, &deep_chunks)]
|
||||
};
|
||||
let mut out = deep[..4].to_vec();
|
||||
out.extend_from_slice(&(2u32 | 0x800 | 0x1000).to_le_bytes());
|
||||
for (header, _) in &parts {
|
||||
out.extend_from_slice(header);
|
||||
out.push(0); // end of this header
|
||||
}
|
||||
out.push(0); // end of the headers
|
||||
let first_table = out.len();
|
||||
let second_table = first_table + parts[0].1.len() * 8;
|
||||
let tables_end = second_table + parts[1].1.len() * 8;
|
||||
let mut pos = tables_end;
|
||||
for (_, chunks) in &parts {
|
||||
for c in chunks.iter() {
|
||||
out.extend_from_slice(&(pos as u64).to_le_bytes());
|
||||
pos += 4 + c.len();
|
||||
}
|
||||
}
|
||||
for (part, (_, chunks)) in parts.iter().enumerate() {
|
||||
for c in chunks.iter() {
|
||||
out.extend_from_slice(&(part as i32).to_le_bytes());
|
||||
out.extend_from_slice(c);
|
||||
}
|
||||
}
|
||||
(out, first_table, second_table, tables_end)
|
||||
}
|
||||
|
||||
/// The deep part of a two-part file decodes as the single-part file does, first or second.
|
||||
/// Its chunks start after every part's offset table: a chunk offset into the other part's
|
||||
/// table, or one byte before the end of the last table, is refused as table data, and a file
|
||||
/// that ends inside the last table is refused even when the deep part's own table is whole.
|
||||
#[test]
|
||||
fn multipart_deep_chunks_start_after_every_offset_table() {
|
||||
let single = gen_deep(4, 4, &RGBAZ, 0, None, COUNTS_ONE, &sample_rgba);
|
||||
let want = decode_deep_exr(&single, &Limits::none()).expect("single part");
|
||||
for deep_first in [true, false] {
|
||||
let (bytes, ..) = two_part(&single, deep_first);
|
||||
let deep = decode_deep_exr(&bytes, &Limits::none()).expect("two parts");
|
||||
assert_eq!(deep.counts, want.counts);
|
||||
assert_eq!(deep.channel("Z").expect("Z").samples, want.channel("Z").expect("Z").samples);
|
||||
}
|
||||
|
||||
let refused = |bytes: &[u8], what: &str| match decode_deep_exr(bytes, &Limits::none()) {
|
||||
Err(CodecError::Malformed { message, .. }) => assert!(message.contains(what), "{message}"),
|
||||
Err(e) => panic!("{e}, want an error about {what}"),
|
||||
Ok(_) => panic!("decoded, want an error about {what}"),
|
||||
};
|
||||
let (bytes, first_table, second_table, tables_end) = two_part(&single, true);
|
||||
for off in [second_table, tables_end - 1] {
|
||||
let mut into_table = bytes.clone();
|
||||
into_table[first_table..first_table + 8].copy_from_slice(&(off as u64).to_le_bytes());
|
||||
refused(&into_table, "outside the chunk area");
|
||||
}
|
||||
refused(&bytes[..tables_end - 4], "past the end of the file");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
// Real reference files: the deep test images of the OpenEXR repository at commit
|
||||
// `69b2604fc76e370615438bdc8d2cd95b9349c12e`, read from `corpus/exr/` at the repo root with the
|
||||
|
||||
@@ -62,6 +62,8 @@ fn default_build_formats_all_symmetric() {
|
||||
Format::Dpx,
|
||||
Format::Cineon,
|
||||
Format::Iff,
|
||||
#[cfg(feature = "sunraster")]
|
||||
Format::SunRaster,
|
||||
#[cfg(feature = "sgi")]
|
||||
Format::Sgi,
|
||||
Format::JpegLs,
|
||||
@@ -258,6 +260,20 @@ fn detect_magic_jpeg_ls() {
|
||||
assert!(decode_as(Format::JpegLs, &[0xFF, 0xD8, 0xFF, 0xF7]).is_err());
|
||||
}
|
||||
|
||||
#[cfg(feature = "sunraster")]
|
||||
#[test]
|
||||
fn detect_magic_sun_raster() {
|
||||
use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode};
|
||||
let img = Image::from_u8(5, 3, ChannelLayout::Rgb, (0..45).collect()).unwrap();
|
||||
let bytes = encode(&img, Format::SunRaster, &EncodeOptions::default()).unwrap();
|
||||
assert_eq!(detect(&bytes), Some(Format::SunRaster));
|
||||
assert_eq!(from_extension("scan.RAS"), Some(Format::SunRaster));
|
||||
assert_eq!(decode(&bytes).unwrap().data(), img.data());
|
||||
assert!(!ASYMMETRIC_EXCEPTIONS.iter().any(|(f, _)| *f == Format::SunRaster));
|
||||
assert_eq!(detect(&[0x59, 0xa6, 0x6a, 0x95]), None);
|
||||
assert!(decode_as(Format::SunRaster, &[0x59, 0xa6, 0x6a, 0x95]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_magic_avif() {
|
||||
assert_eq!(detect(b"\0\0\0\x1cftypavif\0\0\0\0avifmif1miaf"), Some(Format::Avif));
|
||||
@@ -360,3 +376,15 @@ fn sgi_requires_explicit_feature() {
|
||||
header[..12].copy_from_slice(&[1, 218, 0, 1, 0, 2, 0, 1, 0, 1, 0, 1]);
|
||||
assert_eq!(detect(&header), None);
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "sunraster"))]
|
||||
#[test]
|
||||
fn sun_raster_disabled_by_default() {
|
||||
use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode};
|
||||
let bytes = [0x59, 0xa6, 0x6a, 0x95, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0];
|
||||
assert_ne!(detect(&bytes), Some(Format::SunRaster));
|
||||
assert!(!caps(Format::SunRaster).read && !caps(Format::SunRaster).write);
|
||||
assert!(decode_as(Format::SunRaster, &bytes).is_err());
|
||||
let image = Image::from_u8(1, 1, ChannelLayout::Gray, vec![42]).unwrap();
|
||||
assert!(encode(&image, Format::SunRaster, &EncodeOptions::default()).is_err());
|
||||
}
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
//! `jpeg_inks`: four-component JPEGs read as Adobe-inverted inks, as `zune-jpeg`-based readers
|
||||
//! kept them, beside `decode`'s own reading.
|
||||
|
||||
use craft_image_codecs::jpeg_inks::{decode_jpeg_inks, jpeg_components, jpeg_dimensions, jpeg_icc_profile};
|
||||
use craft_image_codecs::*;
|
||||
|
||||
/// A `w` × `h` four-component JPEG of `samples` at quality 100, written by `jpeg-encoder` as
|
||||
/// `ty`, with ICC profile `icc` when given.
|
||||
fn four(w: u16, h: u16, samples: &[u8], ty: jpeg_encoder::ColorType, icc: Option<&[u8]>) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
let mut enc = jpeg_encoder::Encoder::new(&mut out, 100);
|
||||
enc.set_sampling_factor(jpeg_encoder::SamplingFactor::R_4_4_4);
|
||||
if let Some(icc) = icc {
|
||||
enc.add_icc_profile(icc).unwrap();
|
||||
}
|
||||
enc.encode(samples, w, h, ty).unwrap();
|
||||
out
|
||||
}
|
||||
|
||||
/// Flat 8×8 blocks of ink, so the coding keeps each block's value.
|
||||
fn blocks(w: usize, h: usize) -> Vec<u8> {
|
||||
(0..w * h)
|
||||
.flat_map(|i| {
|
||||
let b = ((i % w) / 8 + (i / w) / 8 * 7) as u8;
|
||||
[b.wrapping_mul(37), b.wrapping_mul(91), 255 - b.wrapping_mul(13), b.wrapping_mul(53)]
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The segments of `bytes` up to the scan without Adobe's APP14 marker.
|
||||
fn without_adobe(bytes: &[u8]) -> Vec<u8> {
|
||||
let mut out = bytes[..2].to_vec();
|
||||
let mut at = 2;
|
||||
loop {
|
||||
let marker = bytes[at + 1];
|
||||
if marker == 0xDA {
|
||||
out.extend_from_slice(&bytes[at..]);
|
||||
return out;
|
||||
}
|
||||
let len = usize::from(u16::from_be_bytes([bytes[at + 2], bytes[at + 3]]));
|
||||
if !(marker == 0xEE && &bytes[at + 4..at + 9] == b"Adobe") {
|
||||
out.extend_from_slice(&bytes[at..at + 2 + len]);
|
||||
}
|
||||
at += 2 + len;
|
||||
}
|
||||
}
|
||||
|
||||
/// Adobe CMYK reads to the inks it was written with, as `decode` reads it; without the marker
|
||||
/// the samples are still taken as inverted (where `decode` keeps them as they are), and YCCK
|
||||
/// turns into the same inks closely, truncating where `decode` rounds.
|
||||
#[test]
|
||||
fn four_component_jpegs_read_as_inverted_inks() {
|
||||
let (w, h) = (32u16, 16u16);
|
||||
let inks = blocks(usize::from(w), usize::from(h));
|
||||
let cmyk = four(w, h, &inks, jpeg_encoder::ColorType::Cmyk, None);
|
||||
let read = decode_jpeg_inks(&cmyk, &Limits::default()).unwrap();
|
||||
assert_eq!((read.dimensions(), read.layout()), ((32, 16), ChannelLayout::Cmyk));
|
||||
assert!(read.data().iter().zip(&inks).all(|(a, b)| a.abs_diff(*b) <= 2), "Adobe CMYK keeps its inks");
|
||||
assert_eq!(read.data(), decode(&cmyk).unwrap().data(), "as decode reads Adobe CMYK");
|
||||
// No marker: inverted all the same, where decode keeps the stored samples.
|
||||
let bare = without_adobe(&cmyk);
|
||||
assert_ne!(bare, cmyk);
|
||||
let unmarked = decode_jpeg_inks(&bare, &Limits::default()).unwrap();
|
||||
assert_eq!(unmarked.data(), read.data());
|
||||
let kept = decode(&bare).unwrap();
|
||||
assert!(kept.data().iter().zip(read.data()).all(|(k, r)| *k == 255 - *r), "decode keeps them as stored");
|
||||
// YCCK: libjpeg's conversion, truncated.
|
||||
let ycck = four(w, h, &inks, jpeg_encoder::ColorType::CmykAsYcck, None);
|
||||
let read = decode_jpeg_inks(&ycck, &Limits::default()).unwrap();
|
||||
assert!(read.data().iter().zip(&inks).all(|(a, b)| a.abs_diff(*b) <= 3), "YCCK keeps its inks closely");
|
||||
let rounded = decode(&ycck).unwrap();
|
||||
let differ = read.data().iter().zip(rounded.data()).filter(|(a, b)| a != b).count();
|
||||
assert!(read.data().iter().zip(rounded.data()).all(|(a, b)| a.abs_diff(*b) <= 1), "within a step of decode's rounding");
|
||||
assert!(differ > 0, "truncation differs from rounding somewhere");
|
||||
}
|
||||
|
||||
/// The headers alone give the component count of any JPEG and its ICC profile, put together from
|
||||
/// its numbered segments.
|
||||
#[test]
|
||||
fn headers_give_components_and_icc() {
|
||||
let icc: Vec<u8> = (0..200_000u32).map(|i| (i * 7 % 251) as u8).collect();
|
||||
let cmyk = four(8, 8, &[9; 256], jpeg_encoder::ColorType::Cmyk, Some(&icc));
|
||||
assert_eq!((jpeg_components(&cmyk), jpeg_dimensions(&cmyk)), (Some(4), Some((8, 8))));
|
||||
assert_eq!(jpeg_icc_profile(&cmyk).as_deref(), Some(icc.as_slice()), "three APP2 segments");
|
||||
let gray = encode(&Image::from_u8(8, 8, ChannelLayout::Gray, vec![50; 64]).unwrap(), Format::Jpeg, &EncodeOptions::default()).unwrap();
|
||||
assert_eq!((jpeg_components(&gray), jpeg_icc_profile(&gray)), (Some(1), None));
|
||||
for not in [&b""[..], b"\xFF\xD8", b"\x89PNG\r\n\x1a\n"] {
|
||||
assert_eq!((jpeg_components(not), jpeg_icc_profile(not), jpeg_dimensions(not)), (None, None, None));
|
||||
}
|
||||
}
|
||||
|
||||
/// A frame header past 16384 pixels a side, `zune-jpeg`'s default, isn't read.
|
||||
#[test]
|
||||
fn sides_past_zunes_default_are_not_read() {
|
||||
let mut gray = encode(&Image::from_u8(8, 8, ChannelLayout::Gray, vec![50; 64]).unwrap(), Format::Jpeg, &EncodeOptions::default()).unwrap();
|
||||
let sof = gray.windows(2).position(|w| w == [0xFF, 0xC0]).unwrap();
|
||||
gray[sof + 5..sof + 9].copy_from_slice(&[0x40, 0x00, 0x40, 0x00]);
|
||||
assert_eq!(jpeg_dimensions(&gray), Some((16384, 16384)));
|
||||
gray[sof + 7..sof + 9].copy_from_slice(&[0x40, 0x01]);
|
||||
assert_eq!((jpeg_dimensions(&gray), jpeg_components(&gray)), (None, None));
|
||||
}
|
||||
|
||||
/// What isn't a four-component image `zune-jpeg` decodes is refused: other component counts, an
|
||||
/// arithmetic-coded or 12-bit frame (`decode` hands those to craft-jpeg), a size past the limits,
|
||||
/// and data that isn't a JPEG.
|
||||
#[test]
|
||||
fn other_jpegs_are_refused() {
|
||||
let gray = encode(&Image::from_u8(8, 8, ChannelLayout::Gray, vec![50; 64]).unwrap(), Format::Jpeg, &EncodeOptions::default()).unwrap();
|
||||
assert!(matches!(decode_jpeg_inks(&gray, &Limits::default()), Err(CodecError::Unsupported { .. })));
|
||||
let cmyk = four(16, 8, &[40; 16 * 8 * 4], jpeg_encoder::ColorType::Cmyk, None);
|
||||
let sof = cmyk.windows(2).position(|w| w == [0xFF, 0xC0]).unwrap();
|
||||
for (marker, precision) in [(0xC9u8, 8u8), (0xC1, 12)] {
|
||||
let mut other = cmyk.clone();
|
||||
other[sof + 1] = marker;
|
||||
other[sof + 4] = precision;
|
||||
assert!(decode_jpeg_inks(&other, &Limits::default()).is_err(), "SOF {marker:#x}, {precision} bits");
|
||||
}
|
||||
// A tiny file declaring a large frame is refused before zune-jpeg allocates its output.
|
||||
let mut huge = cmyk.clone();
|
||||
huge[sof + 5..sof + 9].copy_from_slice(&[0x20, 0x00, 0x20, 0x00]);
|
||||
assert!(matches!(decode_jpeg_inks(&huge, &Limits::default()), Err(CodecError::InvalidImage(m)) if m.contains("too short")));
|
||||
let tight = Limits { max_pixels: 100, ..Limits::default() };
|
||||
assert!(matches!(decode_jpeg_inks(&cmyk, &tight), Err(CodecError::LimitExceeded(_))));
|
||||
assert!(matches!(decode_jpeg_inks(b"not a jpeg", &Limits::default()), Err(CodecError::InvalidImage(_))));
|
||||
assert!(decode_jpeg_inks(&cmyk[..12], &Limits::default()).is_err(), "headers cut short");
|
||||
}
|
||||
|
||||
/// The ink reader four-component JPEGs were read with before (vectorcraft's `jpeg_inks`, on
|
||||
/// `zune-jpeg` as this crate uses it), kept as the reference the readers here match.
|
||||
fn reference_inks(bytes: &[u8]) -> Option<(usize, usize, Vec<u8>)> {
|
||||
use zune_core::{bytestream::ZCursor, colorspace::ColorSpace, options::DecoderOptions};
|
||||
let mut d = zune_jpeg::JpegDecoder::new_with_options(ZCursor::new(bytes), DecoderOptions::default());
|
||||
d.decode_headers().ok()?;
|
||||
let space = d.input_colorspace()?;
|
||||
if d.info()?.components != 4 || !matches!(space, ColorSpace::CMYK | ColorSpace::YCCK) {
|
||||
return None;
|
||||
}
|
||||
let (w, h) = d.dimensions()?;
|
||||
d.set_options(DecoderOptions::default().jpeg_set_out_colorspace(space));
|
||||
let mut data = d.decode().ok()?;
|
||||
for p in data.as_chunks_mut::<4>().0 {
|
||||
if space == ColorSpace::YCCK {
|
||||
let [y, cb, cr] = [p[0], p[1], p[2]].map(f32::from);
|
||||
p[0] = (434.456 - y - 1.402 * cr).clamp(0.0, 255.0) as u8;
|
||||
p[1] = (119.541 - y + 0.344 * cb + 0.714 * cr).clamp(0.0, 255.0) as u8;
|
||||
p[2] = (481.816 - y - 1.772 * cb).clamp(0.0, 255.0) as u8;
|
||||
}
|
||||
for v in p.iter_mut() {
|
||||
*v = 255 - *v;
|
||||
}
|
||||
}
|
||||
Some((w, h, data))
|
||||
}
|
||||
|
||||
/// Over CMYK and YCCK JPEGs of many sizes, samplings, qualities and pictures, with and without
|
||||
/// Adobe's marker, baseline and progressive, the inks and component counts read the same as the
|
||||
/// reference reader.
|
||||
#[test]
|
||||
fn matches_the_reference_reader() {
|
||||
use jpeg_encoder::{ColorType, Encoder, SamplingFactor};
|
||||
let (mut seen, mut refused) = (0, 0);
|
||||
for (i, (w, h)) in [(1u16, 1u16), (7, 3), (8, 8), (17, 9), (33, 31), (64, 5), (100, 77)].into_iter().enumerate() {
|
||||
let (wu, n) = (usize::from(w), usize::from(w) * usize::from(h));
|
||||
let noise: Vec<u8> = (0..n * 4).map(|k| ((k as u64).wrapping_mul(2_654_435_761).wrapping_add(i as u64 * 97) >> 7) as u8).collect();
|
||||
let ramp: Vec<u8> = (0..n * 4).map(|k| (k / 4 % wu * 255 / wu.max(2)) as u8).collect();
|
||||
for samples in [&noise, &ramp] {
|
||||
for ty in [ColorType::Cmyk, ColorType::CmykAsYcck] {
|
||||
for (q, sampling, progressive) in
|
||||
[(100, SamplingFactor::R_4_4_4, false), (75, SamplingFactor::R_4_2_0, false), (40, SamplingFactor::R_4_2_2, true)]
|
||||
{
|
||||
let mut out = Vec::new();
|
||||
let mut enc = Encoder::new(&mut out, q);
|
||||
enc.set_sampling_factor(sampling);
|
||||
enc.set_progressive(progressive);
|
||||
enc.encode(samples, w, h, ty).unwrap();
|
||||
for file in [without_adobe(&out), out] {
|
||||
let read = decode_jpeg_inks(&file, &Limits::default());
|
||||
assert_eq!(jpeg_components(&file), Some(4));
|
||||
let (dw, dh) = jpeg_dimensions(&file).unwrap();
|
||||
assert_eq!((usize::from(w), usize::from(h)), (dw as usize, dh as usize));
|
||||
let Some((rw, rh, inks)) = reference_inks(&file) else {
|
||||
eprintln!("refused: {w}x{h} {ty:?} q{q} progressive {progressive}");
|
||||
assert!(read.is_err(), "refused by both: {w}x{h} {ty:?} q{q}");
|
||||
refused += 1;
|
||||
continue;
|
||||
};
|
||||
let read = read.unwrap();
|
||||
assert_eq!(read.dimensions(), (rw as u32, rh as u32));
|
||||
assert_eq!(read.data(), inks.as_slice(), "{w}x{h} {ty:?} q{q}");
|
||||
seen += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
eprintln!("read {seen}, refused {refused}");
|
||||
assert_eq!(seen + refused, 7 * 2 * 2 * 3 * 2);
|
||||
assert!(seen > refused);
|
||||
}
|
||||
|
||||
/// zune-jpeg 0.5 panics on some subsampled CMYK frames whose first component isn't the largest
|
||||
/// (here `jpeg-encoder`'s 2×2 K with C, M and Y at 1×1, progressive): an error, not a panic.
|
||||
#[test]
|
||||
fn frames_zune_panics_on_are_errors() {
|
||||
for (w, h) in [(1u16, 1u16), (3, 3)] {
|
||||
let mut out = Vec::new();
|
||||
let mut enc = jpeg_encoder::Encoder::new(&mut out, 70);
|
||||
enc.set_sampling_factor(jpeg_encoder::SamplingFactor::F_2_2);
|
||||
enc.set_progressive(true);
|
||||
enc.encode(&vec![9; usize::from(w * h) * 4], w, h, jpeg_encoder::ColorType::Cmyk).unwrap();
|
||||
assert!(std::panic::catch_unwind(|| reference_inks(&out)).is_err(), "the reference reader panics on {w}x{h}");
|
||||
assert!(matches!(decode_jpeg_inks(&out, &Limits::default()), Err(CodecError::Unsupported { .. })), "{w}x{h}");
|
||||
assert_eq!((jpeg_components(&out), jpeg_dimensions(&out)), (Some(4), Some((u32::from(w), u32::from(h)))));
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
//! The crate's integration tests, linked into one test binary.
|
||||
|
||||
mod bmp_resolution;
|
||||
mod camera_raw;
|
||||
mod common;
|
||||
mod dds;
|
||||
@@ -14,6 +15,7 @@ mod exr_stream;
|
||||
mod fidelity;
|
||||
mod gif_frames;
|
||||
mod heif;
|
||||
mod jpeg_inks;
|
||||
mod jpegls_oracle;
|
||||
mod limits;
|
||||
mod malformed;
|
||||
|
||||
@@ -759,3 +759,38 @@ fn bounded_png_stream_preserves_late_exif_across_metadata_yields() {
|
||||
assert_eq!(extras.orientation, 6);
|
||||
assert_eq!(extras.meta.text.len(), 150);
|
||||
}
|
||||
|
||||
/// zune-jpeg 0.5 asserts on some subsampled CMYK frames whose first component isn't the largest
|
||||
/// (jpeg-encoder writes C, M and Y at 1×1 and K larger); craft-jpeg reads those instead, and they
|
||||
/// decode to the inks they were written with. The others of these files read as before.
|
||||
#[test]
|
||||
fn cmyk_frames_zune_jpeg_panics_on_decode() {
|
||||
use jpeg_encoder::{ColorType, Encoder, SamplingFactor as S};
|
||||
let mut seen = 0;
|
||||
for (s, progressive) in
|
||||
[(S::F_2_2, true), (S::R_4_2_0, true), (S::F_4_1, false), (S::F_4_2, false), (S::F_2_4, false), (S::R_4_1_1, false), (S::R_4_1_0, false)]
|
||||
{
|
||||
for (w, h) in [(1u16, 1u16), (9, 1), (1, 9), (17, 13), (40, 40)] {
|
||||
let inks: Vec<u8> = (0..usize::from(w) * usize::from(h)).flat_map(|_| [30u8, 90, 150, 210]).collect();
|
||||
let mut out = Vec::new();
|
||||
let mut e = Encoder::new(&mut out, 100);
|
||||
e.set_sampling_factor(s);
|
||||
e.set_progressive(progressive);
|
||||
e.encode(&inks, w, h, ColorType::Cmyk).unwrap();
|
||||
let zune = std::panic::catch_unwind(|| {
|
||||
let mut d = zune_jpeg::JpegDecoder::new(zune_core::bytestream::ZCursor::new(&out[..]));
|
||||
d.decode().is_ok()
|
||||
});
|
||||
if zune.is_ok() {
|
||||
// Read or refused by zune-jpeg as before; no panic either way.
|
||||
let _ = decode(&out);
|
||||
continue;
|
||||
}
|
||||
seen += 1;
|
||||
let img = decode(&out).unwrap_or_else(|e| panic!("{s:?} {w}x{h}: {e}"));
|
||||
assert_eq!((img.dimensions(), img.layout()), ((u32::from(w), u32::from(h)), ChannelLayout::Cmyk));
|
||||
assert!(img.data().iter().zip(&inks).all(|(a, b)| a.abs_diff(*b) <= 3), "{s:?} progressive {progressive} {w}x{h}");
|
||||
}
|
||||
}
|
||||
assert!(seen >= 10, "zune-jpeg panicked on {seen} of them");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
use craft_image_codecs::legacy_tiff_raw_hint;
|
||||
|
||||
fn tiff(tag: u16, ty: u16, value: u32) -> Vec<u8> {
|
||||
let mut b = b"II\x2a\0\x08\0\0\0\x01\0".to_vec();
|
||||
b.extend_from_slice(&tag.to_le_bytes());
|
||||
b.extend_from_slice(&ty.to_le_bytes());
|
||||
b.extend_from_slice(&1u32.to_le_bytes());
|
||||
b.extend_from_slice(&value.to_le_bytes());
|
||||
b.extend_from_slice(&0u32.to_le_bytes());
|
||||
b
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recognizes_legacy_raw_compression_and_broad_cr_marker() {
|
||||
assert!(legacy_tiff_raw_hint(&tiff(0x0103, 3, 34713)));
|
||||
assert!(!legacy_tiff_raw_hint(&tiff(0x0103, 3, 1)));
|
||||
assert!(legacy_tiff_raw_hint(b"II*\0\x08\0\0\0CR"));
|
||||
let mut cut = tiff(0xc612, 1, 1);
|
||||
cut.truncate(17);
|
||||
assert!(!legacy_tiff_raw_hint(&cut));
|
||||
}
|
||||
|
||||
/// A little-endian TIFF with one IFD of (tag, type, count, value) entries and `tail` after it.
|
||||
fn ifd(entries: &[(u16, u16, u32, [u8; 4])], tail: &[u8]) -> Vec<u8> {
|
||||
let mut b = b"II\x2a\0\x08\0\0\0".to_vec();
|
||||
b.extend_from_slice(&(entries.len() as u16).to_le_bytes());
|
||||
for (tag, ty, count, value) in entries {
|
||||
b.extend_from_slice(&tag.to_le_bytes());
|
||||
b.extend_from_slice(&ty.to_le_bytes());
|
||||
b.extend_from_slice(&count.to_le_bytes());
|
||||
b.extend_from_slice(value);
|
||||
}
|
||||
b.extend_from_slice(&0u32.to_le_bytes());
|
||||
b.extend_from_slice(tail);
|
||||
b
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sony_lossless_ycbcr_needs_a_sony_make_and_every_layout_tag() {
|
||||
let short = |v: u16| [v.to_le_bytes()[0], v.to_le_bytes()[1], 0, 0];
|
||||
let layout = [(0x828e, 1, 4, [0, 1, 1, 2]), (0x0103, 3, 1, short(7)), (0x0106, 3, 1, short(6)), (0x0115, 3, 1, short(3))];
|
||||
// Make stored after the IFD (ASCII, 5 bytes): 8 + 2 + 5 * 12 + 4 = 74.
|
||||
let with_make = |make: (u16, u16, u32, [u8; 4]), tail: &[u8]| ifd(&[&[make], &layout[..]].concat(), tail);
|
||||
assert!(legacy_tiff_raw_hint(&with_make((0x010f, 2, 5, 74u32.to_le_bytes()), b"SONY\0")));
|
||||
assert!(legacy_tiff_raw_hint(&with_make((0x010f, 7, 4, *b"sony"), b"")));
|
||||
assert!(!legacy_tiff_raw_hint(&with_make((0x010f, 2, 6, 74u32.to_le_bytes()), b"Nikon\0")));
|
||||
assert!(!legacy_tiff_raw_hint(&with_make((0x010f, 99, 4, *b"SONY"), b"")));
|
||||
assert!(!legacy_tiff_raw_hint(&with_make((0x010f, 2, 5, 9999u32.to_le_bytes()), b"")));
|
||||
// A Sony file without the full lossless layout is not a RAW by this rule.
|
||||
let partial = ifd(&[(0x010f, 7, 4, *b"SONY"), layout[0], layout[1], layout[2]], b"");
|
||||
assert!(!legacy_tiff_raw_hint(&partial));
|
||||
// Repeated tags: the first value wins.
|
||||
let repeated = ifd(&[&[(0x010f, 7, 4, *b"SONY")], &layout[..], &[(0x0103, 3, 1, short(1))]].concat(), b"");
|
||||
assert!(legacy_tiff_raw_hint(&repeated));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_or_cut_directories_are_skipped() {
|
||||
let mut huge = b"II\x2a\0\x08\0\0\0".to_vec();
|
||||
huge.extend_from_slice(&5000u16.to_le_bytes());
|
||||
assert!(!legacy_tiff_raw_hint(&huge));
|
||||
let mut cut = ifd(&[(0x010f, 7, 4, *b"SONY"), (0x0103, 3, 1, [7, 0, 0, 0])], b"");
|
||||
cut.truncate(20);
|
||||
assert!(!legacy_tiff_raw_hint(&cut));
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
# Seeds for failure cases proptest has generated in the past. It is
|
||||
# automatically read and these particular cases re-run before any
|
||||
# novel cases are generated.
|
||||
#
|
||||
# It is recommended to check this file in to source control so that
|
||||
# everyone who runs the test benefits from these saved cases.
|
||||
cc f92b7316155ea46006de7467faa6182a90782b7befd08ed047ebeb70be15c3f7 # shrinks to which = 0, pos = 0, val = 0, cut = 0
|
||||
@@ -914,6 +914,8 @@ fn the_magnetic_lasso_shows_the_width_circle_under_the_precise_cursor() {
|
||||
#[trial]
|
||||
fn width_steps_by_one_and_shift_jumps_to_the_limits() {
|
||||
let mut settings = MagneticSettings::default();
|
||||
settings.step_width(1, false);
|
||||
assert_eq!(settings.width, 11.0, "a step adds one to the default width");
|
||||
settings.width = 10.4;
|
||||
settings.step_width(1, false);
|
||||
assert_eq!(settings.width, 11.0, "a step adds one to the rounded width");
|
||||
|
||||
@@ -380,17 +380,22 @@ fn closed_styled_previews_leave_their_composites_to_eviction() {
|
||||
|
||||
// With nearest interpolation every zoom and scale gives level 0 with nearest filtering.
|
||||
|
||||
/// A preview with the scale it is painted at.
|
||||
type PaintedPreview = (Rc<PreviewTextures>, f32);
|
||||
/// The texture and size a painted preview showed.
|
||||
type ShownPreview = (egui::TextureId, [f32; 2]);
|
||||
|
||||
/// Previews painted every frame of a fixture whose host runs jobs only when a trial says (upstream's 5 × 3 white
|
||||
/// image).
|
||||
struct Mips {
|
||||
fixture: Fixture,
|
||||
jobs: Arc<ManualJobs>,
|
||||
/// The previews the next frames paint, each at its scale.
|
||||
painted: Rc<RefCell<Vec<(Rc<PreviewTextures>, f32)>>>,
|
||||
painted: Rc<RefCell<Vec<PaintedPreview>>>,
|
||||
/// The display every painted preview is converted for.
|
||||
display: Rc<RefCell<Option<PreviewDisplay>>>,
|
||||
/// What each painted preview showed in the last frame.
|
||||
shown: Rc<RefCell<Vec<Option<(egui::TextureId, [f32; 2])>>>>,
|
||||
shown: Rc<RefCell<Vec<Option<ShownPreview>>>>,
|
||||
/// The previews' uploads in the last frame, read in the frame.
|
||||
uploads: Rc<Cell<usize>>,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
[package]
|
||||
name = "craft-macho"
|
||||
description = "Mach-O header pieces from hostile bytes, from Craft Anything: universal (fat and fat64) slice tables, CPU and load command names, load command minimum sizes and the thread-state program counter. Pure Rust, no I/O."
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
thiserror = { workspace = true }
|
||||
# `serde`: `Serialize` on `FatSlice`, for apps that report slice tables as JSON.
|
||||
serde = { workspace = true, optional = true }
|
||||
|
||||
[features]
|
||||
serde = ["dep:serde"]
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
@@ -0,0 +1,39 @@
|
||||
//! Bounds-checked fixed-width reads in either byte order: out of range is `None`.
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(crate) struct Reader<'a> {
|
||||
bytes: &'a [u8],
|
||||
}
|
||||
|
||||
impl<'a> Reader<'a> {
|
||||
pub(crate) fn new(bytes: &'a [u8]) -> Self {
|
||||
Self { bytes }
|
||||
}
|
||||
|
||||
/// `len` bytes starting at `at`.
|
||||
pub(crate) fn slice(&self, at: usize, len: usize) -> Option<&'a [u8]> {
|
||||
let end = at.checked_add(len)?;
|
||||
self.bytes.get(at..end)
|
||||
}
|
||||
|
||||
fn word<const N: usize>(&self, at: usize) -> Option<[u8; N]> {
|
||||
let (word, _) = self.bytes.get(at..)?.split_first_chunk::<N>()?;
|
||||
Some(*word)
|
||||
}
|
||||
|
||||
pub(crate) fn u32_le(&self, at: usize) -> Option<u32> {
|
||||
self.word(at).map(u32::from_le_bytes)
|
||||
}
|
||||
|
||||
pub(crate) fn u64_le(&self, at: usize) -> Option<u64> {
|
||||
self.word(at).map(u64::from_le_bytes)
|
||||
}
|
||||
|
||||
pub(crate) fn u32_be(&self, at: usize) -> Option<u32> {
|
||||
self.word(at).map(u32::from_be_bytes)
|
||||
}
|
||||
|
||||
pub(crate) fn u64_be(&self, at: usize) -> Option<u64> {
|
||||
self.word(at).map(u64::from_be_bytes)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
//! Mach-O header pieces from untrusted bytes: universal (fat and fat64) slice tables
|
||||
//! ([`fat_slices`]), CPU and load command names, the fixed size each known load command needs
|
||||
//! before its fields can be read, and the program counter in a thread command.
|
||||
//!
|
||||
//! From Craft Anything (`crates/formats/src/macho.rs`), where the load command walker, symbol,
|
||||
//! stub and Objective-C code built on these stays. With the `serde` feature, [`FatSlice`] is
|
||||
//! `Serialize`.
|
||||
|
||||
mod bytes;
|
||||
|
||||
use crate::bytes::Reader;
|
||||
|
||||
/// Slices allowed in a universal binary's table.
|
||||
pub const MAX_FAT_SLICES: usize = 64;
|
||||
|
||||
/// Thread-state records walked in one thread command.
|
||||
pub const MAX_THREAD_STATES: usize = 4096;
|
||||
|
||||
/// Why [`fat_slices`] gave up.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum MachoError {
|
||||
/// The header or a slice ends before the file does.
|
||||
#[error("the file ends early ({0})")]
|
||||
Truncated(String),
|
||||
/// Not a universal binary, or an implausible slice count.
|
||||
#[error("the file is malformed ({0})")]
|
||||
Malformed(String),
|
||||
}
|
||||
|
||||
/// One architecture in a universal binary's slice table.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize))]
|
||||
pub struct FatSlice {
|
||||
/// [`cpu_name`] of the slice's CPU type and subtype.
|
||||
pub arch: String,
|
||||
pub cputype: u32,
|
||||
pub cpusubtype: u32,
|
||||
/// File offset of the thin image.
|
||||
pub offset: u64,
|
||||
pub size: u64,
|
||||
/// Alignment as a power of two.
|
||||
pub align: u32,
|
||||
}
|
||||
|
||||
/// A CPU type and subtype as the name `lipo` and `-arch` use (`x86_64h`, `arm64e`…), or
|
||||
/// `cpu0x…` for others.
|
||||
pub fn cpu_name(cputype: u32, sub: u32) -> String {
|
||||
match cputype {
|
||||
7 => "i386".into(),
|
||||
0x0100_0007 if sub & 0x00ff_ffff == 8 => "x86_64h".into(),
|
||||
0x0100_0007 => "x86_64".into(),
|
||||
12 => "arm".into(),
|
||||
0x0100_000c => {
|
||||
if sub & 0xff == 2 {
|
||||
"arm64e".into()
|
||||
} else {
|
||||
"arm64".into()
|
||||
}
|
||||
}
|
||||
0x0200_000c => "arm64_32".into(),
|
||||
18 => "ppc".into(),
|
||||
0x0100_0012 => "ppc64".into(),
|
||||
other => format!("cpu{other:#x}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The slices of a universal (fat, or fat64 with magic `0xcafebabf`) binary. Every slice must
|
||||
/// lie inside `data`. More than [`MAX_FAT_SLICES`] slices is refused: Java class files share the
|
||||
/// magic and carry their version there.
|
||||
pub fn fat_slices(data: &[u8]) -> Result<Vec<FatSlice>, MachoError> {
|
||||
let r = Reader::new(data);
|
||||
let magic = r.u32_be(0).ok_or_else(|| MachoError::Truncated("fat header".into()))?;
|
||||
let wide = match magic {
|
||||
0xcafe_babe => false,
|
||||
0xcafe_babf => true,
|
||||
_ => return Err(MachoError::Malformed("not a universal binary".into())),
|
||||
};
|
||||
let n = r.u32_be(4).ok_or_else(|| MachoError::Truncated("fat header".into()))? as usize;
|
||||
if n == 0 || n > MAX_FAT_SLICES {
|
||||
return Err(MachoError::Malformed(format!("{n} slices (Java class files share this magic)")));
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
for i in 0..n {
|
||||
let s = if wide {
|
||||
let o = 8 + i * 32;
|
||||
(r.u32_be(o), r.u32_be(o + 4), r.u64_be(o + 8), r.u64_be(o + 16), r.u32_be(o + 24))
|
||||
} else {
|
||||
let o = 8 + i * 20;
|
||||
(r.u32_be(o), r.u32_be(o + 4), r.u32_be(o + 8).map(u64::from), r.u32_be(o + 12).map(u64::from), r.u32_be(o + 16))
|
||||
};
|
||||
let (Some(cpu), Some(sub), Some(offset), Some(size), Some(align)) = s else {
|
||||
return Err(MachoError::Truncated("fat arch table".into()));
|
||||
};
|
||||
if offset.checked_add(size).is_none_or(|end| end > data.len() as u64) {
|
||||
return Err(MachoError::Truncated(format!("slice {i} runs past the end of the file")));
|
||||
}
|
||||
out.push(FatSlice { arch: cpu_name(cpu, sub), cputype: cpu, cpusubtype: sub, offset, size, align });
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// A load command's `LC_` name, or its number in hex for ones not listed.
|
||||
pub fn load_command_name(cmd: u32) -> String {
|
||||
match cmd {
|
||||
0x1 => "LC_SEGMENT",
|
||||
0x2 => "LC_SYMTAB",
|
||||
0x5 => "LC_UNIXTHREAD",
|
||||
0xb => "LC_DYSYMTAB",
|
||||
0xc => "LC_LOAD_DYLIB",
|
||||
0xd => "LC_ID_DYLIB",
|
||||
0xe => "LC_LOAD_DYLINKER",
|
||||
0x19 => "LC_SEGMENT_64",
|
||||
0x1b => "LC_UUID",
|
||||
0x1d => "LC_CODE_SIGNATURE",
|
||||
0x1e => "LC_SEGMENT_SPLIT_INFO",
|
||||
0x20 => "LC_LAZY_LOAD_DYLIB",
|
||||
0x21 => "LC_ENCRYPTION_INFO",
|
||||
0x22 => "LC_DYLD_INFO",
|
||||
0x8000_0022 => "LC_DYLD_INFO_ONLY",
|
||||
0x24 => "LC_VERSION_MIN_MACOSX",
|
||||
0x25 => "LC_VERSION_MIN_IPHONEOS",
|
||||
0x26 => "LC_FUNCTION_STARTS",
|
||||
0x29 => "LC_DATA_IN_CODE",
|
||||
0x2a => "LC_SOURCE_VERSION",
|
||||
0x2c => "LC_ENCRYPTION_INFO_64",
|
||||
0x32 => "LC_BUILD_VERSION",
|
||||
0x8000_0018 => "LC_LOAD_WEAK_DYLIB",
|
||||
0x8000_001c => "LC_RPATH",
|
||||
0x8000_001f => "LC_REEXPORT_DYLIB",
|
||||
0x8000_0028 => "LC_MAIN",
|
||||
0x8000_0033 => "LC_DYLD_EXPORTS_TRIE",
|
||||
0x8000_0034 => "LC_DYLD_CHAINED_FIXUPS",
|
||||
other => return format!("{other:#x}"),
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
/// Fixed bytes required before decoding a known load command's fields (8, the command header,
|
||||
/// for unknown ones).
|
||||
pub fn load_command_min_size(cmd: u32) -> usize {
|
||||
match cmd {
|
||||
0x1 => 56,
|
||||
0x2 | 0x1b | 0x2c | 0x32 | 0x8000_0028 => 24,
|
||||
0x5 => 16,
|
||||
0xb => 80,
|
||||
0xc | 0xd | 0x20 | 0x8000_0018 | 0x8000_001f | 0x8000_0023 => 24,
|
||||
0xe | 0xf | 0x12 | 0x13 | 0x14 | 0x15 | 0x27 | 0x8000_001c => 12,
|
||||
0x19 => 72,
|
||||
0x1d | 0x1e | 0x24 | 0x25 | 0x26 | 0x29 | 0x2a | 0x2b | 0x2e | 0x2f | 0x30 | 0x8000_0033 | 0x8000_0034 => 16,
|
||||
0x21 => 20,
|
||||
0x22 | 0x8000_0022 => 48,
|
||||
_ => 8,
|
||||
}
|
||||
}
|
||||
|
||||
/// An `X.Y.Z` version packed as `xxxx.yy.zz` nibbles (`LC_VERSION_MIN_*`, `LC_BUILD_VERSION`).
|
||||
pub fn nibble_version(v: u32) -> String {
|
||||
format!("{}.{}.{}", v >> 16, (v >> 8) & 0xff, v & 0xff)
|
||||
}
|
||||
|
||||
/// The program counter in an `LC_UNIXTHREAD` / `LC_THREAD` body (the bytes after its 8-byte
|
||||
/// command header): the first complete x86_64 (flavor 4), arm64 (flavor 6) or i386 (flavor 1)
|
||||
/// thread state for `cpu`. A command can carry several flavor/count/state records; counts are
|
||||
/// 32-bit words. At most [`MAX_THREAD_STATES`] records are walked.
|
||||
pub fn unixthread_pc(data: &[u8], cpu: u32, is64: bool) -> Option<u64> {
|
||||
let r = Reader::new(data);
|
||||
let mut pos = 0usize;
|
||||
for _ in 0..MAX_THREAD_STATES {
|
||||
let flavor = r.u32_le(pos)?;
|
||||
let count = r.u32_le(pos.checked_add(4)?)?;
|
||||
let state = pos.checked_add(8)?;
|
||||
let size = usize::try_from(count).ok()?.checked_mul(4)?;
|
||||
let record = Reader::new(r.slice(state, size)?);
|
||||
let pc = match (cpu, is64, flavor) {
|
||||
(0x0100_0007, true, 4) if count >= 42 => record.u64_le(16 * 8),
|
||||
(0x0100_000c, true, 6) if count >= 68 => record.u64_le(32 * 8),
|
||||
(7, false, 1) if count >= 16 => record.u32_le(10 * 4).map(u64::from),
|
||||
_ => None,
|
||||
};
|
||||
if pc.is_some() {
|
||||
return pc;
|
||||
}
|
||||
pos = state.checked_add(size)?;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,213 @@
|
||||
//! Slice table, name and thread-state tests. The thread-state cases are Craft Anything's
|
||||
//! `macho_thread_state` integration test, called on the command body directly instead of
|
||||
//! through the app's program model.
|
||||
|
||||
use super::*;
|
||||
|
||||
const X86_64: u32 = 0x0100_0007;
|
||||
const ARM64: u32 = 0x0100_000c;
|
||||
|
||||
/// A universal binary with `slices` of (cputype, cpusubtype, body), each at a 0x100 boundary.
|
||||
fn fat(wide: bool, slices: &[(u32, u32, &[u8])]) -> Vec<u8> {
|
||||
let entry = if wide { 32 } else { 20 };
|
||||
let mut d = vec![0u8; 0x100];
|
||||
d[0..4].copy_from_slice(&(if wide { 0xcafe_babfu32 } else { 0xcafe_babe }).to_be_bytes());
|
||||
d[4..8].copy_from_slice(&(slices.len() as u32).to_be_bytes());
|
||||
for (i, &(cpu, sub, body)) in slices.iter().enumerate() {
|
||||
let offset = d.len();
|
||||
let o = 8 + i * entry;
|
||||
d[o..o + 4].copy_from_slice(&cpu.to_be_bytes());
|
||||
d[o + 4..o + 8].copy_from_slice(&sub.to_be_bytes());
|
||||
if wide {
|
||||
d[o + 8..o + 16].copy_from_slice(&(offset as u64).to_be_bytes());
|
||||
d[o + 16..o + 24].copy_from_slice(&(body.len() as u64).to_be_bytes());
|
||||
d[o + 24..o + 28].copy_from_slice(&8u32.to_be_bytes());
|
||||
} else {
|
||||
d[o + 8..o + 12].copy_from_slice(&(offset as u32).to_be_bytes());
|
||||
d[o + 12..o + 16].copy_from_slice(&(body.len() as u32).to_be_bytes());
|
||||
d[o + 16..o + 20].copy_from_slice(&8u32.to_be_bytes());
|
||||
}
|
||||
d.extend_from_slice(body);
|
||||
d.resize(d.len().next_multiple_of(0x100), 0);
|
||||
}
|
||||
d
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fat_and_fat64_tables_list_their_slices() {
|
||||
for wide in [false, true] {
|
||||
let d = fat(wide, &[(X86_64, 3, &[1; 16]), (ARM64, 2, &[2; 32])]);
|
||||
let s = fat_slices(&d).unwrap();
|
||||
assert_eq!(s.len(), 2, "wide={wide}");
|
||||
assert_eq!(s[0], FatSlice { arch: "x86_64".into(), cputype: X86_64, cpusubtype: 3, offset: 0x100, size: 16, align: 8 });
|
||||
assert_eq!((s[1].arch.as_str(), s[1].offset, s[1].size), ("arm64e", 0x200, 32));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn damaged_tables_are_errors() {
|
||||
assert_eq!(fat_slices(b"\xca\xfe"), Err(MachoError::Truncated("fat header".into())));
|
||||
assert_eq!(fat_slices(&[0xfe, 0xed, 0xfa, 0xcf, 0, 0, 0, 1]), Err(MachoError::Malformed("not a universal binary".into())));
|
||||
assert_eq!(fat_slices(&[0xca, 0xfe, 0xba, 0xbe]), Err(MachoError::Truncated("fat header".into())));
|
||||
// A Java 21 class file: minor 0, major 65.
|
||||
let java = [0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 65];
|
||||
assert_eq!(fat_slices(&java).unwrap_err().to_string(), "the file is malformed (65 slices (Java class files share this magic))");
|
||||
assert!(matches!(fat_slices(&[0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 0]), Err(MachoError::Malformed(_))));
|
||||
let d = fat(false, &[(X86_64, 3, &[1; 16])]);
|
||||
assert_eq!(fat_slices(&d[..20]), Err(MachoError::Truncated("fat arch table".into())));
|
||||
assert_eq!(fat_slices(&d[..0x108]), Err(MachoError::Truncated("slice 0 runs past the end of the file".into())));
|
||||
let mut wrap = fat(true, &[(X86_64, 3, &[1; 16])]);
|
||||
wrap[16..24].copy_from_slice(&u64::MAX.to_be_bytes());
|
||||
assert!(matches!(fat_slices(&wrap), Err(MachoError::Truncated(_))), "offset + size must not wrap");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cpu_names() {
|
||||
let names: Vec<String> = [
|
||||
(7, 3),
|
||||
(X86_64, 3),
|
||||
(X86_64, 8),
|
||||
(X86_64, 0x8000_0008),
|
||||
(X86_64, 0x108),
|
||||
(12, 9),
|
||||
(ARM64, 0),
|
||||
(ARM64, 0x8000_0002),
|
||||
(0x0200_000c, 1),
|
||||
(18, 0),
|
||||
(0x0100_0012, 0),
|
||||
(0x99, 0),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(cpu, sub)| cpu_name(cpu, sub))
|
||||
.collect();
|
||||
assert_eq!(names, ["i386", "x86_64", "x86_64h", "x86_64h", "x86_64", "arm", "arm64", "arm64e", "arm64_32", "ppc", "ppc64", "cpu0x99"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_command_names_and_sizes() {
|
||||
let known = [
|
||||
(0x1, "LC_SEGMENT", 56),
|
||||
(0x2, "LC_SYMTAB", 24),
|
||||
(0x5, "LC_UNIXTHREAD", 16),
|
||||
(0xb, "LC_DYSYMTAB", 80),
|
||||
(0xc, "LC_LOAD_DYLIB", 24),
|
||||
(0xd, "LC_ID_DYLIB", 24),
|
||||
(0xe, "LC_LOAD_DYLINKER", 12),
|
||||
(0x19, "LC_SEGMENT_64", 72),
|
||||
(0x1b, "LC_UUID", 24),
|
||||
(0x1d, "LC_CODE_SIGNATURE", 16),
|
||||
(0x1e, "LC_SEGMENT_SPLIT_INFO", 16),
|
||||
(0x20, "LC_LAZY_LOAD_DYLIB", 24),
|
||||
(0x21, "LC_ENCRYPTION_INFO", 20),
|
||||
(0x22, "LC_DYLD_INFO", 48),
|
||||
(0x8000_0022, "LC_DYLD_INFO_ONLY", 48),
|
||||
(0x24, "LC_VERSION_MIN_MACOSX", 16),
|
||||
(0x25, "LC_VERSION_MIN_IPHONEOS", 16),
|
||||
(0x26, "LC_FUNCTION_STARTS", 16),
|
||||
(0x29, "LC_DATA_IN_CODE", 16),
|
||||
(0x2a, "LC_SOURCE_VERSION", 16),
|
||||
(0x2c, "LC_ENCRYPTION_INFO_64", 24),
|
||||
(0x32, "LC_BUILD_VERSION", 24),
|
||||
(0x8000_0018, "LC_LOAD_WEAK_DYLIB", 24),
|
||||
(0x8000_001c, "LC_RPATH", 12),
|
||||
(0x8000_001f, "LC_REEXPORT_DYLIB", 24),
|
||||
(0x8000_0028, "LC_MAIN", 24),
|
||||
(0x8000_0033, "LC_DYLD_EXPORTS_TRIE", 16),
|
||||
(0x8000_0034, "LC_DYLD_CHAINED_FIXUPS", 16),
|
||||
];
|
||||
for (cmd, name, size) in known {
|
||||
assert_eq!(load_command_name(cmd), name);
|
||||
assert_eq!(load_command_min_size(cmd), size, "{name}");
|
||||
}
|
||||
assert_eq!(load_command_name(0x8000_0023), "0x80000023");
|
||||
assert_eq!(load_command_min_size(0x8000_0023), 24);
|
||||
assert_eq!(load_command_min_size(0x27), 12);
|
||||
assert_eq!(load_command_min_size(0x2e), 16);
|
||||
assert_eq!(load_command_min_size(0x77), 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nibble_versions() {
|
||||
assert_eq!(nibble_version(0x000e_0500), "14.5.0");
|
||||
assert_eq!(nibble_version(0x000a_0f01), "10.15.1");
|
||||
}
|
||||
|
||||
fn state(flavor: u32, count: u32, bytes: usize, pc_offset: usize, pc: u64, wide: bool) -> Vec<u8> {
|
||||
let mut b = vec![0u8; 8 + bytes];
|
||||
b[0..4].copy_from_slice(&flavor.to_le_bytes());
|
||||
b[4..8].copy_from_slice(&count.to_le_bytes());
|
||||
if wide {
|
||||
b[8 + pc_offset..16 + pc_offset].copy_from_slice(&pc.to_le_bytes());
|
||||
} else {
|
||||
b[8 + pc_offset..12 + pc_offset].copy_from_slice(&(pc as u32).to_le_bytes());
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
fn decode(cpu: u32, wide: bool, states: &[Vec<u8>]) -> Option<u64> {
|
||||
unixthread_pc(&states.concat(), cpu, wide)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_program_counter_requires_a_complete_declared_state() {
|
||||
for count in [0, 1, 34, 41] {
|
||||
assert_eq!(decode(X86_64, true, &[state(4, count, 168, 128, 0x1234, true)]), None, "count {count}");
|
||||
}
|
||||
assert_eq!(decode(X86_64, true, &[state(4, 42, 136, 128, 0x1234, true)]), None);
|
||||
assert_eq!(decode(ARM64, true, &[state(6, 68, 264, 256, 0x1234, true)]), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flavor_numbers_are_interpreted_for_the_image_cpu() {
|
||||
assert_eq!(decode(ARM64, true, &[state(4, 42, 168, 128, 0x1234, true)]), None);
|
||||
assert_eq!(decode(X86_64, true, &[state(6, 68, 272, 256, 0x1234, true)]), None);
|
||||
assert_eq!(decode(12, false, &[state(1, 17, 68, 40, 0x1234, false)]), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_later_matching_state_is_reached_using_word_counts() {
|
||||
let unknown = state(99, 2, 8, 0, 0, true);
|
||||
let matching = state(4, 42, 168, 128, 0x1234, true);
|
||||
assert_eq!(decode(X86_64, true, &[unknown, matching]), Some(0x1234));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_supported_states_keep_their_program_counters() {
|
||||
assert_eq!(decode(X86_64, true, &[state(4, 42, 168, 128, 0x1234, true)]), Some(0x1234));
|
||||
assert_eq!(decode(ARM64, true, &[state(6, 68, 272, 256, 0x1234, true)]), Some(0x1234));
|
||||
assert_eq!(decode(7, false, &[state(1, 16, 64, 40, 0x1234, false)]), Some(0x1234));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endless_empty_states_stop() {
|
||||
assert_eq!(unixthread_pc(&vec![0; 8 * (MAX_THREAD_STATES + 4)], X86_64, true), None);
|
||||
assert_eq!(unixthread_pc(&[4, 0, 0, 0, 0xff, 0xff, 0xff, 0xff], X86_64, true), None);
|
||||
}
|
||||
|
||||
#[cfg(feature = "serde")]
|
||||
#[test]
|
||||
fn slices_serialize() {
|
||||
let s = fat_slices(&fat(false, &[(ARM64, 0, &[0; 4])])).unwrap();
|
||||
let value = serde_json::to_value(&s[0]).unwrap();
|
||||
assert_eq!(value["arch"], "arm64");
|
||||
assert_eq!(value["offset"], 0x100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn damaged_input_never_panics() {
|
||||
let sample = fat(true, &[(X86_64, 3, &[1; 16]), (ARM64, 2, &[2; 32])]);
|
||||
let thread = state(4, 42, 168, 128, 0x1234, true);
|
||||
for source in [&sample, &thread] {
|
||||
for i in 0..source.len() {
|
||||
let _ = fat_slices(&source[..i]);
|
||||
let _ = unixthread_pc(&source[..i], X86_64, true);
|
||||
for v in [0x00, 0x7f, 0xff] {
|
||||
let mut m = source.clone();
|
||||
m[i] = v;
|
||||
let _ = fat_slices(&m);
|
||||
let _ = unixthread_pc(&m, ARM64, true);
|
||||
let _ = unixthread_pc(&m, 7, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
[package]
|
||||
name = "craft-mcp-server"
|
||||
description = "MCP server machinery any app may share: JSON-RPC 2.0 over stdio, the MCP lifecycle, tool and resource methods, tool schemas and argument checks, image results"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde_json = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = { workspace = true }
|
||||
@@ -0,0 +1,64 @@
|
||||
//! The content blocks a tool result is made of (MCP `CallToolResult`), and the contents a resource
|
||||
//! read sends (MCP `ReadResourceResult`).
|
||||
|
||||
use serde_json::{Value, json};
|
||||
|
||||
/// What `resources/read` sends for one resource: text of a MIME type, or bytes of a MIME type,
|
||||
/// which go out base64-encoded.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Contents {
|
||||
Text { mime_type: String, text: String },
|
||||
Blob { mime_type: String, data: Vec<u8> },
|
||||
}
|
||||
|
||||
impl Contents {
|
||||
/// `value` as JSON text (`application/json`).
|
||||
pub fn json(value: &Value) -> Self {
|
||||
Self::Text { mime_type: "application/json".into(), text: value.to_string() }
|
||||
}
|
||||
|
||||
/// The MCP contents block of resource `uri`.
|
||||
pub(crate) fn block(&self, uri: &str) -> Value {
|
||||
match self {
|
||||
Self::Text { mime_type, text } => json!({"uri": uri, "mimeType": mime_type, "text": text}),
|
||||
Self::Blob { mime_type, data } => json!({"uri": uri, "mimeType": mime_type, "blob": base64(data)}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A tool result of one text block, flagged as an error when `is_error` (the model sees the
|
||||
/// message; an error is still a result, not a JSON-RPC error).
|
||||
pub fn text_result(text: impl Into<String>, is_error: bool) -> Value {
|
||||
let mut r = json!({"content": [{"type": "text", "text": text.into()}]});
|
||||
if is_error {
|
||||
r["isError"] = json!(true);
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
/// A tool result of an image (`data` of type `mime`, base64-encoded here) followed by a text
|
||||
/// block saying what it shows.
|
||||
pub fn image_result(data: &[u8], mime: &str, text: impl Into<String>) -> Value {
|
||||
json!({"content": [
|
||||
{"type": "image", "data": base64(data), "mimeType": mime},
|
||||
{"type": "text", "text": text.into()},
|
||||
]})
|
||||
}
|
||||
|
||||
/// Standard base64 with padding (RFC 4648), as MCP image content carries it.
|
||||
pub fn base64(data: &[u8]) -> String {
|
||||
const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
let mut out = String::with_capacity(data.len().div_ceil(3) * 4);
|
||||
for chunk in data.chunks(3) {
|
||||
let b = [chunk.first().copied().unwrap_or(0), chunk.get(1).copied().unwrap_or(0), chunk.get(2).copied().unwrap_or(0)];
|
||||
let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
|
||||
for (i, shift) in [18u32, 12, 6, 0].into_iter().enumerate() {
|
||||
if i <= chunk.len() {
|
||||
out.push(char::from(T[((n >> shift) & 63) as usize]));
|
||||
} else {
|
||||
out.push('=');
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
//! MCP server machinery any app may share: JSON-RPC 2.0 over stdio (newline-delimited), the MCP
|
||||
//! lifecycle, the tools, resources, prompts and completion methods, tool schemas with the check of
|
||||
//! a call's arguments against them, and the content blocks tools reply with (text, images).
|
||||
//!
|
||||
//! An app implements [`Tools`]: what its tools are, how each one runs, its instructions, and the
|
||||
//! resources, resource templates and prompts it serves, if any. [`Server`] speaks the protocol
|
||||
//! around them and tells a client at `initialize` only what the app serves. Nothing here knows any
|
||||
//! app's design, engine or renderer.
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
mod content;
|
||||
mod schema;
|
||||
mod server;
|
||||
|
||||
pub use content::{Contents, base64, image_result, text_result};
|
||||
pub use schema::{Effect, check_arguments, object, string, tool};
|
||||
pub use server::{Incoming, Info, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, read_message, response};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,132 @@
|
||||
//! Tool definitions (MCP `Tool`: name, title, description, input schema, annotations) and the
|
||||
//! check of a call's arguments against the definition it was listed with.
|
||||
|
||||
use serde_json::{Value, json};
|
||||
|
||||
/// What a tool may do, for the client's approval prompts (MCP tool annotations). Clients that
|
||||
/// auto-approve on these hints must not be told less than the truth.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Effect {
|
||||
/// Only looks.
|
||||
ReadOnly,
|
||||
/// Changes the app's document in memory, undoably.
|
||||
Edits,
|
||||
/// Replaces the app's document in memory (unsaved changes are lost).
|
||||
Replaces,
|
||||
/// May read, write, overwrite or delete files on disk.
|
||||
Files,
|
||||
}
|
||||
|
||||
/// A tool's definition, its annotations from `effect`.
|
||||
pub fn tool(name: &str, title: &str, desc: &str, schema: Value, effect: Effect) -> Value {
|
||||
let (read_only, destructive, open_world) = match effect {
|
||||
Effect::ReadOnly => (true, false, false),
|
||||
Effect::Edits => (false, false, false),
|
||||
Effect::Replaces => (false, true, false),
|
||||
Effect::Files => (false, true, true),
|
||||
};
|
||||
json!({
|
||||
"name": name,
|
||||
"title": title,
|
||||
"description": desc,
|
||||
"inputSchema": schema,
|
||||
"annotations": {"title": title, "readOnlyHint": read_only, "destructiveHint": destructive, "idempotentHint": read_only, "openWorldHint": open_world},
|
||||
})
|
||||
}
|
||||
|
||||
/// An object schema with these `props`, the `required` ones, and no other keys.
|
||||
pub fn object(props: Value, required: &[&str]) -> Value {
|
||||
let mut o = json!({"type": "object", "properties": props, "additionalProperties": false});
|
||||
if !required.is_empty() {
|
||||
o["required"] = json!(required);
|
||||
}
|
||||
o
|
||||
}
|
||||
|
||||
/// A string property described by `desc`.
|
||||
pub fn string(desc: &str) -> Value {
|
||||
json!({"type": "string", "description": desc})
|
||||
}
|
||||
|
||||
fn type_matches(t: &str, v: &Value) -> bool {
|
||||
match t {
|
||||
"string" => v.is_string(),
|
||||
"number" => v.is_number(),
|
||||
"integer" => v.is_i64() || v.is_u64(),
|
||||
"boolean" => v.is_boolean(),
|
||||
"array" => v.is_array(),
|
||||
"object" => v.is_object(),
|
||||
"null" => v.is_null(),
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check `args` against tool `name`'s input schema as `defs` list it (top level): unknown tool,
|
||||
/// missing required arguments, unknown arguments, type and enum mismatches.
|
||||
pub fn check_arguments(defs: &[Value], name: &str, args: &Value) -> Result<(), String> {
|
||||
let def = defs.iter().find(|d| d["name"] == name).ok_or_else(|| format!("unknown tool `{name}`"))?;
|
||||
let schema = &def["inputSchema"];
|
||||
let Some(a) = args.as_object() else { return Err(format!("{name}: arguments must be an object")) };
|
||||
for r in schema["required"].as_array().into_iter().flatten().filter_map(Value::as_str) {
|
||||
if a.get(r).is_none_or(Value::is_null) {
|
||||
return Err(format!("{name}: missing required argument `{r}`"));
|
||||
}
|
||||
}
|
||||
let props = schema["properties"].as_object();
|
||||
for (k, v) in a {
|
||||
let Some(p) = props.and_then(|p| p.get(k)) else {
|
||||
let valid: Vec<&str> = props.map(|p| p.keys().map(String::as_str).collect()).unwrap_or_default();
|
||||
let valid = if valid.is_empty() { "none".to_string() } else { valid.join(", ") };
|
||||
let k: String = k.chars().take(64).collect();
|
||||
return Err(format!("{name}: unknown argument `{k}`; its arguments are: {valid}"));
|
||||
};
|
||||
if v.is_null() {
|
||||
continue;
|
||||
}
|
||||
// An argument of several types lists them as `anyOf` (Gemini refuses a list of types).
|
||||
let types: Vec<&Value> = match p["anyOf"].as_array() {
|
||||
Some(alts) => alts.iter().map(|alt| &alt["type"]).collect(),
|
||||
None => vec![&p["type"]],
|
||||
};
|
||||
let ok = types.iter().any(|t| match t {
|
||||
Value::String(t) => type_matches(t, v),
|
||||
Value::Array(ts) => ts.iter().filter_map(Value::as_str).any(|t| type_matches(t, v)),
|
||||
_ => true,
|
||||
});
|
||||
if !ok {
|
||||
let expected = if types.len() == 1 { types[0].clone() } else { Value::Array(types.into_iter().cloned().collect()) };
|
||||
return Err(format!("{name}: argument `{k}` must be of type {expected}"));
|
||||
}
|
||||
if let Some(allowed) = p["enum"].as_array()
|
||||
&& !allowed.contains(v)
|
||||
{
|
||||
return Err(format!("{name}: `{k}` must be one of {}", Value::Array(allowed.clone())));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check `args` against prompt `name` as `defs` list it (MCP `Prompt`): unknown prompt, missing
|
||||
/// required arguments, unknown arguments, values that are not strings.
|
||||
pub(crate) fn check_prompt_arguments(defs: &[Value], name: &str, args: &Value) -> Result<(), String> {
|
||||
let def = defs.iter().find(|d| d["name"] == name).ok_or_else(|| format!("unknown prompt `{name}`"))?;
|
||||
let declared = def["arguments"].as_array().map(Vec::as_slice).unwrap_or_default();
|
||||
let Some(a) = args.as_object() else { return Err(format!("{name}: arguments must be an object")) };
|
||||
for r in declared.iter().filter(|d| d["required"] == true).filter_map(|d| d["name"].as_str()) {
|
||||
if !a.contains_key(r) {
|
||||
return Err(format!("{name}: missing required argument `{r}`"));
|
||||
}
|
||||
}
|
||||
for (k, v) in a {
|
||||
if !declared.iter().any(|d| d["name"] == k.as_str()) {
|
||||
let valid: Vec<&str> = declared.iter().filter_map(|d| d["name"].as_str()).collect();
|
||||
let valid = if valid.is_empty() { "none".to_string() } else { valid.join(", ") };
|
||||
let k: String = k.chars().take(64).collect();
|
||||
return Err(format!("{name}: unknown argument `{k}`; its arguments are: {valid}"));
|
||||
}
|
||||
if !v.is_string() {
|
||||
return Err(format!("{name}: argument `{k}` must be a string"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
//! JSON-RPC 2.0 framing and the MCP lifecycle / tools / resources / prompts / completion methods.
|
||||
|
||||
use std::io::{BufRead, Write};
|
||||
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::content::Contents;
|
||||
use crate::schema::{check_arguments, check_prompt_arguments};
|
||||
|
||||
/// The newest protocol revision we speak; offered when the client asks for one we don't know.
|
||||
pub const PROTOCOL_VERSION: &str = "2025-06-18";
|
||||
pub const SUPPORTED_VERSIONS: &[&str] = &[PROTOCOL_VERSION, "2025-03-26", "2024-11-05"];
|
||||
|
||||
const PARSE_ERROR: i64 = -32700;
|
||||
const INVALID_REQUEST: i64 = -32600;
|
||||
const METHOD_NOT_FOUND: i64 = -32601;
|
||||
const INVALID_PARAMS: i64 = -32602;
|
||||
const INTERNAL_ERROR: i64 = -32603;
|
||||
const RESOURCE_NOT_FOUND: i64 = -32002;
|
||||
|
||||
/// Most messages in one JSON-RPC batch.
|
||||
const MAX_BATCH: usize = 1000;
|
||||
/// Most suggestions in one `completion/complete` reply (the MCP limit).
|
||||
const MAX_COMPLETIONS: usize = 100;
|
||||
|
||||
/// What `completion/complete` asks about: a prompt by its name, or a resource by its URI or URI
|
||||
/// template.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Reference<'a> {
|
||||
Prompt(&'a str),
|
||||
Resource(&'a str),
|
||||
}
|
||||
|
||||
/// An app's tools, served by [`Server`].
|
||||
pub trait Tools {
|
||||
/// What every client is told at `initialize`.
|
||||
fn instructions(&self) -> String;
|
||||
/// Every tool's definition ([`crate::tool`]), as `tools/list` serves it now.
|
||||
fn list(&mut self) -> Vec<Value>;
|
||||
/// Run tool `name`. The server calls it right after the [`Tools::list`] its arguments were
|
||||
/// checked against ([`crate::check_arguments`]), so it runs the tool as that listing defined
|
||||
/// it. Failures are tool results with `isError: true` ([`crate::text_result`]); it never
|
||||
/// panics on hostile arguments.
|
||||
fn call(&mut self, name: &str, args: &Value) -> Value;
|
||||
/// The resources `resources/list` serves (MCP `Resource`: uri, name, description, mimeType).
|
||||
fn resources(&self) -> Vec<Value> {
|
||||
Vec::new()
|
||||
}
|
||||
/// The resource templates `resources/templates/list` serves (MCP `ResourceTemplate`:
|
||||
/// uriTemplate, name, description, mimeType). A URI made from one is read with
|
||||
/// [`Tools::read_resource`].
|
||||
fn resource_templates(&self) -> Vec<Value> {
|
||||
Vec::new()
|
||||
}
|
||||
/// The contents of resource `uri`: `None` when there is no such resource, `Err` when it could
|
||||
/// not be read now.
|
||||
fn read_resource(&mut self, uri: &str) -> Option<Result<Contents, String>> {
|
||||
let _ = uri;
|
||||
None
|
||||
}
|
||||
/// The prompts `prompts/list` serves (MCP `Prompt`: name, title, description, arguments).
|
||||
fn prompts(&self) -> Vec<Value> {
|
||||
Vec::new()
|
||||
}
|
||||
/// Prompt `name` filled in with `args` (MCP `GetPromptResult`: description, messages). The
|
||||
/// server calls it only for a listed prompt, with `args` checked against that listing: an
|
||||
/// object of strings with every required argument and no other. `Err` is an internal error.
|
||||
fn get_prompt(&mut self, name: &str, args: &Value) -> Result<Value, String> {
|
||||
let _ = args;
|
||||
Err(format!("prompt `{name}` cannot be filled in"))
|
||||
}
|
||||
/// What argument `argument` of `reference` could be, given what has been typed (`value`).
|
||||
/// Asked only of an app that serves prompts or resource templates; a prompt reference is
|
||||
/// always a listed prompt. The reply holds the first 100.
|
||||
fn complete(&mut self, reference: Reference<'_>, argument: &str, value: &str) -> Vec<String> {
|
||||
let _ = (reference, argument, value);
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Who the server says it is at `initialize`, and the longest request line it reads.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Info {
|
||||
pub name: &'static str,
|
||||
pub title: &'static str,
|
||||
pub version: &'static str,
|
||||
/// Longest accepted request line, in bytes; a longer one is refused in bounded space.
|
||||
pub max_line: usize,
|
||||
}
|
||||
|
||||
/// An MCP server over the app's [`Tools`].
|
||||
pub struct Server<T: Tools> {
|
||||
info: Info,
|
||||
tools: T,
|
||||
initialized: bool,
|
||||
}
|
||||
|
||||
fn error(id: Value, code: i64, message: impl Into<String>) -> Value {
|
||||
json!({"jsonrpc": "2.0", "id": id, "error": {"code": code, "message": message.into()}})
|
||||
}
|
||||
|
||||
/// One message read from a newline-delimited stream by [`read_message`].
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Incoming {
|
||||
/// A line to handle ([`Server::handle_line`]), without its newline.
|
||||
Line(String),
|
||||
/// A line refused unread (too long, not UTF-8): the error reply to send for it.
|
||||
Refused(String),
|
||||
}
|
||||
|
||||
/// Read the next line of newline-delimited JSON-RPC from `input`; `None` once it closes. A line
|
||||
/// longer than `max_line` is consumed in bounded space, so a client cannot exhaust memory by
|
||||
/// sending an arbitrarily long request without a newline, and an unterminated last line counts.
|
||||
pub fn read_message(input: &mut impl BufRead, max_line: usize) -> std::io::Result<Option<Incoming>> {
|
||||
let mut line = Vec::new();
|
||||
let mut oversized = false;
|
||||
let mut received = false;
|
||||
loop {
|
||||
let chunk = input.fill_buf()?;
|
||||
if chunk.is_empty() {
|
||||
break;
|
||||
}
|
||||
let count = chunk.iter().position(|&byte| byte == b'\n').map_or(chunk.len(), |i| i + 1);
|
||||
let newline = chunk[count - 1] == b'\n';
|
||||
let content_len = count - usize::from(newline);
|
||||
if !oversized {
|
||||
if line.len().saturating_add(content_len) > max_line {
|
||||
oversized = true;
|
||||
line.clear();
|
||||
} else {
|
||||
line.extend_from_slice(&chunk[..content_len]);
|
||||
}
|
||||
}
|
||||
input.consume(count);
|
||||
received = true;
|
||||
if newline {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !received {
|
||||
return Ok(None);
|
||||
}
|
||||
if oversized {
|
||||
return Ok(Some(Incoming::Refused(error(Value::Null, INVALID_REQUEST, "request too large").to_string())));
|
||||
}
|
||||
Ok(Some(match String::from_utf8(line) {
|
||||
Ok(line) => Incoming::Line(line),
|
||||
// Not UTF-8: answer with a parse error and keep serving.
|
||||
Err(_) => Incoming::Refused(error(Value::Null, PARSE_ERROR, "parse error: input is not UTF-8").to_string()),
|
||||
}))
|
||||
}
|
||||
|
||||
/// The JSON-RPC response to request `id`, as [`Server::handle`] sends it.
|
||||
pub fn response(id: Value, result: Value) -> Value {
|
||||
json!({"jsonrpc": "2.0", "id": id, "result": result})
|
||||
}
|
||||
|
||||
impl<T: Tools> Server<T> {
|
||||
pub fn new(info: Info, tools: T) -> Self {
|
||||
Self { info, tools, initialized: false }
|
||||
}
|
||||
pub fn tools(&mut self) -> &mut T {
|
||||
&mut self.tools
|
||||
}
|
||||
pub fn is_initialized(&self) -> bool {
|
||||
self.initialized
|
||||
}
|
||||
|
||||
/// Serve newline-delimited JSON-RPC until `input` closes. Only protocol messages go to
|
||||
/// `output`; diagnostics belong on stderr. Consume oversized lines in bounded space so a
|
||||
/// client cannot exhaust memory by sending an arbitrarily long request without a newline.
|
||||
pub fn serve(&mut self, mut input: impl BufRead, mut output: impl Write) -> std::io::Result<()> {
|
||||
while let Some(incoming) = read_message(&mut input, self.info.max_line)? {
|
||||
let reply = match incoming {
|
||||
Incoming::Line(line) => self.handle_line(&line),
|
||||
Incoming::Refused(reply) => Some(reply),
|
||||
};
|
||||
if let Some(reply) = reply {
|
||||
output.write_all(reply.as_bytes())?;
|
||||
output.write_all(b"\n")?;
|
||||
output.flush()?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// One line in, at most one line out (notifications get no reply).
|
||||
pub fn handle_line(&mut self, line: &str) -> Option<String> {
|
||||
// Check the raw input before trimming: a huge whitespace prefix is still a huge line.
|
||||
if line.len() > self.info.max_line {
|
||||
return Some(error(Value::Null, INVALID_REQUEST, "request too large").to_string());
|
||||
}
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let reply = match serde_json::from_str::<Value>(line) {
|
||||
Ok(Value::Array(batch)) if batch.is_empty() => Some(error(Value::Null, INVALID_REQUEST, "empty batch")),
|
||||
Ok(Value::Array(batch)) if batch.len() > MAX_BATCH => Some(error(Value::Null, INVALID_REQUEST, "batch too large")),
|
||||
Ok(Value::Array(batch)) => {
|
||||
let replies: Vec<Value> = batch.into_iter().filter_map(|m| self.handle(m)).collect();
|
||||
(!replies.is_empty()).then_some(Value::Array(replies))
|
||||
}
|
||||
Ok(msg) => self.handle(msg),
|
||||
Err(e) => Some(error(Value::Null, PARSE_ERROR, format!("parse error: {e}"))),
|
||||
};
|
||||
reply.map(|r| r.to_string())
|
||||
}
|
||||
|
||||
pub fn handle(&mut self, msg: Value) -> Option<Value> {
|
||||
let Value::Object(o) = &msg else { return Some(error(Value::Null, INVALID_REQUEST, "message must be an object")) };
|
||||
let id = o.get("id").cloned();
|
||||
let Some(method) = o.get("method").and_then(Value::as_str) else {
|
||||
// A response to a request of ours (we send none) or junk.
|
||||
if o.contains_key("result") || o.contains_key("error") {
|
||||
return None;
|
||||
}
|
||||
return Some(error(id.unwrap_or(Value::Null), INVALID_REQUEST, "missing `method`"));
|
||||
};
|
||||
let params = o.get("params").cloned().unwrap_or(Value::Null);
|
||||
let Some(id) = id else {
|
||||
if method == "notifications/initialized" {
|
||||
self.initialized = true;
|
||||
}
|
||||
return None;
|
||||
};
|
||||
Some(match self.request(method, ¶ms) {
|
||||
Ok(r) => response(id, r),
|
||||
Err((code, m)) => error(id, code, m),
|
||||
})
|
||||
}
|
||||
|
||||
fn request(&mut self, method: &str, params: &Value) -> Result<Value, (i64, String)> {
|
||||
match method {
|
||||
"initialize" => {
|
||||
let asked = params.get("protocolVersion").and_then(Value::as_str).unwrap_or(PROTOCOL_VERSION);
|
||||
let version = if SUPPORTED_VERSIONS.contains(&asked) { asked } else { PROTOCOL_VERSION };
|
||||
let Info { name, title, version: server_version, .. } = self.info;
|
||||
Ok(json!({
|
||||
"protocolVersion": version,
|
||||
"capabilities": self.capabilities(),
|
||||
"serverInfo": {"name": name, "title": title, "version": server_version},
|
||||
"instructions": self.tools.instructions(),
|
||||
}))
|
||||
}
|
||||
"ping" => Ok(json!({})),
|
||||
"tools/list" => Ok(json!({"tools": self.tools.list()})),
|
||||
"tools/call" => {
|
||||
let name = params.get("name").and_then(Value::as_str).ok_or((INVALID_PARAMS, "missing tool `name`".to_string()))?;
|
||||
let args = match params.get("arguments") {
|
||||
None | Some(Value::Null) => json!({}),
|
||||
Some(a) => a.clone(),
|
||||
};
|
||||
check_arguments(&self.tools.list(), name, &args).map_err(|e| (INVALID_PARAMS, e))?;
|
||||
Ok(self.tools.call(name, &args))
|
||||
}
|
||||
"resources/list" => Ok(json!({"resources": self.tools.resources()})),
|
||||
"resources/templates/list" => Ok(json!({"resourceTemplates": self.tools.resource_templates()})),
|
||||
"prompts/list" => Ok(json!({"prompts": self.tools.prompts()})),
|
||||
"resources/read" => {
|
||||
let uri = params.get("uri").and_then(Value::as_str).unwrap_or("");
|
||||
let contents = self
|
||||
.tools
|
||||
.read_resource(uri)
|
||||
.ok_or_else(|| (RESOURCE_NOT_FOUND, format!("unknown resource `{uri}`")))?
|
||||
.map_err(|e| (INTERNAL_ERROR, e))?;
|
||||
Ok(json!({"contents": [contents.block(uri)]}))
|
||||
}
|
||||
"prompts/get" if !self.tools.prompts().is_empty() => {
|
||||
let name = params.get("name").and_then(Value::as_str).ok_or((INVALID_PARAMS, "missing prompt `name`".to_string()))?;
|
||||
let args = match params.get("arguments") {
|
||||
None | Some(Value::Null) => json!({}),
|
||||
Some(a) => a.clone(),
|
||||
};
|
||||
check_prompt_arguments(&self.tools.prompts(), name, &args).map_err(|e| (INVALID_PARAMS, e))?;
|
||||
self.tools.get_prompt(name, &args).map_err(|e| (INTERNAL_ERROR, e))
|
||||
}
|
||||
"completion/complete" if self.completes() => {
|
||||
let reference = params.get("ref").unwrap_or(&Value::Null);
|
||||
let reference = match (reference.get("type").and_then(Value::as_str), reference.get("name"), reference.get("uri")) {
|
||||
(Some("ref/prompt"), Some(Value::String(name)), _) => {
|
||||
if !self.tools.prompts().iter().any(|p| p["name"] == name.as_str()) {
|
||||
return Err((INVALID_PARAMS, format!("unknown prompt `{name}`")));
|
||||
}
|
||||
Reference::Prompt(name)
|
||||
}
|
||||
(Some("ref/resource"), _, Some(Value::String(uri))) => Reference::Resource(uri),
|
||||
_ => {
|
||||
return Err((
|
||||
INVALID_PARAMS,
|
||||
"`ref` must be a prompt's {type: ref/prompt, name} or a resource's {type: ref/resource, uri}".into(),
|
||||
));
|
||||
}
|
||||
};
|
||||
let argument = params.get("argument");
|
||||
let (Some(argument), Some(value)) =
|
||||
(argument.and_then(|a| a.get("name")).and_then(Value::as_str), argument.and_then(|a| a.get("value")).and_then(Value::as_str))
|
||||
else {
|
||||
return Err((INVALID_PARAMS, "`argument` must have a `name` and a `value`, both strings".into()));
|
||||
};
|
||||
let mut values = self.tools.complete(reference, argument, value);
|
||||
let total = values.len();
|
||||
values.truncate(MAX_COMPLETIONS);
|
||||
Ok(json!({"completion": {"values": values, "total": total, "hasMore": total > MAX_COMPLETIONS}}))
|
||||
}
|
||||
_ => Err((METHOD_NOT_FOUND, format!("method `{method}` not found"))),
|
||||
}
|
||||
}
|
||||
|
||||
/// What the server tells a client at `initialize` that it serves: the tools always, and the
|
||||
/// rest only when the app has some.
|
||||
fn capabilities(&self) -> Value {
|
||||
let mut capabilities = json!({"tools": {"listChanged": false}});
|
||||
if !self.tools.resources().is_empty() || !self.tools.resource_templates().is_empty() {
|
||||
capabilities["resources"] = json!({"listChanged": false});
|
||||
}
|
||||
if !self.tools.prompts().is_empty() {
|
||||
capabilities["prompts"] = json!({"listChanged": false});
|
||||
}
|
||||
if self.completes() {
|
||||
capabilities["completions"] = json!({});
|
||||
}
|
||||
capabilities
|
||||
}
|
||||
|
||||
/// Whether the app has arguments to complete: those of its prompts and resource templates.
|
||||
fn completes(&self) -> bool {
|
||||
!self.tools.prompts().is_empty() || !self.tools.resource_templates().is_empty()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,368 @@
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::{
|
||||
Contents, Effect, Info, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, check_arguments, object, string, text_result, tool,
|
||||
};
|
||||
|
||||
/// One app's tools: `echo` and `level`, nothing behind them (no engine, no design). `log` records
|
||||
/// every listing and call in order.
|
||||
#[derive(Default)]
|
||||
struct Echo {
|
||||
calls: Vec<Value>,
|
||||
log: Vec<String>,
|
||||
}
|
||||
|
||||
impl Tools for Echo {
|
||||
fn instructions(&self) -> String {
|
||||
"Echo test tools.".into()
|
||||
}
|
||||
fn list(&mut self) -> Vec<Value> {
|
||||
self.log.push("list".into());
|
||||
vec![
|
||||
tool("echo", "Echo", "Say `text` back.", object(json!({"text": string("What to say")}), &["text"]), Effect::ReadOnly),
|
||||
tool("level", "Level", "Set the level.", object(json!({"level": {"type": "string", "enum": ["low", "high"]}}), &[]), Effect::Edits),
|
||||
]
|
||||
}
|
||||
fn call(&mut self, name: &str, args: &Value) -> Value {
|
||||
self.log.push(format!("call {name}"));
|
||||
self.calls.push(json!({"name": name, "args": args}));
|
||||
text_result(args["text"].as_str().unwrap_or_default(), false)
|
||||
}
|
||||
}
|
||||
|
||||
fn info() -> Info {
|
||||
Info { name: "echo", title: "Echo", version: "1.0.0", max_line: 1 << 16 }
|
||||
}
|
||||
|
||||
fn serve(lines: &[Value]) -> (Server<Echo>, Vec<Value>) {
|
||||
serve_with(Echo::default(), lines)
|
||||
}
|
||||
|
||||
fn serve_with<T: Tools>(tools: T, lines: &[Value]) -> (Server<T>, Vec<Value>) {
|
||||
let input: String = lines.iter().map(|l| format!("{l}\n")).collect();
|
||||
let mut output = Vec::new();
|
||||
let mut s = Server::new(info(), tools);
|
||||
s.serve(std::io::Cursor::new(input), &mut output).unwrap();
|
||||
let replies = String::from_utf8(output).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect();
|
||||
(s, replies)
|
||||
}
|
||||
|
||||
/// A registered tool is listed and called over the stdio transport, with no engine behind it.
|
||||
#[test]
|
||||
fn a_registered_tool_is_served_over_stdio() {
|
||||
let (mut s, replies) = serve(&[
|
||||
json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": "2025-03-26"}}),
|
||||
json!({"jsonrpc": "2.0", "method": "notifications/initialized"}),
|
||||
json!({"jsonrpc": "2.0", "id": 2, "method": "tools/list"}),
|
||||
json!({"jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": {"name": "echo", "arguments": {"text": "hi"}}}),
|
||||
]);
|
||||
assert_eq!(replies.len(), 3, "the notification gets no reply: {replies:?}");
|
||||
let init = &replies[0]["result"];
|
||||
assert_eq!(init["protocolVersion"], "2025-03-26");
|
||||
assert_eq!(init["serverInfo"], json!({"name": "echo", "title": "Echo", "version": "1.0.0"}));
|
||||
assert_eq!(init["instructions"], "Echo test tools.");
|
||||
assert!(s.is_initialized());
|
||||
assert_eq!(replies[1]["result"]["tools"][0]["name"], "echo");
|
||||
assert_eq!(replies[1]["result"]["tools"][0]["annotations"]["readOnlyHint"], true);
|
||||
assert_eq!(replies[2], json!({"jsonrpc": "2.0", "id": 3, "result": {"content": [{"type": "text", "text": "hi"}]}}));
|
||||
assert_eq!(s.tools().calls, vec![json!({"name": "echo", "args": {"text": "hi"}})]);
|
||||
}
|
||||
|
||||
/// Arguments are checked against the tool's schema before the tool runs; unknown tools and
|
||||
/// methods are JSON-RPC errors.
|
||||
#[test]
|
||||
fn arguments_break_the_schema_before_the_tool_runs() {
|
||||
let (mut s, replies) = serve(&[
|
||||
json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "echo", "arguments": {}}}),
|
||||
json!({"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "echo", "arguments": {"text": 3}}}),
|
||||
json!({"jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": {"name": "echo", "arguments": {"text": "a", "loud": true}}}),
|
||||
json!({"jsonrpc": "2.0", "id": 4, "method": "tools/call", "params": {"name": "shout", "arguments": {}}}),
|
||||
json!({"jsonrpc": "2.0", "id": 5, "method": "nope"}),
|
||||
json!({"jsonrpc": "2.0", "id": 6, "method": "resources/read", "params": {"uri": "x://y"}}),
|
||||
]);
|
||||
let codes: Vec<&Value> = replies.iter().map(|r| &r["error"]["code"]).collect();
|
||||
assert_eq!(codes, [&json!(-32602), &json!(-32602), &json!(-32602), &json!(-32602), &json!(-32601), &json!(-32002)], "{replies:?}");
|
||||
assert_eq!(replies[0]["error"]["message"], "echo: missing required argument `text`");
|
||||
assert_eq!(replies[2]["error"]["message"], "echo: unknown argument `loud`; its arguments are: text");
|
||||
assert!(s.tools().calls.is_empty());
|
||||
let defs = s.tools().list();
|
||||
assert!(check_arguments(&defs, "echo", &json!({"text": "a"})).is_ok());
|
||||
}
|
||||
|
||||
/// Hostile framing: oversized lines, bad UTF-8 and bad JSON are answered and the server goes on.
|
||||
#[test]
|
||||
fn hostile_lines_are_answered_and_serving_goes_on() {
|
||||
let mut input = vec![b'x'; (1 << 16) + 10];
|
||||
input.push(b'\n');
|
||||
input.extend_from_slice(b"\xff\xfe\n{not json\n[]\n");
|
||||
input.extend_from_slice(json!({"jsonrpc": "2.0", "id": 9, "method": "ping"}).to_string().as_bytes());
|
||||
let mut output = Vec::new();
|
||||
Server::new(info(), Echo::default()).serve(std::io::Cursor::new(input), &mut output).unwrap();
|
||||
let replies: Vec<Value> = String::from_utf8(output).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect();
|
||||
let codes: Vec<&Value> = replies.iter().map(|r| r.get("error").map_or(&r["result"], |e| &e["code"])).collect();
|
||||
assert_eq!(codes, [&json!(-32600), &json!(-32700), &json!(-32700), &json!(-32600), &json!({})], "{replies:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn base64_matches_rfc4648() {
|
||||
for (i, o) in [("", ""), ("f", "Zg=="), ("fo", "Zm8="), ("foo", "Zm9v"), ("foob", "Zm9vYg=="), ("fooba", "Zm9vYmE="), ("foobar", "Zm9vYmFy")] {
|
||||
assert_eq!(crate::base64(i.as_bytes()), o);
|
||||
}
|
||||
let r = crate::image_result(b"foo", "image/png", "info");
|
||||
assert_eq!(r, json!({"content": [{"type": "image", "data": "Zm9v", "mimeType": "image/png"}, {"type": "text", "text": "info"}]}));
|
||||
}
|
||||
|
||||
fn ping(id: usize) -> Value {
|
||||
json!({"jsonrpc": "2.0", "id": id, "method": "ping"})
|
||||
}
|
||||
|
||||
fn notification() -> Value {
|
||||
json!({"jsonrpc": "2.0", "method": "notifications/progress"})
|
||||
}
|
||||
|
||||
fn reply<T: Tools>(s: &mut Server<T>, msg: &Value) -> Option<Value> {
|
||||
s.handle_line(&msg.to_string()).map(|r| serde_json::from_str(&r).unwrap())
|
||||
}
|
||||
|
||||
/// A batch answers each request in it and none of its notifications; a batch of notifications
|
||||
/// only gets no reply at all; a batch holds at most 1000 messages.
|
||||
#[test]
|
||||
fn batches_answer_their_requests_up_to_the_limit() {
|
||||
let mut s = Server::new(info(), Echo::default());
|
||||
let mixed =
|
||||
json!([ping(1), notification(), {"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "echo", "arguments": {"text": "b"}}}]);
|
||||
let replies = reply(&mut s, &mixed).unwrap();
|
||||
assert_eq!(
|
||||
replies,
|
||||
json!([{"jsonrpc": "2.0", "id": 1, "result": {}}, {"jsonrpc": "2.0", "id": 2, "result": {"content": [{"type": "text", "text": "b"}]}}])
|
||||
);
|
||||
assert_eq!(reply(&mut s, &json!([notification(), notification()])), None);
|
||||
let full: Vec<Value> = (0..1000).map(ping).collect();
|
||||
assert_eq!(reply(&mut s, &Value::Array(full)).unwrap().as_array().map(Vec::len), Some(1000));
|
||||
let over: Vec<Value> = (0..1001).map(ping).collect();
|
||||
let r = reply(&mut s, &Value::Array(over)).unwrap();
|
||||
assert_eq!(r, json!({"jsonrpc": "2.0", "id": null, "error": {"code": -32600, "message": "batch too large"}}));
|
||||
}
|
||||
|
||||
/// We send no requests, so a response is never answered, alone or in a batch; a message with
|
||||
/// neither a method nor a result, or one that is not an object, is refused; a blank line is skipped.
|
||||
#[test]
|
||||
fn a_response_we_never_asked_for_is_ignored() {
|
||||
let mut s = Server::new(info(), Echo::default());
|
||||
assert_eq!(reply(&mut s, &json!({"jsonrpc": "2.0", "id": 1, "result": {}})), None);
|
||||
assert_eq!(reply(&mut s, &json!({"jsonrpc": "2.0", "id": 2, "error": {"code": -1, "message": "no"}})), None);
|
||||
assert_eq!(
|
||||
reply(&mut s, &json!([{"jsonrpc": "2.0", "id": 3, "result": {}}, ping(4)])).unwrap(),
|
||||
json!([{"jsonrpc": "2.0", "id": 4, "result": {}}])
|
||||
);
|
||||
let r = reply(&mut s, &json!({"jsonrpc": "2.0", "id": 5})).unwrap();
|
||||
assert_eq!(r, json!({"jsonrpc": "2.0", "id": 5, "error": {"code": -32600, "message": "missing `method`"}}));
|
||||
let r = reply(&mut s, &json!(42)).unwrap();
|
||||
assert_eq!(r, json!({"jsonrpc": "2.0", "id": null, "error": {"code": -32600, "message": "message must be an object"}}));
|
||||
assert_eq!(s.handle_line(" "), None);
|
||||
assert!(s.tools().log.is_empty());
|
||||
}
|
||||
|
||||
/// A version we speak is echoed; one we don't, or none, gets ours.
|
||||
#[test]
|
||||
fn an_unknown_protocol_version_gets_ours() {
|
||||
let mut s = Server::new(info(), Echo::default());
|
||||
let version = |s: &mut Server<Echo>, params: Value| {
|
||||
reply(s, &json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": params})).unwrap()["result"]["protocolVersion"].clone()
|
||||
};
|
||||
for v in SUPPORTED_VERSIONS {
|
||||
assert_eq!(version(&mut s, json!({"protocolVersion": v})), json!(v));
|
||||
}
|
||||
assert_eq!(version(&mut s, json!({"protocolVersion": "1999-01-01"})), json!(PROTOCOL_VERSION));
|
||||
assert_eq!(version(&mut s, json!({"protocolVersion": 7})), json!(PROTOCOL_VERSION));
|
||||
assert_eq!(version(&mut s, json!({})), json!(PROTOCOL_VERSION));
|
||||
assert!(!s.is_initialized(), "initialized only by the client's notification");
|
||||
}
|
||||
|
||||
/// `arguments: null` is no arguments; a value outside a property's `enum` is refused before the
|
||||
/// tool runs.
|
||||
#[test]
|
||||
fn null_arguments_are_none_and_enums_are_checked() {
|
||||
let mut s = Server::new(info(), Echo::default());
|
||||
let call = |s: &mut Server<Echo>, id: u64, args: Value| {
|
||||
reply(s, &json!({"jsonrpc": "2.0", "id": id, "method": "tools/call", "params": {"name": "level", "arguments": args}})).unwrap()
|
||||
};
|
||||
assert!(call(&mut s, 1, Value::Null).get("result").is_some());
|
||||
let none = reply(&mut s, &json!({"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "level"}})).unwrap();
|
||||
assert!(none.get("result").is_some(), "{none}");
|
||||
assert!(call(&mut s, 3, json!({"level": "high"})).get("result").is_some());
|
||||
let r = call(&mut s, 4, json!({"level": "loud"}));
|
||||
assert_eq!(r["error"], json!({"code": -32602, "message": "level: `level` must be one of [\"low\",\"high\"]"}));
|
||||
let r = call(&mut s, 5, json!([]));
|
||||
assert_eq!(r["error"], json!({"code": -32602, "message": "level: arguments must be an object"}));
|
||||
let args: Vec<&Value> = s.tools().calls.iter().map(|c| &c["args"]).collect();
|
||||
assert_eq!(args, [&json!({}), &json!({}), &json!({"level": "high"})]);
|
||||
}
|
||||
|
||||
/// An app that serves no resources, templates or prompts lists none of each, says it serves only
|
||||
/// tools, and has no prompt to fill in or argument to complete.
|
||||
#[test]
|
||||
fn the_default_lists_are_empty() {
|
||||
let mut s = Server::new(info(), Echo::default());
|
||||
let result = |s: &mut Server<Echo>, method: &str| reply(s, &json!({"jsonrpc": "2.0", "id": 1, "method": method})).unwrap()["result"].clone();
|
||||
assert_eq!(result(&mut s, "resources/list"), json!({"resources": []}));
|
||||
assert_eq!(result(&mut s, "resources/templates/list"), json!({"resourceTemplates": []}));
|
||||
assert_eq!(result(&mut s, "prompts/list"), json!({"prompts": []}));
|
||||
assert_eq!(result(&mut s, "initialize")["capabilities"], json!({"tools": {"listChanged": false}}));
|
||||
for method in ["prompts/get", "completion/complete"] {
|
||||
let r = reply(&mut s, &json!({"jsonrpc": "2.0", "id": 2, "method": method, "params": {"name": "describe"}})).unwrap();
|
||||
assert_eq!(r["error"]["code"], -32601, "{method}: {r}");
|
||||
}
|
||||
}
|
||||
|
||||
/// An app that serves more than tools: a prompt with a required and an optional argument, a
|
||||
/// resource template, a Markdown resource and a PNG, and suggestions for any argument. `asked`
|
||||
/// records every prompt filled in and every completion asked for.
|
||||
#[derive(Default)]
|
||||
struct Library {
|
||||
asked: Vec<String>,
|
||||
}
|
||||
|
||||
impl Tools for Library {
|
||||
fn instructions(&self) -> String {
|
||||
"Library test.".into()
|
||||
}
|
||||
fn list(&mut self) -> Vec<Value> {
|
||||
Vec::new()
|
||||
}
|
||||
fn call(&mut self, name: &str, _: &Value) -> Value {
|
||||
text_result(format!("no tool `{name}`"), true)
|
||||
}
|
||||
fn resources(&self) -> Vec<Value> {
|
||||
vec![
|
||||
json!({"uri": "lib://readme", "name": "Readme", "mimeType": "text/markdown"}),
|
||||
json!({"uri": "lib://logo", "name": "Logo", "mimeType": "image/png"}),
|
||||
]
|
||||
}
|
||||
fn resource_templates(&self) -> Vec<Value> {
|
||||
vec![json!({"uriTemplate": "lib://parts/{name}", "name": "Part", "mimeType": "text/markdown"})]
|
||||
}
|
||||
fn read_resource(&mut self, uri: &str) -> Option<Result<Contents, String>> {
|
||||
let markdown = |text: String| Contents::Text { mime_type: "text/markdown".into(), text };
|
||||
Some(Ok(match uri {
|
||||
"lib://readme" => markdown("# Parts\n\nA \"quoted\" line.".into()),
|
||||
"lib://logo" => Contents::Blob { mime_type: "image/png".into(), data: b"foo".to_vec() },
|
||||
"lib://parts/broken" => return Some(Err("the part is being rebuilt".into())),
|
||||
_ => markdown(format!("# {}", uri.strip_prefix("lib://parts/")?)),
|
||||
}))
|
||||
}
|
||||
fn prompts(&self) -> Vec<Value> {
|
||||
vec![json!({"name": "describe", "title": "Describe a part", "arguments": [{"name": "part", "required": true}, {"name": "tone"}]})]
|
||||
}
|
||||
fn get_prompt(&mut self, name: &str, args: &Value) -> Result<Value, String> {
|
||||
self.asked.push(format!("{name} {args}"));
|
||||
let text = format!("Describe {} in a {} tone.", args["part"].as_str().unwrap_or_default(), args["tone"].as_str().unwrap_or("plain"));
|
||||
Ok(json!({"messages": [{"role": "user", "content": {"type": "text", "text": text}}]}))
|
||||
}
|
||||
fn complete(&mut self, reference: Reference<'_>, argument: &str, value: &str) -> Vec<String> {
|
||||
self.asked.push(format!("{reference:?} {argument}={value}"));
|
||||
(0..150).map(|i| format!("{value}{i}")).collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// The shared crate serves an app's prompt, resource template and Markdown and binary resources,
|
||||
/// and tells the client at `initialize` that it does (#13336).
|
||||
#[test]
|
||||
fn an_app_serves_prompts_templates_and_markdown_or_binary_resources() {
|
||||
let request = |id: u64, method: &str, params: Value| json!({"jsonrpc": "2.0", "id": id, "method": method, "params": params});
|
||||
let (mut s, replies) = serve_with(
|
||||
Library::default(),
|
||||
&[
|
||||
request(1, "initialize", json!({"protocolVersion": PROTOCOL_VERSION})),
|
||||
request(2, "prompts/list", json!({})),
|
||||
request(3, "prompts/get", json!({"name": "describe", "arguments": {"part": "bracket"}})),
|
||||
request(4, "resources/templates/list", json!({})),
|
||||
request(5, "resources/read", json!({"uri": "lib://readme"})),
|
||||
request(6, "resources/read", json!({"uri": "lib://logo"})),
|
||||
request(7, "resources/read", json!({"uri": "lib://parts/bracket"})),
|
||||
request(8, "resources/read", json!({"uri": "lib://parts/broken"})),
|
||||
request(9, "resources/read", json!({"uri": "lib://nope"})),
|
||||
],
|
||||
);
|
||||
let result = |i: usize| &replies[i]["result"];
|
||||
assert_eq!(
|
||||
result(0)["capabilities"],
|
||||
json!({"tools": {"listChanged": false}, "resources": {"listChanged": false}, "prompts": {"listChanged": false}, "completions": {}})
|
||||
);
|
||||
assert_eq!(result(1)["prompts"][0]["name"], "describe");
|
||||
assert_eq!(result(2), &json!({"messages": [{"role": "user", "content": {"type": "text", "text": "Describe bracket in a plain tone."}}]}));
|
||||
assert_eq!(result(3), &json!({"resourceTemplates": [{"uriTemplate": "lib://parts/{name}", "name": "Part", "mimeType": "text/markdown"}]}));
|
||||
// The Markdown goes out as it is, not as a JSON string of it.
|
||||
assert_eq!(result(4), &json!({"contents": [{"uri": "lib://readme", "mimeType": "text/markdown", "text": "# Parts\n\nA \"quoted\" line."}]}));
|
||||
assert_eq!(result(5), &json!({"contents": [{"uri": "lib://logo", "mimeType": "image/png", "blob": "Zm9v"}]}));
|
||||
assert_eq!(result(6), &json!({"contents": [{"uri": "lib://parts/bracket", "mimeType": "text/markdown", "text": "# bracket"}]}));
|
||||
assert_eq!(replies[7]["error"], json!({"code": -32603, "message": "the part is being rebuilt"}));
|
||||
assert_eq!(replies[8]["error"]["code"], -32002);
|
||||
assert_eq!(s.tools().asked, [r#"describe {"part":"bracket"}"#]);
|
||||
}
|
||||
|
||||
/// A prompt is filled in only with the arguments its listing declares, each a string, the
|
||||
/// required ones present; anything else is invalid params and the app is never asked.
|
||||
#[test]
|
||||
fn prompt_arguments_are_checked_before_the_prompt_is_filled_in() {
|
||||
let mut s = Server::new(info(), Library::default());
|
||||
let get =
|
||||
|s: &mut Server<Library>, params: Value| reply(s, &json!({"jsonrpc": "2.0", "id": 1, "method": "prompts/get", "params": params})).unwrap();
|
||||
for (params, message) in [
|
||||
(json!({"arguments": {"part": "a"}}), "missing prompt `name`"),
|
||||
(json!({"name": "summarise"}), "unknown prompt `summarise`"),
|
||||
(json!({"name": "describe"}), "describe: missing required argument `part`"),
|
||||
(json!({"name": "describe", "arguments": ["a"]}), "describe: arguments must be an object"),
|
||||
(json!({"name": "describe", "arguments": {"part": 3}}), "describe: argument `part` must be a string"),
|
||||
(json!({"name": "describe", "arguments": {"part": null}}), "describe: argument `part` must be a string"),
|
||||
(json!({"name": "describe", "arguments": {"part": "a", "size": "big"}}), "describe: unknown argument `size`; its arguments are: part, tone"),
|
||||
] {
|
||||
assert_eq!(get(&mut s, params.clone())["error"], json!({"code": -32602, "message": message}), "{params}");
|
||||
}
|
||||
assert!(s.tools().asked.is_empty());
|
||||
let filled = get(&mut s, json!({"name": "describe", "arguments": {"part": "hinge", "tone": "dry"}}));
|
||||
assert_eq!(filled["result"]["messages"][0]["content"]["text"], "Describe hinge in a dry tone.");
|
||||
}
|
||||
|
||||
/// Completion asks the app about a listed prompt or a resource, and sends the first 100
|
||||
/// suggestions with how many there are; a bad reference or argument is invalid params.
|
||||
#[test]
|
||||
fn completion_suggests_up_to_a_hundred_values() {
|
||||
let mut s = Server::new(info(), Library::default());
|
||||
let complete = |s: &mut Server<Library>, params: Value| {
|
||||
reply(s, &json!({"jsonrpc": "2.0", "id": 1, "method": "completion/complete", "params": params})).unwrap()
|
||||
};
|
||||
let r = complete(&mut s, json!({"ref": {"type": "ref/prompt", "name": "describe"}, "argument": {"name": "part", "value": "br"}}));
|
||||
let completion = &r["result"]["completion"];
|
||||
assert_eq!(completion["values"].as_array().map(Vec::len), Some(100), "{r}");
|
||||
assert_eq!((&completion["values"][0], &completion["total"], &completion["hasMore"]), (&json!("br0"), &json!(150), &json!(true)));
|
||||
let r = complete(&mut s, json!({"ref": {"type": "ref/resource", "uri": "lib://parts/{name}"}, "argument": {"name": "name", "value": ""}}));
|
||||
assert_eq!(r["result"]["completion"]["values"][99], "99");
|
||||
assert_eq!(s.tools().asked, [r#"Prompt("describe") part=br"#, r#"Resource("lib://parts/{name}") name="#]);
|
||||
let bad_ref = "`ref` must be a prompt's {type: ref/prompt, name} or a resource's {type: ref/resource, uri}";
|
||||
let bad_argument = "`argument` must have a `name` and a `value`, both strings";
|
||||
for (params, message) in [
|
||||
(json!({"ref": {"type": "ref/prompt", "name": "summarise"}, "argument": {"name": "a", "value": ""}}), "unknown prompt `summarise`"),
|
||||
(json!({"ref": {"type": "ref/tool", "name": "x"}, "argument": {"name": "a", "value": ""}}), bad_ref),
|
||||
(json!({"ref": {"type": "ref/resource"}, "argument": {"name": "a", "value": ""}}), bad_ref),
|
||||
(json!({"argument": {"name": "a", "value": ""}}), bad_ref),
|
||||
(json!({"ref": {"type": "ref/prompt", "name": "describe"}, "argument": {"name": "part"}}), bad_argument),
|
||||
(json!({"ref": {"type": "ref/prompt", "name": "describe"}}), bad_argument),
|
||||
] {
|
||||
assert_eq!(complete(&mut s, params.clone())["error"], json!({"code": -32602, "message": message}), "{params}");
|
||||
}
|
||||
assert_eq!(s.tools().asked.len(), 2, "a refused completion never reaches the app");
|
||||
}
|
||||
|
||||
/// `Tools::call` runs right after the listing its arguments were checked against, every time,
|
||||
/// whatever was listed before it; a refused call is listed and never run.
|
||||
#[test]
|
||||
fn every_call_runs_right_after_its_listing() {
|
||||
let mut s = Server::new(info(), Echo::default());
|
||||
let call = |name: &str, args: Value| json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": name, "arguments": args}});
|
||||
reply(&mut s, &json!({"jsonrpc": "2.0", "id": 1, "method": "tools/list"}));
|
||||
reply(&mut s, &call("echo", json!({"text": "a"})));
|
||||
reply(&mut s, &call("echo", json!({})));
|
||||
reply(&mut s, &json!([call("level", json!({})), call("echo", json!({"text": "b"}))]));
|
||||
assert_eq!(s.tools().log, ["list", "list", "call echo", "list", "list", "call level", "list", "call echo"]);
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
//! Tolerant fields from AVC and HEVC configuration records.
|
||||
|
||||
/// NAL length prefix width from AVC configuration, defaulting to four on a short record.
|
||||
pub fn avcc_length_size(avcc: &[u8]) -> usize {
|
||||
avcc.get(4).map_or(4, |b| (b & 3) as usize + 1)
|
||||
}
|
||||
|
||||
/// NAL length prefix width and highest temporal ID from HEVC configuration.
|
||||
pub fn hvcc_length_size_and_tid(hvcc: &[u8]) -> (usize, Option<u8>) {
|
||||
let length_size = hvcc.get(21).map_or(4, |b| (b & 3) as usize + 1);
|
||||
let highest_tid = hvcc.get(21).map(|b| (b >> 3) & 7).filter(|&n| n > 0).map(|n| n - 1);
|
||||
(length_size, highest_tid)
|
||||
}
|
||||
|
||||
/// objectTypeIndication of the DecoderConfigDescriptor inside an `esds` box in `raw`.
|
||||
pub fn esds_object_type(raw: &[u8]) -> Option<u8> {
|
||||
let at = raw.windows(4).position(|w| w == b"esds")? + 8;
|
||||
let mut p = at;
|
||||
let size = |p: &mut usize| -> Option<usize> {
|
||||
let mut n = 0usize;
|
||||
for _ in 0..4 {
|
||||
let b = *raw.get(*p)?;
|
||||
*p += 1;
|
||||
n = (n << 7) | (b & 0x7F) as usize;
|
||||
if b & 0x80 == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Some(n)
|
||||
};
|
||||
if *raw.get(p)? != 0x03 {
|
||||
return None;
|
||||
}
|
||||
p += 1;
|
||||
size(&mut p)?;
|
||||
let flags = *raw.get(p + 2)?;
|
||||
p += 3;
|
||||
if flags & 0x80 != 0 {
|
||||
p += 2;
|
||||
}
|
||||
if flags & 0x40 != 0 {
|
||||
p += 1 + *raw.get(p)? as usize;
|
||||
}
|
||||
if flags & 0x20 != 0 {
|
||||
p += 2;
|
||||
}
|
||||
if *raw.get(p)? != 0x04 {
|
||||
return None;
|
||||
}
|
||||
p += 1;
|
||||
size(&mut p)?;
|
||||
raw.get(p).copied()
|
||||
}
|
||||
|
||||
/// E-AC-3 `ec-3` entry: the program-layout fields are the 24 bits starting at byte 2 of the
|
||||
/// descriptor, then `bsmod(5) | acmod(3) | lfeon(1)`. Matches FFmpeg's `dec3` demuxer.
|
||||
pub fn dec3_layout(payload: &[u8]) -> Option<(u32, u32)> {
|
||||
if payload.len() < 5 {
|
||||
return None;
|
||||
}
|
||||
let bits = u32::from(payload[2]) << 16 | u32::from(payload[3]) << 8 | u32::from(payload[4]);
|
||||
let acmod = ((bits >> 9) & 0x7) as u8;
|
||||
let lfeon = bits & 0x100 != 0;
|
||||
let channels = dac3_channels(acmod, lfeon)?;
|
||||
Some((0, channels))
|
||||
}
|
||||
|
||||
/// AC-3 `ac-3` entry: `dac3` begins with the syncframe bitstream information: sample rate code,
|
||||
/// bit stream id, bit stream mode, channel mode and LFE flag. The descriptor has no sample rate,
|
||||
/// so the caller keeps the sample-entry rate.
|
||||
pub fn dac3_layout(payload: &[u8]) -> Option<(u32, u32)> {
|
||||
if payload.len() < 3 {
|
||||
return None;
|
||||
}
|
||||
// The descriptor begins with the bitstream information: fscod(2), bsid(5), bsmod(3),
|
||||
// acmod(3) and lfeon(1), continuing in the second payload byte.
|
||||
let bits = u32::from_be_bytes([0, 0, payload[1], payload[2]]);
|
||||
let acmod = ((bits >> 11) & 0x7) as u8;
|
||||
let lfeon = bits & 0x400 != 0;
|
||||
let channels = dac3_channels(acmod, lfeon)?;
|
||||
Some((0, channels))
|
||||
}
|
||||
|
||||
/// Full-bandwidth channels per AC-3 channel mode (Table 5.8) plus the optional LFE channel.
|
||||
fn dac3_channels(acmod: u8, lfeon: bool) -> Option<u32> {
|
||||
let n = [2u32, 1, 2, 3, 3, 4, 4, 5].get(usize::from(acmod & 7))?.checked_add(u32::from(lfeon))?;
|
||||
(n > 0).then_some(n)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn audio_layout_and_elementary_descriptor_are_bounded() {
|
||||
assert_eq!(dac3_layout(&[0, 0, 0]), Some((0, 2)));
|
||||
assert_eq!(dec3_layout(&[0, 0, 0, 0, 0]), Some((0, 2)));
|
||||
assert_eq!(dac3_layout(&[0, 0]), None);
|
||||
assert_eq!(dec3_layout(&[0; 4]), None);
|
||||
let esds = [b'e', b's', b'd', b's', 0, 0, 0, 0, 0x03, 0x04, 0, 0, 0, 0x04, 0x01, 0x60];
|
||||
assert_eq!(esds_object_type(&esds), Some(0x60));
|
||||
for end in 0..esds.len() {
|
||||
assert_eq!(esds_object_type(&esds[..end]), None);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn short_records_and_temporal_layers() {
|
||||
assert_eq!(avcc_length_size(&[]), 4);
|
||||
assert_eq!(avcc_length_size(&[0, 0, 0, 0, 1]), 2);
|
||||
assert_eq!(hvcc_length_size_and_tid(&[]), (4, None));
|
||||
let mut h = [0; 22];
|
||||
h[21] = (3 << 3) | 2;
|
||||
assert_eq!(hvcc_length_size_and_tid(&h), (3, Some(2)));
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user