Merge next lint fixes before updating the reviewed SDK PR

This commit is contained in:
skylar taylor-barrick
2026-11-16 10:14:12 -06:00
203 changed files with 23241 additions and 1123 deletions
+1 -1
View File
@@ -23,5 +23,5 @@ jobs:
# The codec and format code behind features that are off by default. Without these the
# coverage build never compiles it, and new code there reads as untested.
cargo-features: >-
craft-video/dnx,craft-video/vp8,craft-video/prores,craft-video/h264,craft-video/hevc,craft-video/vp9,craft-video/av1,craft-video/threads,craft-audio/decode,craft-audio/encode,craft-audio/peaks,craft-audio/rayon,craft-audio/serde,craft-media/matroska,craft-media/ogg,craft-media/bitstream,craft-media/serde,craft-mesh/stl,craft-mesh/obj,craft-mesh/ply,craft-mesh/gltf,craft-metafile/write,craft-image-codecs/sgi
craft-image-codecs/avif,craft-image-codecs/avif-decode,craft-video/dnx,craft-video/vp8,craft-video/prores,craft-video/h264,craft-video/hevc,craft-video/vp9,craft-video/av1,craft-video/threads,craft-audio/decode,craft-audio/encode,craft-audio/peaks,craft-audio/rayon,craft-audio/serde,craft-media/matroska,craft-media/ogg,craft-media/bitstream,craft-media/serde,craft-mesh/stl,craft-mesh/obj,craft-mesh/ply,craft-mesh/gltf,craft-metafile/write,craft-image-codecs/sgi,craft-image-codecs/sunraster,craft-jsonrpc/async-lines
secrets: inherit
Generated
+85 -8
View File
@@ -1152,9 +1152,9 @@ dependencies = [
[[package]]
name = "cfg-expr"
version = "0.20.10"
version = "0.20.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ba9e9ec16c447027685b1f897b720e18e9a8afd00bd7332c483537e38086c9f"
checksum = "edd44d82755fb9210fb10d9daf8dd7e522da0efe1d5f263cffc5db07725663ff"
dependencies = [
"smallvec",
"target-lexicon",
@@ -1595,6 +1595,7 @@ dependencies = [
"craft-dwg",
"craft-dxf",
"craft-ecproj-format",
"craft-elf",
"craft-eps",
"craft-epub",
"craft-ews",
@@ -1611,6 +1612,7 @@ dependencies = [
"craft-font-manager",
"craft-font-store",
"craft-font-subset",
"craft-format",
"craft-gainmap",
"craft-gcode",
"craft-gif",
@@ -1637,11 +1639,13 @@ dependencies = [
"craft-latex",
"craft-layers",
"craft-lottie",
"craft-macho",
"craft-markdown",
"craft-materialx",
"craft-math",
"craft-mcmat",
"craft-mcp",
"craft-mcp-server",
"craft-media",
"craft-menu",
"craft-mesh",
@@ -1660,6 +1664,7 @@ dependencies = [
"craft-pcraft-format",
"craft-pdf 0.1.0",
"craft-pdn",
"craft-pe",
"craft-penpot",
"craft-pkcs7",
"craft-platform",
@@ -1687,11 +1692,13 @@ dependencies = [
"craft-silhouette-studio",
"craft-sketch",
"craft-smime",
"craft-sniff",
"craft-stencilcraft-document",
"craft-stencilcraft-format",
"craft-stream-codecs",
"craft-svg",
"craft-table",
"craft-tar",
"craft-tcraft",
"craft-text",
"craft-texture",
@@ -2164,6 +2171,15 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-elf"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-eps"
version = "0.1.0"
@@ -2383,6 +2399,16 @@ dependencies = [
"craft-fonts-build",
]
[[package]]
name = "craft-format"
version = "0.1.0"
dependencies = [
"craft-zip",
"proptest",
"serde_json",
"sha2 0.10.9",
]
[[package]]
name = "craft-gainmap"
version = "0.1.0"
@@ -2433,6 +2459,7 @@ name = "craft-heif"
version = "0.1.0"
dependencies = [
"heic-rs",
"proptest",
"rayon",
]
@@ -2669,6 +2696,15 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-macho"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-markdown"
version = "0.1.0"
@@ -2714,6 +2750,14 @@ dependencies = [
"serde_json",
]
[[package]]
name = "craft-mcp-server"
version = "0.1.0"
dependencies = [
"proptest",
"serde_json",
]
[[package]]
name = "craft-media"
version = "0.1.0"
@@ -2756,6 +2800,7 @@ version = "0.1.0"
dependencies = [
"craft-opc",
"craft-xml",
"craft-zip",
"miniz_oxide 0.9.1",
"serde_json",
]
@@ -2982,6 +3027,15 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-pe"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-penpot"
version = "0.1.0"
@@ -3165,6 +3219,7 @@ name = "craft-ps"
version = "0.1.0"
dependencies = [
"craft-eps",
"flate2",
"kurbo",
"miniz_oxide 0.8.9",
"pdf-writer",
@@ -3233,6 +3288,13 @@ dependencies = [
"zune-jpegxl",
]
[[package]]
name = "craft-render-parity"
version = "0.1.0"
dependencies = [
"craft-cms",
]
[[package]]
name = "craft-review"
version = "0.1.0"
@@ -3360,6 +3422,10 @@ dependencies = [
"zeroize",
]
[[package]]
name = "craft-sniff"
version = "0.1.0"
[[package]]
name = "craft-stencilcraft-document"
version = "0.1.0"
@@ -3432,6 +3498,13 @@ dependencies = [
"egui",
]
[[package]]
name = "craft-tar"
version = "0.1.0"
dependencies = [
"serde",
]
[[package]]
name = "craft-tcraft"
version = "0.1.0"
@@ -3455,6 +3528,7 @@ dependencies = [
"egui",
"hypher",
"icu_segmenter",
"kurbo",
"miniz_oxide 0.8.9",
"regex",
"regex-syntax",
@@ -3841,6 +3915,7 @@ dependencies = [
name = "craft-video"
version = "0.1.0"
dependencies = [
"craft-jpeg",
"craft-media",
"craft-media-testkit",
"rav1e",
@@ -5173,9 +5248,9 @@ dependencies = [
[[package]]
name = "fixedbitset"
version = "0.5.7"
version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
checksum = "5905ec755cc8386d7928353e4d356d2be3e89e81e5ca01fe630a1ca4d08dc175"
[[package]]
name = "flagset"
@@ -7224,6 +7299,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519"
dependencies = [
"cfg-if",
"rayon",
]
[[package]]
@@ -9269,6 +9345,7 @@ dependencies = [
"loop9",
"quick-error 2.0.1",
"rav1e",
"rayon",
"rgb",
]
@@ -9843,9 +9920,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f"
[[package]]
name = "rustls-webpki"
version = "0.103.15"
version = "0.103.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281"
dependencies = [
"aws-lc-rs",
"ring",
@@ -13433,9 +13510,9 @@ dependencies = [
[[package]]
name = "zip"
version = "9.1.0"
version = "9.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2886b00003a72c4ccee3f42cfc34908d33b2508a74d827dc2a4244706475c704"
checksum = "1156b39d142a34b999c2310a50455e0cb3760bf7efad0122f796d813bee26ccb"
dependencies = [
"crc32fast",
"flate2",
+8 -2
View File
@@ -17,6 +17,7 @@ craft-usd = { path = "crates/usd" }
craft-smime = { path = "crates/smime" }
craft-ber = { path = "crates/ber" }
craft-proto = { path = "crates/proto" }
craft-render-parity = { path = "crates/render-parity" }
acadrust = "=0.6.6"
log = "0.4"
# internal: add each crate here as `craft-<name> = { path = "crates/<name>" }`
@@ -28,6 +29,7 @@ craft-win32 = { path = "crates/win32" }
craft-ai-core = { path = "crates/ai-core" }
craft-anicraft-format = { path = "crates/anicraft-format" }
craft-pdf = { path = "crates/pdf", version = "=0.1.0" }
craft-pe = { path = "crates/pe" }
craft-ps = { path = "crates/ps", version = "=0.1.0" }
craft-pdn = { path = "crates/pdn" }
craft-plate-test = { path = "crates/plate-test" }
@@ -70,6 +72,7 @@ craft-designcraft-format = { path = "crates/designcraft-format" }
craft-dxf = { path = "crates/dxf" }
craft-drawingml = { path = "crates/drawingml" }
craft-eps = { path = "crates/eps" }
craft-elf = { path = "crates/elf" }
craft-file-formats = { path = "crates/file-formats" }
craft-file-import = { path = "crates/file-import" }
vectorcraft-doc = { git = "https://github.com/storytold/vectorcraft.git", rev = "6b3a6b8788ebcb420d69449199cbdf323888d27e" }
@@ -124,6 +127,7 @@ craft-layers = { path = "crates/layers" }
craft-audio = { path = "crates/audio" }
craft-latex = { path = "crates/latex" }
craft-markdown = { path = "crates/markdown" }
craft-macho = { path = "crates/macho" }
craft-math = { path = "crates/math" }
# No default features: craft-audio turns on the craft-media modules each of its features needs.
craft-media = { path = "crates/media", default-features = false }
@@ -204,6 +208,10 @@ craft-xcf = { path = "crates/xcf" }
craft-xml = { path = "crates/xml" }
craft-zip = { path = "crates/zip" }
craft-y4m = { path = "crates/y4m" }
craft-sniff = { path = "crates/sniff" }
craft-tar = { path = "crates/tar" }
craft-format = { path = "crates/format" }
craft-mcp-server = { path = "crates/mcp-server" }
craft-epub = { path = "crates/epub" }
craft-ews = { path = "crates/ews" }
craft-video = { path = "crates/video" }
@@ -230,13 +238,11 @@ chrono = { version = "0.4.45", default-features = false, features = ["std"] }
[workspace.lints.rust]
unsafe_code = "forbid"
unused_workspace_dependencies = "allow"
unexpected_cfgs = { level = "warn", check-cfg = ["cfg(craft_pdf_unicode_fixtures)"] }
[workspace.lints.clippy]
cognitive_complexity = "deny"
too_many_arguments = "deny"
unused_workspace_dependencies = "allow"
wildcard_imports = "deny"
dbg_macro = "warn"
todo = "warn"
+8
View File
@@ -61,6 +61,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
Clean room, from EffectCraft. Mapping onto an app's document stays in the app.
- [`craft-ai-core`](crates/ai-core/README.md): the data model and contract for AI in the Craft Apps (connections, models, tools, requests, jobs, consent, history, route spec). Pure, no I/O, wasm-safe. Contract in [docs/ai-shared-layer.md](docs/ai-shared-layer.md).
- [`craft-pe`](crates/pe/src/lib.rs): PE/COFF headers (`.exe`, `.dll`, `.sys`, `.efi`) from untrusted bytes, from Craft Anything: the COFF header, PE32/PE32+ optional header with its data directories bounded by the header's real size, the section table, RVA-to-file mapping and the decoded Rich header.
- [`craft-pdf`](crates/pdf/README.md): independently optional PDF parsing, interpretation, rasterization and guarded authoring. Apps retain layout, fonts, editing and export policy; writer-only consumers do not enable the shared reader.
- [`craft-pdn`](crates/pdn/README.md): Paint.NET documents (`.pdn`, PDN3), read: the layer list, blend modes and chunked gzip pixel blocks, with a data-only MS-NRBF (.NET binary serialization) reader. From PhotoCraft.
- [`craft-aaf`](crates/aaf/README.md): clean-room AAF (Edit Protocol, structured storage) and OMF Interchange 2.0 (Bento) readers and writers over one mob-style composition model with exact times, clip gain, markers, nested compositions and embedded PCM essence. From FilmCraft.
@@ -117,6 +118,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
EMU units and `a:xfrm`, colours with theme slots and colour transforms, preset shape outlines
(`a:prstGeom`), custom geometry with guide formulas (`a:custGeom`) and preset text warps
(`a:prstTxWarp`). From DeckCraft. Takes `craft-xml` trees; packages stay with each format.
- [`craft-elf`](crates/elf/src/lib.rs): ELF program headers and dynamic section entries from untrusted bytes, from Craft Anything: 32- or 64-bit in either byte order, bounded counts and overflow-checked offsets, virtual-address-to-file mapping through `PT_LOAD`, and segment, permission and dynamic tag names.
- [`craft-eps`](crates/eps/README.md): Encapsulated PostScript files as a container: the binary
preview header (read and write), the DSC header comments and bounding boxes (also `(atend)`),
the EPSI preview bitmap, ASCII85 data, and telling EPS, PostScript and Illustrator files
@@ -213,6 +215,9 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
Hasselblad 3FR/FFF, Minolta MRW, Epson ERF, Mamiya MEF, Kodak DCS, Pentax PEF, Samsung SRW, Canon CR2, Nikon NEF/NRW, Panasonic RW2, Fujifilm RAF, Olympus ORF (uncompressed), Sony ARW, Canon CR3 and Canon CRW (through
`craft-crw`) so far; demosaicing and everything after stay with the develop
pipeline.
- [`craft-render-parity`](crates/render-parity/README.md): test support for the shared develop
pipeline (#587): how far two renders of the same photo are apart (per-channel error, CIEDE2000,
the worst pixel) and whether that is within a stated tolerance.
- [`craft-tga`](crates/tga/src/lib.rs): Truevision Targa, from FilmCraft (reader) and PhotoCraft
(writer): true-colour, grayscale and colour-mapped pictures, raw or run-length encoded, any
origin, to RGBA8, and a run-length writer whose packets end with their scan line. Bounded on
@@ -270,6 +275,7 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
AC-3/E-AC-3, MP1/MP2, MP3 encoding), streaming and seeking, and packet decoding for any demuxer. See its
README for formats and features.
- [`craft-latex`](crates/latex/README.md): LaTeX documents read (the everyday subset) and written (a standalone article), to and from the craft-flow model.
- [`craft-macho`](crates/macho/src/lib.rs): Mach-O header pieces from untrusted bytes, from Craft Anything: universal (fat and fat64) slice tables checked against the file, CPU and load command names, the fixed size each load command needs before its fields are read, and the thread-state program counter.
- [`craft-markdown`](crates/markdown/README.md): Markdown read and written (a CommonMark subset with GitHub tables and strikethrough), to and from the craft-flow model.
- [`craft-math`](crates/math/README.md): equations as one tree (the Office Math object model) with its syntaxes, read and written: OMML, presentation MathML, LaTeX math and the UnicodeMath linear format.
- [`craft-media`](crates/media/README.md): bit-level I/O (`bitstream`), clean-room MP4/MOV
@@ -414,6 +420,8 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
text-format crates read into and write from. WordCraft's model without its editing code.
- [`craft-xlsx`](crates/xlsx/README.md): Excel workbooks (`.xlsx`, `.xlsm`, SpreadsheetML, transitional and strict) and binary workbooks (`.xlsb`) on `craft-opc`: the workbook part, shared strings, the styles tables and worksheet rows streamed one at a time, with limits on the package, the XML and the binary records.
- [`craft-ora`](crates/ora/README.md): OpenRaster documents (`.ora`): the ZIP archive and the `stack.xml` layer tree as written, read with bounded sizes and written. From PhotoCraft.
- [`craft-sniff`](crates/sniff/src/lib.rs): what a file is from its first bytes: one dependency-free, wasm-safe signature table naming the format, its media type, extension and family for images, audio, video, 3D models, archives, compressed streams, executables and databases. Identifies only; parsing stays with each format's reader.
- [`craft-tar`](crates/tar/src/lib.rs): TAR (v7, ustar, GNU long names and base-256 sizes, local and global pax overrides) and cpio (newc, crc, odc) member walkers for hostile input, from Craft Anything: list each member, its kind, mode, link target and where its data lies, without extracting. Bounded by the input and a member cap; an optional `serde` feature serializes listings.
- [`craft-y4m`](crates/y4m/src/lib.rs): YUV4MPEG2 (`.y4m`) reader, from FilmCraft: the stream
header, colour space layouts at 8 to 16 bits, a frame index built from the `FRAME` lines without
loading the file, and plane access. Bounded on hostile input.
+45 -2
View File
@@ -58,13 +58,56 @@ pub struct WriteOptions<'a> {
/// reported.
pub fn write(doc: &Document, opts: &WriteOptions<'_>, report: &mut Report) -> Result<Vec<u8>> {
let version = if opts.small_sectors { craft_cfb::Version::V3 } else { craft_cfb::Version::V4 };
write::write(doc, version, &opts.identity, report)
write::write(doc, version, &opts.identity, &ReferenceOptions::default(), report)
}
/// Optional interpretation choices for applications retaining older interchange behavior.
#[derive(Clone, Copy, Debug)]
pub struct ReadOptions {
/// Interpret a source clip's start using its target slot's edit rate.
pub source_slot_rate: bool,
/// Use the referenced mob name when the clip has no explicit name.
pub infer_clip_name: bool,
/// Decode percent escapes in relative network locators.
pub decode_relative_locator: bool,
/// Report selector and nested-scope flattening.
pub report_flattening: bool,
/// Accept truncated PCM WAVE essence summaries.
pub permissive_pcm: bool,
}
impl Default for ReadOptions {
fn default() -> Self {
Self { source_slot_rate: true, infer_clip_name: true, decode_relative_locator: true, report_flattening: true, permissive_pcm: false }
}
}
/// Read using explicitly selected interpretation choices.
pub fn read_with_options(bytes: &[u8], options: &ReadOptions, report: &mut Report) -> Result<Document> {
read::read(bytes, options, report)
}
/// Optional compatibility choices for writing source references and locators.
#[derive(Clone, Copy, Debug)]
pub struct ReferenceOptions {
/// Write source offsets in target-slot units.
pub source_slot_rate: bool,
/// Percent-encode relative locators.
pub encode_relative_locator: bool,
}
impl Default for ReferenceOptions {
fn default() -> Self {
Self { source_slot_rate: true, encode_relative_locator: true }
}
}
/// Write with explicitly selected reference and locator conventions.
pub fn write_with_references(doc: &Document, opts: &WriteOptions<'_>, references: &ReferenceOptions, report: &mut Report) -> Result<Vec<u8>> {
let version = if opts.small_sectors { craft_cfb::Version::V3 } else { craft_cfb::Version::V4 };
write::write(doc, version, &opts.identity, references, report)
}
/// Read an AAF file: its top-level compositions, the compositions nested in them and their
/// sources. What cannot be represented is skipped and named in `report`.
pub fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
read::read(bytes, report)
read::read(bytes, &ReadOptions::default(), report)
}
/// Whether `bytes` look like an AAF file (a compound file whose root has the AAF root class or a
+25 -7
View File
@@ -67,6 +67,7 @@ fn indirect_f64(d: &[u8]) -> Option<f64> {
struct R<'a> {
report: &'a mut Report,
options: super::ReadOptions,
mobs: HashMap<MobId, &'a Obj>,
essence: HashMap<MobId, &'a [u8]>,
doc: Document,
@@ -90,7 +91,7 @@ struct NestClip<'a> {
/// Compositions nested deeper than this are read as gaps.
const MAX_NESTING: usize = 16;
pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
pub(crate) fn read(bytes: &[u8], options: &super::ReadOptions, report: &mut Report) -> Result<Document> {
let root = store::read(bytes).map_err(|e| Error::Parse { format: "AAF", message: e })?;
let header = root.strong(pid::ROOT_HEADER).ok_or_else(|| Error::Parse { format: "AAF", message: "no header object".into() })?;
let content = header.strong(pid::CONTENT).ok_or_else(|| Error::Parse { format: "AAF", message: "no content storage".into() })?;
@@ -106,7 +107,8 @@ pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
essence.insert(id, d);
}
}
let mut r = R { report, mobs, essence, doc: Document::default(), sources: HashMap::new(), nests: HashMap::new(), open: Vec::new() };
let mut r =
R { report, options: *options, mobs, essence, doc: Document::default(), sources: HashMap::new(), nests: HashMap::new(), open: Vec::new() };
// Top-level compositions: those no other composition uses, among the ones tagged top-level
// when any are. (Premiere Pro tags a nested sequence's composition top-level too; it is still
// only a nested sequence.) Compositions that only use each other in a circle all count. The
@@ -331,7 +333,9 @@ impl R<'_> {
items
}
x if x == cls::SELECTOR => {
self.report.info("a selector was flattened to its selected input");
if self.options.report_flattening {
self.report.info("a selector was flattened to its selected input");
}
match c.strong(pid::SELECTED) {
Some(s) => {
let mut items = self.segment(s, rate, kind, depth);
@@ -342,7 +346,9 @@ impl R<'_> {
}
}
x if x == cls::NESTED_SCOPE => {
self.report.info("a nested scope was flattened to its final input");
if self.options.report_flattening {
self.report.info("a nested scope was flattened to its final input");
}
match c.objs(pid::NESTED_SLOTS).last() {
Some(s) => {
let mut items = self.segment(s, rate, kind, depth);
@@ -388,7 +394,12 @@ impl R<'_> {
let len = units_to_ticks(len_of(c), rate.0, rate.1);
let tagged_name = tagged_values(c, pid::COMPONENT_USER_COMMENTS).into_iter().find(|(k, _)| k == "Clip Name").map(|(_, v)| v);
let (Some(id), Some(slot)) = (c.mob_id(pid::SOURCE_ID), c.u32(pid::SOURCE_MOB_SLOT_ID)) else { return CItem::Filler(len) };
let name = tagged_name.or_else(|| self.mobs.get(&id).and_then(|mob| mob.string(pid::MOB_NAME))).unwrap_or_default();
let name = if self.options.infer_clip_name {
tagged_name.or_else(|| self.mobs.get(&id).and_then(|mob| mob.string(pid::MOB_NAME)))
} else {
tagged_name
}
.unwrap_or_default();
if id == [0; 32] {
return CItem::Filler(len);
}
@@ -401,6 +412,7 @@ impl R<'_> {
.and_then(|mob| mob.objs(pid::SLOTS).iter().find(|s| s.u32(pid::SLOT_ID) == Some(slot)))
.and_then(|s| rate_of(s, pid::EDIT_RATE))
.unwrap_or(rate);
let source_rate = if self.options.source_slot_rate { source_rate } else { rate };
let start = units_to_ticks(c.i64(pid::START_TIME).unwrap_or(0), source_rate.0, source_rate.1);
match self.resolve(id, slot, start, kind, master_key, 0) {
Some((source, t)) => CItem::Clip(CClip { len, source, start: t, gain: None, name }),
@@ -506,6 +518,7 @@ impl R<'_> {
.and_then(|m| m.objs(pid::SLOTS).iter().find(|s| s.u32(pid::SLOT_ID) == Some(next_slot)))
.and_then(|s| rate_of(s, pid::EDIT_RATE))
.unwrap_or(rate);
let source_rate = if self.options.source_slot_rate { source_rate } else { rate };
let inner = units_to_ticks(clip.i64(pid::START_TIME).unwrap_or(0), source_rate.0, source_rate.1);
if next == [0; 32] {
return None;
@@ -545,7 +558,11 @@ impl R<'_> {
DataKind::Sound => CKind::Sound,
_ => kind,
};
let path = d.objs(pid::LOCATOR).iter().find_map(|l| l.string(pid::URL_STRING)).map(|u| locator_path(&u));
let path = d
.objs(pid::LOCATOR)
.iter()
.find_map(|l| l.string(pid::URL_STRING))
.map(|u| if self.options.decode_relative_locator { locator_path(&u) } else { craft_timeline::path::file_url_to_path(&u) });
let file_rate = rate_of(d, pid::SAMPLE_RATE).unwrap_or(rate);
let length = d
.i64(pid::FILE_LENGTH)
@@ -604,7 +621,8 @@ impl R<'_> {
if let Some(data) = self.essence.get(&id) {
s.embedded = Some(data.to_vec());
// embedded essence may be a whole WAVE / AIFF file
if let Some((pcm, ch, sr, bits)) = crate::pcm::parse_wav(data) {
let pcm = if self.options.permissive_pcm { craft_audio::essence_pcm::parse_wav(data) } else { crate::pcm::parse_wav(data) };
if let Some((pcm, ch, sr, bits)) = pcm {
s.embedded = Some(pcm);
s.file_channels = ch as u32;
s.channels = if s.channel.is_some() { 1 } else { ch as u32 };
+29
View File
@@ -557,3 +557,32 @@ fn relative_media_locator_round_trips_escaped_characters() {
let got = read_clean(&bytes);
assert!(got.sources.iter().any(|s| s.path.as_deref() == Some("Audio Files/100%20 #1.wav")));
}
#[test]
fn compatibility_options_keep_relative_paths_and_parent_rate() {
let mut doc = crate::testdoc::sound_only(None);
doc.compositions[0].sample_rate = 44_100;
doc.sources[0].path = Some("Audio Files/100%20 #1.wav".into());
let mut report = Report::default();
let references = ReferenceOptions { source_slot_rate: false, encode_relative_locator: false };
let bytes = write_with_references(&doc, &WriteOptions { identity: IDENTITY, small_sectors: false }, &references, &mut report).unwrap();
let root = store::read(&bytes).unwrap();
let mut all = Vec::new();
walk(&root, &mut all);
assert!(
all.iter().filter(|o| o.class == cls::NETWORK_LOCATOR).any(|o| o.string(pid::URL_STRING).as_deref() == Some("Audio Files/100%20 #1.wav"))
);
let starts: Vec<i64> = all.iter().filter(|o| o.class == cls::SOURCE_CLIP).filter_map(|o| o.i64(pid::START_TIME)).collect();
assert!(starts.contains(&17_640), "parent rate start: {starts:?}");
let opts = ReadOptions {
source_slot_rate: false,
infer_clip_name: false,
decode_relative_locator: false,
report_flattening: false,
permissive_pcm: true,
};
let back = read_with_options(&bytes, &opts, &mut report).unwrap();
assert!(back.sources.iter().any(|s| s.path.as_deref() == Some("Audio Files/100%20 #1.wav")));
let clip = back.compositions[0].tracks[0].items.iter().find_map(|i| if let CItem::Clip(c) = i { Some(c) } else { None }).unwrap();
assert_eq!(clip.start, FrameRate::FPS_25.tick_of(10));
}
+14 -4
View File
@@ -75,6 +75,7 @@ fn definition(class: Auid, id: Auid, name: &str) -> Obj {
struct W<'a> {
report: &'a mut Report,
inexact: bool,
references: super::ReferenceOptions,
}
struct IdGen {
@@ -350,9 +351,15 @@ fn add_tape_mob(first: &Source, tape_id: [u8; 32], timecode: Option<(i64, FrameR
}
/// Serialise a composition document.
pub(crate) fn write(doc: &Document, version: craft_cfb::Version, identity: &Identity<'_>, report: &mut Report) -> Result<Vec<u8>> {
pub(crate) fn write(
doc: &Document,
version: craft_cfb::Version,
identity: &Identity<'_>,
references: &super::ReferenceOptions,
report: &mut Report,
) -> Result<Vec<u8>> {
let comp = doc.compositions.first().ok_or(Error::Empty)?;
let mut w = W { report, inexact: false };
let mut w = W { report, inexact: false, references: *references };
let mut ids = IdGen::for_document(doc, comp);
let MediaOutput { mut mobs, essence, master_ref } = write_media(doc, &mut w, &mut ids);
let mut used_ops: Vec<Auid> = Vec::new();
@@ -360,7 +367,8 @@ pub(crate) fn write(doc: &Document, version: craft_cfb::Version, identity: &Iden
// ---- nested compositions: a clip refers to one as it does to a master mob, through the
// composition's first track of the clip's kind (a broken-out channel: that channel's track)
let mut refs = Refs { mobs: master_ref, rates: doc.sources.iter().enumerate().map(|(i, s)| (i, source_rate(s))).collect() };
let rates = if references.source_slot_rate { doc.sources.iter().enumerate().map(|(i, s)| (i, source_rate(s))).collect() } else { HashMap::new() };
let mut refs = Refs { mobs: master_ref, rates };
let nested_ids: Vec<[u8; 32]> = doc.nested.iter().map(|_| ids.next()).collect();
for (si, s) in doc.sources.iter().enumerate() {
let Some((comp, id)) = s.nested.and_then(|n| Some((doc.nested.get(n)?, nested_ids.get(n)?))) else { continue };
@@ -795,8 +803,10 @@ fn descriptor(w: &mut W, s: &Source) -> Obj {
if let Some(p) = &s.path {
let url = if craft_timeline::path::is_absolute(p) {
craft_timeline::path::path_to_file_url(p, false)
} else {
} else if w.references.encode_relative_locator {
craft_timeline::path::encode_uri_path(p)
} else {
p.clone()
};
d.set(pid::LOCATOR, Value::StrongVec(vec![Obj::new(cls::NETWORK_LOCATOR).data_prop(pid::URL_STRING, utf16z(&url))]));
}
+4
View File
@@ -30,7 +30,11 @@ mod roundtrip_tests;
#[cfg(test)]
mod testdoc;
/// Diagnostic types used by readers and writers.
pub use craft_timeline::report;
pub use craft_timeline::report::Report;
/// Exact time types used by the composition model.
pub use craft_timeline::time;
/// Error from a reader or writer. Unsupported *features* never error; they go in the [`Report`].
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
+340 -10
View File
@@ -12,6 +12,10 @@ type Uid = [u8; 12];
const MAX_REFERENCE_VISITS: usize = 65_536;
/// A source-clip search stopped at its work limit.
#[derive(Clone, Copy)]
struct SearchLimit;
struct R<'a, 'b> {
c: &'b Container<'a>,
report: &'b mut Report,
@@ -21,15 +25,49 @@ struct R<'a, 'b> {
media: HashMap<Uid, Vec<u8>>,
doc: Document,
sources: HashMap<(Uid, u32, String), usize>,
/// Component visits left for [`R::segment`] over the whole document.
budget: usize,
/// Outcome of the source-clip search under each segment: deterministic, so it is made once.
clip_search: HashMap<u32, std::result::Result<Option<u32>, SearchLimit>>,
/// Visits and queued references left for source-clip searches over the whole document:
/// distinct roots over one shared graph would otherwise each search it in full.
search_budget: usize,
}
/// Components one document may expand into timeline items; a small file can reference the same
/// sequence from many places and would otherwise multiply without bound.
const MAX_COMPONENT_VISITS: usize = 1_000_000;
/// Reported when the shared component budget runs out.
const BUDGET_WARNING: &str = "OMF composition is too large to import completely (the remaining components are dropped)";
/// Most visited components, and most queued references, one source-clip search may spend.
const MAX_CLIP_SEARCH_VISITS: usize = 100_000;
const MAX_CLIP_SEARCH_PENDING: usize = 1_000_000;
/// Work all source-clip searches in one document may spend together.
const MAX_DOCUMENT_SEARCH_WORK: usize = 2_000_000;
pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
read_with_budget(bytes, report, MAX_COMPONENT_VISITS)
}
fn read_with_budget(bytes: &[u8], report: &mut Report, budget: usize) -> Result<Document> {
let perr = |m: String| Error::Parse { format: "OMF", message: m };
let c = Container::open(bytes).map_err(perr)?;
let class_of = |o: u32| c.get(o, "OMFI:OOBJ:ObjClass").and_then(|v| v.get(..4).map(|b| [b[0], b[1], b[2], b[3]]));
let head = c.objects.keys().copied().find(|&o| class_of(o) == Some(*b"HEAD")).ok_or_else(|| perr("no OMF header object".into()))?;
let be = c.get(head, "OMFI:HEAD:ByteOrder").is_none_or(|v| v.first() != Some(&0x49));
let mut r = R { c: &c, report, be, mobs: HashMap::new(), media: HashMap::new(), doc: Document::default(), sources: HashMap::new() };
let mut r = R {
c: &c,
report,
be,
mobs: HashMap::new(),
media: HashMap::new(),
doc: Document::default(),
sources: HashMap::new(),
budget,
clip_search: HashMap::new(),
search_budget: MAX_DOCUMENT_SEARCH_WORK,
};
let mob_list = r.refs(head, "OMFI:HEAD:Mobs");
let mob_list = if mob_list.is_empty() {
c.objects.keys().copied().filter(|&o| matches!(class_of(o).as_ref(), Some(b"CMOB" | b"MMOB" | b"SMOB"))).collect()
@@ -66,8 +104,14 @@ pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
cmobs.iter().copied().filter(|m| r.uid(*m, "OMFI:MOBJ:MobID").is_none_or(|id| !referenced.contains(&id))).collect()
};
for m in top {
// every composition spends from the budget too: a list of up to 65,535 repeated entries must not yield that many
if r.budget == 0 && !r.doc.compositions.is_empty() {
r.report.warn(BUDGET_WARNING);
break;
}
let comp = r.composition(m);
r.doc.compositions.push(comp);
r.budget = r.budget.saturating_sub(1);
}
if r.doc.compositions.is_empty() {
return Err(Error::Empty);
@@ -166,6 +210,13 @@ impl R<'_, '_> {
let slots = self.refs(m, "OMFI:MOBJ:Slots");
let mut na = 0;
for s in slots {
if self.budget == 0 {
self.report.warn(BUDGET_WARNING);
break;
}
// Every slot spends a unit, skipped ones too: a list of up to 65,535 repeated slots
// must not cost that much work in each of 65,535 repeated compositions.
self.budget -= 1;
let Some(seg) = self.r#ref(s, "OMFI:MSLT:Segment") else { continue };
let rate = self.rational(s, "OMFI:MSLT:EditRate").filter(|r| r.0 > 0 && r.1 > 0).unwrap_or((25, 1));
let kind = self.data_kind(seg);
@@ -213,13 +264,32 @@ impl R<'_, '_> {
fn segment(&mut self, seg: u32, rate: (i64, i64), depth: usize) -> Vec<CItem> {
let len = units_to_ticks(self.len(seg), rate.0, rate.1);
// Depth alone does not bound a graph that references the same component many times at
// every level, so each visit also spends from a per-document budget.
// A visit stopped by depth spends from it too.
if self.budget == 0 {
self.report.warn(BUDGET_WARNING);
return vec![CItem::Filler(len)];
}
self.budget -= 1;
if depth > 32 {
return vec![CItem::Filler(len)];
}
match &self.class(seg) {
b"SEQU" => {
let mut out = Vec::new();
let mut expanded = 0i64;
for c in self.refs(seg, "OMFI:SEQU:Components") {
if self.budget == 0 {
// the rest of the sequence keeps its duration as one gap, not a gap per reference
self.report.warn(BUDGET_WARNING);
let rest = self.len(seg).saturating_sub(expanded);
if rest > 0 {
out.push(CItem::Filler(units_to_ticks(rest, rate.0, rate.1)));
}
break;
}
expanded = expanded.saturating_add(self.len(c));
out.extend(self.segment(c, rate, depth + 1));
}
out
@@ -306,12 +376,66 @@ impl R<'_, '_> {
.find(|&s| self.r#ref(s, "OMFI:MSLT:TrackDesc").and_then(|t| self.u32(t, "OMFI:TRKD:TrackID")) == Some(track))
}
fn first_clip(&self, seg: u32) -> Option<u32> {
match &self.class(seg) {
b"SCLP" => Some(seg),
b"SEQU" => self.refs(seg, "OMFI:SEQU:Components").into_iter().find_map(|c| self.first_clip(c)),
_ => None,
/// First source clip under `seg`, depth-first; a sequence referencing itself (directly or
/// through a cycle) ends the search rather than the stack. `Err` when the search spent its
/// budget of visited components or queued references before finishing (the answer is unknown).
fn first_clip(&mut self, seg: u32) -> std::result::Result<Option<u32>, SearchLimit> {
if let Some(found) = self.clip_search.get(&seg) {
return *found;
}
let mut spent = 0;
let found = if self.search_budget == 0 {
Err(SearchLimit)
} else {
let visits = MAX_CLIP_SEARCH_VISITS.min(self.search_budget);
let pending = MAX_CLIP_SEARCH_PENDING.min(self.search_budget);
self.first_clip_within(seg, visits, pending, &mut spent)
};
self.search_budget = self.search_budget.saturating_sub(spent);
self.clip_search.insert(seg, found);
found
}
/// `spent` receives the visits plus queued references the search used.
fn first_clip_within(&self, seg: u32, max_visits: usize, max_pending: usize, spent: &mut usize) -> std::result::Result<Option<u32>, SearchLimit> {
let mut visited = HashSet::new();
let mut todo = vec![seg];
let mut queued = 1usize;
let found = Self::search_from(self, &mut visited, &mut todo, &mut queued, max_visits, max_pending);
*spent = visited.len().saturating_add(queued);
found
}
fn search_from(
&self,
visited: &mut HashSet<u32>,
todo: &mut Vec<u32>,
queued: &mut usize,
max_visits: usize,
max_pending: usize,
) -> std::result::Result<Option<u32>, SearchLimit> {
while let Some(seg) = todo.pop() {
if !visited.insert(seg) {
continue;
}
if visited.len() > max_visits {
return Err(SearchLimit);
}
match &self.class(seg) {
b"SCLP" => return Ok(Some(seg)),
b"SEQU" => {
let comps = self.refs(seg, "OMFI:SEQU:Components");
// sequences may share one component list, so queued references grow with expansions, not visits
*queued = queued.saturating_add(comps.len());
if *queued > max_pending {
return Err(SearchLimit);
}
todo.extend(comps.into_iter().rev());
}
_ => {}
}
}
Ok(None)
}
fn resolve(&mut self, id: Uid, track: u32, start: Tick, master: Option<String>, depth: usize) -> Option<(usize, Tick)> {
@@ -328,7 +452,13 @@ impl R<'_, '_> {
match &self.class(mob) {
b"MMOB" => {
let key = master.unwrap_or_else(|| id.iter().map(|b| format!("{b:02x}")).collect());
let clip = self.first_clip(seg)?;
let clip = match self.first_clip(seg) {
Ok(c) => c?,
Err(SearchLimit) => {
self.report.warn("a master mob is too large to search for its source clip (imported as a gap)");
return None;
}
};
let inner = units_to_ticks(self.i32(clip, "OMFI:SCLP:StartTime").unwrap_or(0), rate.0, rate.1);
let next = self.uid(clip, "OMFI:SCLP:SourceID")?;
let next_track = self.u32(clip, "OMFI:SCLP:SourceTrackID")?;
@@ -353,7 +483,9 @@ impl R<'_, '_> {
let file_rate = sr.filter(|r| r.0 > 0 && r.1 > 0).unwrap_or(rate);
let length = units_to_ticks(self.i32(desc, "OMFI:MDFL:Length").unwrap_or(0), file_rate.0, file_rate.1);
let summary = self.c.get(desc, "OMFI:WAVD:Summary").or_else(|| self.c.get(desc, "OMFI:AIFD:Summary")).unwrap_or_default();
let fmt = crate::pcm::parse_wav(&summary).or_else(|| crate::pcm::parse_aiff(&summary)).map(|(_, ch, sr, bits)| (ch as u32, sr, bits));
let fmt = craft_audio::essence_pcm::parse_wav(&summary)
.or_else(|| craft_audio::essence_pcm::parse_aiff(&summary))
.map(|(_, ch, sr, bits)| (ch as u32, sr, bits));
let mut path = None;
for l in self.refs(desc, "OMFI:MDES:Locator") {
let p = match &self.class(l) {
@@ -373,7 +505,8 @@ impl R<'_, '_> {
let (file_channels, sample_rate, bits) = fmt.unwrap_or((1, file_rate.0 as u32, 16));
let channel = (file_channels > 1 && sound_slots > 1).then(|| track.saturating_sub(1).min(file_channels - 1));
// timecode of the physical source
let tc = self.first_clip(seg).and_then(|clip| {
let first = self.first_clip(seg).ok().flatten();
let tc = first.and_then(|clip| {
let tid = self.uid(clip, "OMFI:SCLP:SourceID")?;
let tape = *self.mobs.get(&tid)?;
let (start_tc, fps, trate) = self.refs(tape, "OMFI:MOBJ:Slots").into_iter().find_map(|s| {
@@ -415,7 +548,7 @@ impl R<'_, '_> {
nested: None,
};
if let Some(d) = self.media.get(&id) {
if let Some((pcm, ch, sr, b)) = crate::pcm::parse_wav(d).or_else(|| crate::pcm::parse_aiff(d)) {
if let Some((pcm, ch, sr, b)) = craft_audio::essence_pcm::parse_wav(d).or_else(|| craft_audio::essence_pcm::parse_aiff(d)) {
s.embedded = Some(pcm);
s.file_channels = ch as u32;
s.channels = if s.channel.is_some() { 1 } else { ch as u32 };
@@ -437,3 +570,200 @@ impl R<'_, '_> {
}
}
}
#[cfg(test)]
mod tests {
use super::super::bento::Writer;
use super::*;
fn obj(w: &mut Writer, class: &[u8; 4]) -> u32 {
let id = w.new_object();
w.set(id, "OMFI:OOBJ:ObjClass", "omfi:ClassID", class);
id
}
fn list(w: &mut Writer, o: u32, p: &str, targets: &[u32]) {
let mut v = (targets.len() as u16).to_be_bytes().to_vec();
for t in targets {
v.extend_from_slice(&t.to_be_bytes());
}
w.set(o, p, "omfi:ObjRefArray", &v);
}
fn search(max_visits: usize, max_pending: usize, build: impl FnOnce(&mut Writer) -> u32) -> std::result::Result<Option<u32>, SearchLimit> {
let mut w = Writer::new();
let root = build(&mut w);
let bytes = w.finish();
let c = Container::open(&bytes).unwrap();
let mut report = Report::default();
let r = R {
c: &c,
report: &mut report,
be: true,
mobs: HashMap::new(),
media: HashMap::new(),
doc: Document::default(),
sources: HashMap::new(),
budget: 0,
clip_search: HashMap::new(),
search_budget: MAX_DOCUMENT_SEARCH_WORK,
};
r.first_clip_within(root, max_visits, max_pending, &mut 0)
}
#[test]
fn clip_search_reports_its_limits() {
// found within the limits
assert!(matches!(
search(10, 10, |w| {
let (s, c) = (obj(w, b"SEQU"), obj(w, b"SCLP"));
list(w, s, "OMFI:SEQU:Components", &[c]);
s
}),
Ok(Some(_))
));
// too many distinct components before any clip
assert!(
search(5, 1000, |w| {
let (s, f) = (obj(w, b"SEQU"), (0..20).map(|_| obj(w, b"FILL")).collect::<Vec<_>>());
list(w, s, "OMFI:SEQU:Components", &f);
s
})
.is_err()
);
// few visits, but sequences that each queue a long list of references
assert!(
search(1000, 500, |w| {
let f = obj(w, b"FILL");
let inner: Vec<u32> = (0..20).map(|_| obj(w, b"SEQU")).collect();
for &i in &inner {
list(w, i, "OMFI:SEQU:Components", &[f; 100]);
}
let s = obj(w, b"SEQU");
list(w, s, "OMFI:SEQU:Components", &inner);
s
})
.is_err()
);
// a search that ends without a clip is not a limit
assert!(matches!(search(10, 10, |w| obj(w, b"FILL")), Ok(None)));
}
#[test]
fn skipped_slots_spend_the_budget() {
let mut w = Writer::new();
let head = obj(&mut w, b"HEAD");
let cm = obj(&mut w, b"CMOB");
let slot = obj(&mut w, b"MSLT");
list(&mut w, cm, "OMFI:MOBJ:Slots", &[slot; 200]);
list(&mut w, head, "OMFI:HEAD:Mobs", &[cm]);
list(&mut w, head, "OMFI:HEAD:PrimaryMobs", &[cm]);
let bytes = w.finish();
let mut report = Report::default();
read_with_budget(&bytes, &mut report, 50).unwrap();
assert!(report.mentions(BUDGET_WARNING), "slots with no segment cost nothing before");
}
#[test]
fn a_source_clip_search_is_made_once_per_segment() {
let mut w = Writer::new();
let (s, c) = (obj(&mut w, b"SEQU"), obj(&mut w, b"SCLP"));
list(&mut w, s, "OMFI:SEQU:Components", &[c]);
let bytes = w.finish();
let c = Container::open(&bytes).unwrap();
let mut report = Report::default();
let mut r = R {
c: &c,
report: &mut report,
be: true,
mobs: HashMap::new(),
media: HashMap::new(),
doc: Document::default(),
sources: HashMap::new(),
budget: 0,
clip_search: HashMap::new(),
search_budget: MAX_DOCUMENT_SEARCH_WORK,
};
let first = r.first_clip(s).ok();
assert!(matches!(first, Some(Some(_))));
assert_eq!(r.clip_search.len(), 1);
assert_eq!(r.first_clip(s).ok(), first);
assert_eq!(r.clip_search.len(), 1);
}
#[test]
fn repeated_slots_and_compositions_spend_the_budget() {
let mut w = Writer::new();
let head = obj(&mut w, b"HEAD");
let cm = obj(&mut w, b"CMOB");
let dd = obj(&mut w, b"DDEF");
w.set(dd, "OMFI:DDEF:DataKindID", "omfi:UniqueName", b"omfi:data:Sound\0");
let fill = obj(&mut w, b"FILL");
w.set(fill, "OMFI:CPNT:DataKind", "omfi:ObjRef", &dd.to_be_bytes());
let slot = obj(&mut w, b"MSLT");
w.set(slot, "OMFI:MSLT:Segment", "omfi:ObjRef", &fill.to_be_bytes());
list(&mut w, cm, "OMFI:MOBJ:Slots", &[slot; 200]);
list(&mut w, head, "OMFI:HEAD:Mobs", &[cm]);
list(&mut w, head, "OMFI:HEAD:PrimaryMobs", &[cm; 200]);
let bytes = w.finish();
let mut report = Report::default();
let doc = read_with_budget(&bytes, &mut report, 50).unwrap();
let tracks: usize = doc.compositions.iter().map(|c| c.tracks.len()).sum();
assert!(tracks <= 50, "{tracks} tracks from a budget of 50");
assert!(doc.compositions.len() <= 2, "{} compositions", doc.compositions.len());
assert!(report.has_warnings());
}
fn with_reader<T>(bytes: &[u8], budget: usize, search_budget: usize, f: impl FnOnce(&mut R) -> T) -> (T, Report) {
let c = Container::open(bytes).unwrap();
let mut report = Report::default();
let mut r = R {
c: &c,
report: &mut report,
be: true,
mobs: HashMap::new(),
media: HashMap::new(),
doc: Document::default(),
sources: HashMap::new(),
budget,
clip_search: HashMap::new(),
search_budget,
};
let out = f(&mut r);
(out, report)
}
#[test]
fn depth_limited_components_spend_the_budget() {
// a chain 33 sequences deep whose last one repeats a depth-limited sequence many times
let mut w = Writer::new();
let deep = obj(&mut w, b"SEQU");
let chain: Vec<u32> = (0..33).map(|_| obj(&mut w, b"SEQU")).collect();
for pair in chain.windows(2) {
list(&mut w, pair[0], "OMFI:SEQU:Components", &[pair[1]]);
}
list(&mut w, chain[32], "OMFI:SEQU:Components", &[deep; 1000]);
let bytes = w.finish();
let (items, report) = with_reader(&bytes, 100, MAX_DOCUMENT_SEARCH_WORK, |r| r.segment(chain[0], (48_000, 1), 0));
assert!(items.len() <= 100, "{} items from a budget of 100", items.len());
assert!(report.mentions(BUDGET_WARNING));
}
#[test]
fn source_clip_searches_share_a_document_budget() {
// distinct roots over one shared clip-free graph
let mut w = Writer::new();
let fills: Vec<u32> = (0..50).map(|_| obj(&mut w, b"FILL")).collect();
let shared = obj(&mut w, b"SEQU");
list(&mut w, shared, "OMFI:SEQU:Components", &fills);
let roots: Vec<u32> = (0..30).map(|_| obj(&mut w, b"SEQU")).collect();
for &root in &roots {
list(&mut w, root, "OMFI:SEQU:Components", &[shared]);
}
let bytes = w.finish();
let (results, _) = with_reader(&bytes, 0, 500, |r| roots.iter().map(|&root| r.first_clip(root)).collect::<Vec<_>>());
assert!(matches!(results[0], Ok(None)), "the first search fits");
assert!(results.iter().any(|r| r.is_err()), "later searches run out of the shared budget");
let (results, _) = with_reader(&bytes, 0, MAX_DOCUMENT_SEARCH_WORK, |r| roots.iter().map(|&root| r.first_clip(root)).collect::<Vec<_>>());
assert!(results.iter().all(|r| matches!(r, Ok(None))));
}
}
-5
View File
@@ -4,8 +4,3 @@
pub(crate) fn parse_wav(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
craft_audio::wav_pcm(b).map(|p| (p.data, p.channels, p.sample_rate, p.bits))
}
/// The same for an AIFF / AIFF-C (`NONE`, `twos`, `sowt`) file.
pub(crate) fn parse_aiff(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
craft_audio::aiff_pcm(b).map(|p| (p.data, p.channels, p.sample_rate, p.bits))
}
+43 -1
View File
@@ -63,6 +63,7 @@ pub fn parse_aiff(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
if b.len() < 12 || &b[..4] != b"FORM" || !(&b[8..12] == b"AIFF" || &b[8..12] == b"AIFC") {
return None;
}
let aifc = &b[8..12] == b"AIFC";
let mut at = 12;
let mut comm = None;
while at + 8 <= b.len() {
@@ -73,7 +74,17 @@ pub fn parse_aiff(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
let ch = u16::from_be_bytes([body[0], body[1]]);
let bits = u16::from_be_bytes([body[6], body[7]]);
let sr = from_extended(body[8..18].try_into().ok()?).round() as u32;
let little = body.len() >= 22 && &body[18..22] == b"sowt";
// AIFF has no compression code (trailing COMM bytes mean nothing); AIFF-C must name one
// of the integer PCM codes, or the samples are not integer PCM at all
let little = if aifc {
match body.get(18..22)? {
b"NONE" | b"twos" => false,
b"sowt" => true,
_ => return None,
}
} else {
false
};
comm = Some((ch, sr, bits, little));
} else if id == b"SSND" && body.len() >= 8 {
let (ch, sr, bits, little) = comm?;
@@ -144,4 +155,35 @@ mod tests {
let _ = parse_aiff(&a[..cut.min(a.len())]);
}
}
#[test]
fn aifc_needs_an_integer_pcm_compression_code() {
let file = |kind: &[u8; 4], code: &[u8]| {
let mut comm = vec![0, 1, 0, 0, 0, 1, 0, 16];
comm.extend_from_slice(&[0x40, 0x0E, 0xBB, 0x80, 0, 0, 0, 0, 0, 0]);
comm.extend_from_slice(code);
let mut body = kind.to_vec();
body.extend_from_slice(b"COMM");
body.extend_from_slice(&(comm.len() as u32).to_be_bytes());
body.extend_from_slice(&comm);
if comm.len() & 1 == 1 {
body.push(0);
}
body.extend_from_slice(b"SSND");
body.extend_from_slice(&10u32.to_be_bytes());
body.extend_from_slice(&[0; 8]);
body.extend_from_slice(&[0x12, 0x34]);
let mut v = b"FORM".to_vec();
v.extend_from_slice(&(body.len() as u32).to_be_bytes());
v.extend_from_slice(&body);
v
};
assert_eq!(parse_aiff(&file(b"AIFC", b"NONE")).map(|p| p.0), Some(vec![0x34, 0x12]));
assert_eq!(parse_aiff(&file(b"AIFC", b"twos")).map(|p| p.0), Some(vec![0x34, 0x12]));
assert_eq!(parse_aiff(&file(b"AIFC", b"sowt")).map(|p| p.0), Some(vec![0x12, 0x34]));
assert!(parse_aiff(&file(b"AIFC", b"fl32")).is_none(), "float samples are not integer PCM");
assert!(parse_aiff(&file(b"AIFC", b"")).is_none(), "AIFF-C without a compression code");
// plain AIFF ignores trailing COMM bytes
assert_eq!(parse_aiff(&file(b"AIFF", b"sowt")).map(|p| p.0), Some(vec![0x34, 0x12]));
}
}
+20
View File
@@ -79,6 +79,8 @@ mod write;
/// PCM WAVE and AIFF wrappers for interchange essence.
#[cfg(feature = "pcm-encode")]
pub mod essence_pcm;
/// Streaming WAVE and AIFF headers and sample packing.
pub mod pcm_export;
/// Interleaved PCM packet decoding.
pub mod pcm_interleaved;
/// Range-backed PCM WAVE access and a 16-bit PCM writer.
@@ -762,3 +764,21 @@ extern crate self as craft_audio;
#[cfg(test)]
#[path = "../tests/main.rs"]
mod integration;
// An encoder refusing its settings is an encode error, not damaged input.
#[cfg(all(test, feature = "aac-encode"))]
mod tests {
use super::*;
#[test]
fn aac_encoder_settings_are_encode_errors() {
let e = match aac::Encoder::new(aac::EncoderConfig::cbr(44_000, 2, 128_000)) {
Err(e) => AudioError::from(e),
Ok(_) => panic!("44 kHz is not an AAC rate"),
};
assert!(matches!(e, AudioError::Encode(_)), "{e}");
}
}
/// DVD and Blu-ray LPCM packet decoding.
pub mod lpcm;
+118
View File
@@ -0,0 +1,118 @@
//! DVD-Video and Blu-ray LPCM packet grammar and sample decoding.
/// Blu-ray / AVCHD LPCM packet header (4 bytes before the samples).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct BlurayLpcm {
/// Sample rate in hertz.
pub sample_rate: u32,
/// Number of decoded channels.
pub channels: usize,
/// Significant bits per sample.
pub bits: u32,
}
/// Channels of each channel_assignment code (mono, stereo, 3/0, 2/1, 3/1, 2/2, 3/2, 3/2+LFE,
/// 3/4, 3/4+LFE).
const BD_LPCM_CHANNELS: [usize; 12] = [0, 1, 0, 2, 3, 3, 4, 4, 5, 6, 7, 8];
/// Parse and validate the four-byte Blu-ray LPCM header.
pub fn bluray_lpcm_header(p: &[u8]) -> Option<BlurayLpcm> {
if p.len() < 4 {
return None;
}
let channels = *BD_LPCM_CHANNELS.get((p[2] >> 4) as usize).filter(|&&c| c > 0)?;
let sample_rate = match p[2] & 15 {
1 => 48_000,
4 => 96_000,
5 => 192_000,
_ => return None,
};
let bits = match p[3] >> 6 {
1 => 16,
2 => 20,
3 => 24,
_ => return None,
};
Some(BlurayLpcm { sample_rate, channels, bits })
}
/// Decode one Blu-ray LPCM packet (big-endian; channels padded to an even count; 20-bit
/// samples are stored in 24 bits).
pub fn decode_bluray_lpcm(p: &[u8]) -> Option<Vec<Vec<f32>>> {
let h = bluray_lpcm_header(p)?;
let coded = (h.channels + 1) & !1;
let bps = if h.bits == 16 { 2 } else { 3 };
let data = &p[4..];
let n = data.len() / (coded * bps);
let mut out = vec![Vec::with_capacity(n); h.channels];
for k in 0..n {
for (c, o) in out.iter_mut().enumerate() {
let at = (k * coded + c) * bps;
let v = if bps == 2 {
i16::from_be_bytes([data[at], data[at + 1]]) as f32 / 32768.0
} else {
(i32::from_be_bytes([data[at], data[at + 1], data[at + 2], 0]) >> 8) as f32 / 8_388_608.0
};
o.push(v);
}
}
Some(out)
}
/// Bytes per two sample frames of DVD LPCM. The caller must bound `channels` to the DVD channel range.
pub fn dvd_lpcm_group_bytes(channels: usize, bits: u32) -> usize {
match bits {
16 => 4 * channels,
20 => 5 * channels,
_ => 6 * channels,
}
}
/// Decode DVD-Video LPCM (big-endian). 16-bit samples are interleaved; 20- and 24-bit samples
/// come in groups of two frames: the upper 16 bits of all the group's samples, then their low
/// 4 / 8 bits. The caller must bound `channels` to the DVD channel range.
pub fn decode_dvd_lpcm(data: &[u8], channels: usize, bits: u32) -> Vec<Vec<f32>> {
let ch = channels.max(1);
let group = dvd_lpcm_group_bytes(ch, bits);
let groups = data.len() / group;
let mut out = vec![Vec::with_capacity(groups * 2); ch];
for g in 0..groups {
let b = &data[g * group..(g + 1) * group];
for f in 0..2 {
for (c, o) in out.iter_mut().enumerate() {
let k = f * ch + c;
let hi = i16::from_be_bytes([b[2 * k], b[2 * k + 1]]) as i32;
let v = match bits {
16 => hi as f32 / 32768.0,
20 => {
let nib = b[4 * ch + k / 2];
let lo = if k.is_multiple_of(2) { nib >> 4 } else { nib & 15 } as i32;
((hi << 4) | lo) as f32 / 524_288.0
}
_ => ((hi << 8) | b[4 * ch + k] as i32) as f32 / 8_388_608.0,
};
o.push(v);
}
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn bluray_header_and_even_channel_padding() {
let packet = [0, 0, 0x31, 0x40, 0x40, 0, 0xc0, 0];
assert_eq!(bluray_lpcm_header(&packet), Some(BlurayLpcm { sample_rate: 48_000, channels: 2, bits: 16 }));
assert_eq!(decode_bluray_lpcm(&packet), Some(vec![vec![0.5], vec![-0.5]]));
assert_eq!(decode_bluray_lpcm(&packet[..3]), None);
}
#[test]
fn dvd_lpcm_16_20_and_24_bit_groups() {
assert_eq!(decode_dvd_lpcm(&[0x40, 0, 0xc0, 0], 1, 16), vec![vec![0.5, -0.5]]);
assert_eq!(decode_dvd_lpcm(&[0x40, 0, 0xc0, 0, 0], 1, 20), vec![vec![0.5, -0.5]]);
assert_eq!(decode_dvd_lpcm(&[0x40, 0, 0xc0, 0, 0, 0], 1, 24), vec![vec![0.5, -0.5]]);
assert_eq!(decode_dvd_lpcm(&[1, 2, 3], 1, 16), vec![Vec::<f32>::new()]);
}
}
+287
View File
@@ -0,0 +1,287 @@
//! Audio-only files: WAV (RIFF, little-endian PCM; RF64 past 4 GiB) and AIFF (big-endian PCM),
//! 16, 24 or 32-bit integer or 32-bit float ([`SampleFormat`]), for streaming export.
//! Float WAV is `WAVE_FORMAT_IEEE_FLOAT` with a `fact` chunk; float AIFF is AIFF-C (`fl32`).
//!
//! The header is written first from the known length (`header`), then the samples in chunks
//! (`encode`) and the pad byte (`pad`), so an export streams to disk instead of building the
//! file in memory.
/// PCM representation in an exported file.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SampleFormat {
/// 16-bit signed integer.
Int16,
/// 24-bit signed integer.
Int24,
/// 32-bit signed integer.
Int32,
/// 32-bit IEEE float.
Float32,
}
impl SampleFormat {
/// Bits per sample.
pub fn bits(self) -> u16 {
match self {
Self::Int16 => 16,
Self::Int24 => 24,
Self::Int32 | Self::Float32 => 32,
}
}
/// Bytes per sample.
pub fn bytes(self) -> usize {
usize::from(self.bits() / 8)
}
/// Whether this is a floating point format.
pub fn is_float(self) -> bool {
self == Self::Float32
}
/// Clip and quantize one sample with symmetric integer full scale; NaN becomes silence.
pub fn quantise(self, s: f32) -> i32 {
let s = s.clamp(-1.0, 1.0);
match self {
Self::Int16 => (s * 32_767.0).round() as i32,
Self::Int24 => (s * 8_388_607.0).round() as i32,
Self::Int32 => (f64::from(s) * 2_147_483_647.0).round() as i32,
Self::Float32 => 0,
}
}
/// Sanitize a floating sample: NaN becomes silence, infinity becomes the largest finite value.
pub fn float(s: f32) -> f32 {
if s.is_nan() { 0.0 } else { s.clamp(f32::MIN, f32::MAX) }
}
}
/// The container of an audio-only export.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum PcmContainer {
/// RIFF/WAVE or RF64.
Wav,
/// AIFF or AIFF-C.
Aiff,
}
/// Bytes of sample data for `frames` frames, or None when that does not fit 64 bits.
fn data_len(frames: u64, channels: u16, format: SampleFormat) -> Option<u64> {
frames.checked_mul(u64::from(channels))?.checked_mul(format.bytes() as u64)
}
/// The pad byte after an odd-sized data chunk (RIFF and AIFF chunks are word-aligned).
pub fn pad(frames: u64, channels: u16, format: SampleFormat) -> &'static [u8] {
if data_len(frames, channels, format).is_some_and(|n| n & 1 == 1) { &[0] } else { &[] }
}
/// `KSDATAFORMAT_SUBTYPE_PCM` / `_IEEE_FLOAT` after their first two bytes (the format tag).
const SUBTYPE_TAIL: [u8; 14] = [0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71];
/// AIFF-C format version 1 (`FVER`).
const AIFC_VERSION_1: u32 = 0xA280_5140;
/// The AIFF-C compression name of `fl32`, as a Pascal string of even length.
const FL32_NAME: &[u8] = b"\x1532-bit floating point";
/// Everything before the samples of a file of `frames` frames. A WAV whose RIFF size would not
/// fit 32 bits (4 GiB: 24-bit 5.1 at 96 kHz after about 41 minutes) is written as RF64 (EBU Tech
/// 3306), with the 64-bit sizes in a `ds64` chunk. AIFF has no 64-bit form, so a longer AIFF is
/// an error naming WAV instead.
pub fn header(container: PcmContainer, channels: u16, sample_rate: u32, format: SampleFormat, frames: u64) -> Result<Vec<u8>, String> {
let too_long = || "the audio is too long to write".to_string();
if channels == 0 {
return Err("audio needs at least one channel".into());
}
if sample_rate == 0 {
return Err("audio needs a sample rate".into());
}
let bits = format.bits();
let bps = bits / 8;
let float = format.is_float();
let data = data_len(frames, channels, format).ok_or_else(too_long)?;
let block_align = channels.checked_mul(bps).ok_or("too many audio channels")?;
match container {
PcmContainer::Wav => {
let byte_rate =
u32::try_from(u64::from(sample_rate) * u64::from(block_align)).map_err(|_| "the audio sample rate is too high".to_string())?;
let ext = channels > 2;
// plain PCM has no cbSize; IEEE float has an empty one
let fmt_len: u32 = if ext {
40
} else if float {
18
} else {
16
};
// every format but integer PCM has a `fact` chunk (the frame count; RF64: in `ds64`)
let fact_len: u64 = if float { 12 } else { 0 };
let padded = data.checked_add(data & 1).ok_or_else(too_long)?;
// "WAVE" + fmt chunk + fact chunk + data chunk header + samples
let riff = (4 + 8 + u64::from(fmt_len) + fact_len + 8).checked_add(padded).ok_or_else(too_long)?;
let rf64 = riff > u64::from(u32::MAX);
let mut v = Vec::with_capacity(100);
if rf64 {
const DS64: u64 = 8 + 28;
v.extend_from_slice(b"RF64");
v.extend_from_slice(&u32::MAX.to_le_bytes());
v.extend_from_slice(b"WAVEds64");
v.extend_from_slice(&28u32.to_le_bytes());
v.extend_from_slice(&riff.checked_add(DS64).ok_or_else(too_long)?.to_le_bytes());
v.extend_from_slice(&data.to_le_bytes());
v.extend_from_slice(&frames.to_le_bytes());
v.extend_from_slice(&0u32.to_le_bytes()); // no table entries
} else {
v.extend_from_slice(b"RIFF");
v.extend_from_slice(&(riff as u32).to_le_bytes());
v.extend_from_slice(b"WAVE");
}
let tag: u16 = if float { 3 } else { 1 };
v.extend_from_slice(b"fmt ");
v.extend_from_slice(&fmt_len.to_le_bytes());
v.extend_from_slice(&(if ext { 0xFFFE } else { tag }).to_le_bytes());
v.extend_from_slice(&channels.to_le_bytes());
v.extend_from_slice(&sample_rate.to_le_bytes());
v.extend_from_slice(&byte_rate.to_le_bytes());
v.extend_from_slice(&block_align.to_le_bytes());
v.extend_from_slice(&bits.to_le_bytes());
if ext {
// cbSize, valid bits, channel mask, KSDATAFORMAT_SUBTYPE_PCM / _IEEE_FLOAT
v.extend_from_slice(&22u16.to_le_bytes());
v.extend_from_slice(&bits.to_le_bytes());
let mask: u32 = if channels == 6 { 0x3F } else { 0 };
v.extend_from_slice(&mask.to_le_bytes());
v.extend_from_slice(&tag.to_le_bytes());
v.extend_from_slice(&SUBTYPE_TAIL);
} else if float {
v.extend_from_slice(&0u16.to_le_bytes());
}
if float {
v.extend_from_slice(b"fact");
v.extend_from_slice(&4u32.to_le_bytes());
v.extend_from_slice(&(if rf64 { u32::MAX } else { u32::try_from(frames).unwrap_or(u32::MAX) }).to_le_bytes());
}
v.extend_from_slice(b"data");
v.extend_from_slice(&(if rf64 { u32::MAX } else { data as u32 }).to_le_bytes());
Ok(v)
}
PcmContainer::Aiff => {
let aiff_too_long = || "AIFF holds at most 4 GB of audio: export WAV, which has no such limit".to_string();
let frames32 = u32::try_from(frames).map_err(|_| aiff_too_long())?;
// SSND: offset + block size + samples
let ssnd = u32::try_from(data.checked_add(8).ok_or_else(aiff_too_long)?).map_err(|_| aiff_too_long())?;
// floats need AIFF-C: a version chunk, and the compression type and name in COMM
let comm: u32 = if float { 18 + 4 + FL32_NAME.len() as u32 } else { 18 };
let fver: u32 = if float { 8 + 4 } else { 0 };
let form = (4u32 + fver + (8 + comm))
.checked_add(8)
.and_then(|n| n.checked_add(ssnd))
.and_then(|n| n.checked_add(ssnd & 1))
.ok_or_else(aiff_too_long)?;
let mut v = Vec::with_capacity(90);
v.extend_from_slice(b"FORM");
v.extend_from_slice(&form.to_be_bytes());
if float {
v.extend_from_slice(b"AIFCFVER");
v.extend_from_slice(&4u32.to_be_bytes());
v.extend_from_slice(&AIFC_VERSION_1.to_be_bytes());
} else {
v.extend_from_slice(b"AIFF");
}
v.extend_from_slice(b"COMM");
v.extend_from_slice(&comm.to_be_bytes());
v.extend_from_slice(&channels.to_be_bytes());
v.extend_from_slice(&frames32.to_be_bytes());
v.extend_from_slice(&bits.to_be_bytes());
v.extend_from_slice(&extended(sample_rate));
if float {
v.extend_from_slice(b"fl32");
v.extend_from_slice(FL32_NAME);
}
v.extend_from_slice(b"SSND");
v.extend_from_slice(&ssnd.to_be_bytes());
v.extend_from_slice(&0u32.to_be_bytes()); // offset
v.extend_from_slice(&0u32.to_be_bytes()); // block size
Ok(v)
}
}
}
/// Append interleaved samples in `format` (little-endian for WAV, big-endian for AIFF).
pub fn encode(container: PcmContainer, interleaved: &[f32], format: SampleFormat, out: &mut Vec<u8>) {
let bps = format.bytes();
out.reserve(interleaved.len().saturating_mul(bps));
for &s in interleaved {
let q = if format.is_float() { SampleFormat::float(s).to_bits() as i32 } else { format.quantise(s) };
match container {
PcmContainer::Wav => out.extend_from_slice(q.to_le_bytes().get(..bps).unwrap_or_default()),
PcmContainer::Aiff => out.extend_from_slice(q.to_be_bytes().get(4 - bps..).unwrap_or_default()),
}
}
}
/// A whole file in memory (short audio; exports stream with [`header`] and [`encode`]).
fn write_whole(container: PcmContainer, interleaved: &[f32], channels: u16, sample_rate: u32, format: SampleFormat) -> Result<Vec<u8>, String> {
let per_frame = usize::from(channels.max(1));
if !interleaved.len().is_multiple_of(per_frame) {
return Err("the samples do not make whole frames".into());
}
let frames = (interleaved.len() / per_frame) as u64;
let mut v = header(container, channels, sample_rate, format, frames)?;
encode(container, interleaved, format, &mut v);
v.extend_from_slice(pad(frames, channels, format));
Ok(v)
}
/// A RIFF/WAVE file of interleaved samples. More than two channels are written as
/// `WAVE_FORMAT_EXTENSIBLE` with the speaker mask of the layout (6 channels: L, R, C, LFE, Ls, Rs
/// = `0x3F`).
pub fn write_wav(interleaved: &[f32], channels: u16, sample_rate: u32, format: SampleFormat) -> Result<Vec<u8>, String> {
write_whole(PcmContainer::Wav, interleaved, channels, sample_rate, format)
}
/// IEEE 754 80-bit extended encoding of a sample rate (AIFF `COMM`).
pub fn extended(rate: u32) -> [u8; 10] {
let mut out = [0u8; 10];
if rate == 0 {
return out;
}
let shift = rate.leading_zeros();
let mant = (rate as u64) << (32 + shift);
let exp = 16383 + 31 - shift as u16;
out[..2].copy_from_slice(&exp.to_be_bytes());
out[2..].copy_from_slice(&mant.to_be_bytes());
out
}
/// An AIFF file (`FORM`/`AIFF`, `COMM` + `SSND`; AIFF-C for floats) of interleaved samples.
pub fn write_aiff(interleaved: &[f32], channels: u16, sample_rate: u32, format: SampleFormat) -> Result<Vec<u8>, String> {
write_whole(PcmContainer::Aiff, interleaved, channels, sample_rate, format)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn streaming_formats_preserve_samples_and_bounds() {
let samples = [1.0, -1.0, f32::NAN];
let wav = write_wav(&samples, 1, 48_000, SampleFormat::Int24).unwrap();
assert_eq!(&wav[..4], b"RIFF");
assert_eq!(&wav[44..53], &[255, 255, 127, 1, 0, 128, 0, 0, 0]);
let h = header(PcmContainer::Wav, 6, 96_000, SampleFormat::Int24, 96_000 * 3600).unwrap();
assert_eq!(&h[..4], b"RF64");
assert!(header(PcmContainer::Aiff, 2, 48_000, SampleFormat::Int24, u64::MAX).is_err());
let aiff = write_aiff(&[0.25, -3.0], 1, 44_100, SampleFormat::Float32).unwrap();
assert_eq!(&aiff[8..12], b"AIFC");
assert_eq!(&aiff[aiff.len() - 8..], [0.25f32.to_be_bytes(), (-3.0f32).to_be_bytes()].concat());
}
#[test]
fn zero_channels_rates_and_partial_frames_are_refused() {
for container in [PcmContainer::Wav, PcmContainer::Aiff] {
assert!(header(container, 0, 48_000, SampleFormat::Int16, 1).is_err());
assert!(header(container, 1, 0, SampleFormat::Int16, 1).is_err());
assert!(header(container, 2, 48_000, SampleFormat::Int16, 0).is_ok(), "zero frames stay valid");
}
assert!(write_wav(&[0.5], 0, 48_000, SampleFormat::Int16).is_err());
assert!(write_wav(&[0.1, 0.2, 0.3], 2, 48_000, SampleFormat::Int16).is_err());
assert!(write_aiff(&[0.1, 0.2, 0.3], 2, 48_000, SampleFormat::Int16).is_err());
assert!(write_aiff(&[0.1, 0.2], 2, 48_000, SampleFormat::Int16).is_ok());
}
}
+1 -1
View File
@@ -9,7 +9,7 @@ use super::model::Action;
///
/// A file PhotoCraft wrote with its folder occurrences (`set_rows`, and per action `set_slot`,
/// `action_id`, `legacy_shortcut_owner`) loads too: those fields are read into
/// [`FileOccurrences`] and [`ActionList::merge_file`](super::ActionList::merge_file) turns them
/// `FileOccurrences` and [`ActionList::merge_file`](super::ActionList::merge_file) turns them
/// into unique set and action names. Names are the list's only identity, so they are never
/// written back.
#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)]
+2 -2
View File
@@ -108,7 +108,7 @@ impl<'a> AgentsPage<'a> {
let mut next = None;
let mut field = Field::new(words.control_port.clone()).id_salt("port");
if mcp_flag {
field = field.help(Help::Text(words.port_hint.clone().into()));
field = field.help(Help::Text(words.port_hint.clone()));
}
field.show(ui, |ui, _| {
let number = NumberField::new(words.control_port.clone(), Bind::given(Some(port as f64)))
@@ -140,7 +140,7 @@ impl<'a> AgentsPage<'a> {
.map(|choice| {
let label: Label = match choice {
WhenAppClosed::Launch => words.launch.fill(&[("app", app)]).into(),
WhenAppClosed::Headless => words.headless.clone().into(),
WhenAppClosed::Headless => words.headless.clone(),
WhenAppClosed::Error => words.ask.fill(&[("app", app)]).into(),
};
Entry::new(choice, label)
+35 -3
View File
@@ -176,6 +176,13 @@ craft-tcraft = { workspace = true, optional = true }
craft-texture = { workspace = true, optional = true }
craft-usd = { workspace = true, optional = true }
craft-xfl = { workspace = true, optional = true }
craft-elf = { workspace = true, optional = true }
craft-format = { workspace = true, optional = true }
craft-macho = { workspace = true, optional = true }
craft-mcp-server = { workspace = true, optional = true }
craft-pe = { workspace = true, optional = true }
craft-sniff = { workspace = true, optional = true }
craft-tar = { workspace = true, optional = true }
[features]
ui = [
@@ -228,7 +235,6 @@ aep = ["dep:craft-aep"]
affinity = ["dep:craft-affinity"]
affinity-synth = ["affinity", "craft-affinity/synth"]
ai-agents = ["dep:craft-ai-agents"]
ai-core = ["dep:craft-ai-core"]
anicraft-format = ["dep:craft-anicraft-format"]
@@ -267,8 +273,7 @@ audio-symphonia = ["audio", "craft-audio/symphonia"]
audio-vorbis-encode = ["audio", "craft-audio/vorbis-encode"]
automation = ["dep:craft-automation", "keymap"]
# Agents in the open window: the launcher's switch, the connection file, the link host and the Agents page.
automation-agents = ["automation", "craft-automation/agents", "ai-agents", "flags"]
automation-agents = ["automation", "craft-automation/agents"]
ber = ["dep:craft-ber"]
@@ -323,6 +328,9 @@ dxf = ["dep:craft-dxf"]
ecproj-format = ["dep:craft-ecproj-format"]
elf = ["dep:craft-elf"]
elf-serde = ["elf", "craft-elf/serde"]
eps = ["dep:craft-eps"]
eps-paths = ["eps", "craft-eps/paths"]
@@ -377,6 +385,8 @@ font-store-bundled-index = ["font-store", "craft-font-store/bundled-index"]
font-subset = ["dep:craft-font-subset"] # native only: HarfBuzz's C++ builds with the target's C++ compiler
format = ["dep:craft-format"]
gainmap = ["dep:craft-gainmap"]
gcode = ["dep:craft-gcode"]
@@ -412,6 +422,8 @@ image-codecs-avif-decode = ["image-codecs", "craft-image-codecs/avif-decode"]
image-codecs-corpus = ["image-codecs", "craft-image-codecs/corpus"]
image-codecs-heif = ["image-codecs", "craft-image-codecs/heif"]
image-codecs-sgi = ["image-codecs", "craft-image-codecs/sgi"]
image-codecs-sunraster = ["image-codecs", "craft-image-codecs/sunraster"]
image-codecs-threads = ["image-codecs", "craft-image-codecs/threads"]
imaging = ["dep:craft-imaging", "color", "keymap"]
@@ -442,6 +454,9 @@ layers = ["dep:craft-layers"]
lottie = ["dep:craft-lottie"]
macho = ["dep:craft-macho"]
macho-serde = ["macho", "craft-macho/serde"]
markdown = ["dep:craft-markdown"]
materialx = ["dep:craft-materialx"]
@@ -452,6 +467,8 @@ mcmat = ["dep:craft-mcmat"]
mcp = ["dep:craft-mcp"]
mcp-server = ["dep:craft-mcp-server"]
media = ["dep:craft-media"]
media-avi = ["media", "craft-media/avi"]
media-bitstream = ["media", "craft-media/bitstream"]
@@ -504,6 +521,7 @@ pcraft-bundle = ["dep:craft-pcraft-bundle"]
pcraft-format = ["dep:craft-pcraft-format"]
pdf = ["dep:craft-pdf"]
pdf-content = ["pdf", "craft-pdf/content"]
pdf-cos = ["pdf", "craft-pdf/cos"]
pdf-crypt = ["pdf", "craft-pdf/crypt"]
pdf-filters = ["pdf", "craft-pdf/filters"]
@@ -519,6 +537,9 @@ pdf-write-svg = ["pdf", "craft-pdf/write-svg"]
pdn = ["dep:craft-pdn"]
pe = ["dep:craft-pe"]
pe-serde = ["pe", "craft-pe/serde"]
penpot = ["dep:craft-penpot"]
pkcs7 = ["dep:craft-pkcs7"]
@@ -552,6 +573,7 @@ proto = ["dep:craft-proto"]
ps = ["dep:craft-ps"]
ps-read = ["ps", "craft-ps/read"]
ps-write = ["ps", "craft-ps/write"]
psd = ["dep:craft-psd"]
psd-corpus = ["psd", "craft-psd/corpus"]
@@ -575,6 +597,8 @@ raw-erf = ["raw", "craft-raw/erf"]
raw-hasselblad = ["raw", "craft-raw/hasselblad"]
raw-jxl = ["raw", "craft-raw/jxl"]
raw-kodak = ["raw", "craft-raw/kodak"]
raw-line-dng = ["raw", "craft-raw/line-dng"]
raw-line-dng-threads = ["raw", "craft-raw/line-dng-threads"]
raw-mef = ["raw", "craft-raw/mef"]
raw-mrw = ["raw", "craft-raw/mrw"]
raw-nef = ["raw", "craft-raw/nef"]
@@ -608,6 +632,8 @@ sketch = ["dep:craft-sketch"]
smime = ["dep:craft-smime"]
sniff = ["dep:craft-sniff"]
stencilcraft-document = ["dep:craft-stencilcraft-document"]
stencilcraft-format = ["dep:craft-stencilcraft-format"]
@@ -616,6 +642,7 @@ stream-codecs = ["dep:craft-stream-codecs"]
svg = ["dep:craft-svg"]
svg-draw = ["svg", "craft-svg/draw"]
svg-paths = ["svg", "craft-svg/paths"]
svg-tree = ["svg", "craft-svg/tree"]
svg-tree-core = ["svg", "craft-svg/tree-core"]
svg-tree-render = ["svg", "craft-svg/tree-render"]
@@ -625,6 +652,9 @@ svg-tree-text = ["svg", "craft-svg/tree-text"]
table = ["dep:craft-table", "keymap"]
tar = ["dep:craft-tar"]
tar-serde = ["tar", "craft-tar/serde"]
tcraft = ["dep:craft-tcraft"]
text = ["dep:craft-text", "keymap"]
@@ -673,6 +703,7 @@ text-oriya = ["text", "craft-text/oriya"]
text-panjabi = ["text", "craft-text/panjabi"]
text-polish = ["text", "craft-text/polish"]
text-portuguese = ["text", "craft-text/portuguese"]
text-positioned = ["text", "craft-text/positioned"]
text-russian = ["text", "craft-text/russian"]
text-sanskrit = ["text", "craft-text/sanskrit"]
text-segment = ["text", "craft-text/segment"]
@@ -737,6 +768,7 @@ video-h264-encode = ["video", "craft-video/h264-encode"]
video-hevc = ["video", "craft-video/hevc"]
video-hevc-decode = ["video", "craft-video/hevc-decode"]
video-hevc-encode = ["video", "craft-video/hevc-encode"]
video-mjpeg = ["video", "craft-video/mjpeg"]
video-mpeg2 = ["video", "craft-video/mpeg2"]
video-mpeg2-decode = ["video", "craft-video/mpeg2-decode"]
video-mpeg2-encode = ["video", "craft-video/mpeg2-encode"]
+1 -1
View File
@@ -26,7 +26,7 @@ The `[features]` table in `Cargo.toml` is written by `tests/features.rs` from th
Reached through other crates, not named in the facade: the six library crates (`craft-ui-contract` to `craft-ui-shell`), `craft-ui-macros` and `craft-behavior` through `craft-ui`, and `craft-win32` through `craft-window`.
Not in the facade, because they are tools or test harnesses: `craft-ui-gallery`, `story-render`, `feature-root` (the features local builds resolve with), `craft-ui-bench`, `craft-ui-test-kit`, `craft-fonts-build` (a build-script helper), `craft-design-lint`, `craft-plate-test`, `craft-media-testkit`.
Not in the facade, because they are tools or test harnesses: `craft-ui-gallery`, `story-render`, `feature-root` (the features local builds resolve with), `craft-ui-bench`, `craft-ui-test-kit`, `craft-fonts-build` (a build-script helper), `craft-design-lint`, `craft-plate-test`, `craft-media-testkit`, `craft-render-parity`.
## Family dependencies
+14
View File
@@ -79,6 +79,8 @@ pub use craft_dwg as dwg;
pub use craft_dxf as dxf;
#[cfg(feature = "ecproj-format")]
pub use craft_ecproj_format as ecproj_format;
#[cfg(feature = "elf")]
pub use craft_elf as elf;
#[cfg(feature = "eps")]
pub use craft_eps as eps;
#[cfg(feature = "epub")]
@@ -111,6 +113,8 @@ pub use craft_font_manager as font_manager;
pub use craft_font_store as font_store;
#[cfg(feature = "font-subset")]
pub use craft_font_subset as font_subset;
#[cfg(feature = "format")]
pub use craft_format as format;
#[cfg(feature = "gainmap")]
pub use craft_gainmap as gainmap;
#[cfg(feature = "gcode")]
@@ -161,6 +165,8 @@ pub use craft_latex as latex;
pub use craft_layers as layers;
#[cfg(feature = "lottie")]
pub use craft_lottie as lottie;
#[cfg(feature = "macho")]
pub use craft_macho as macho;
#[cfg(feature = "markdown")]
pub use craft_markdown as markdown;
#[cfg(feature = "materialx")]
@@ -171,6 +177,8 @@ pub use craft_math as math;
pub use craft_mcmat as mcmat;
#[cfg(feature = "mcp")]
pub use craft_mcp as mcp;
#[cfg(feature = "mcp-server")]
pub use craft_mcp_server as mcp_server;
#[cfg(feature = "media")]
pub use craft_media as media;
#[cfg(feature = "menu")]
@@ -207,6 +215,8 @@ pub use craft_pcraft_format as pcraft_format;
pub use craft_pdf as pdf;
#[cfg(feature = "pdn")]
pub use craft_pdn as pdn;
#[cfg(feature = "pe")]
pub use craft_pe as pe;
#[cfg(feature = "penpot")]
pub use craft_penpot as penpot;
#[cfg(feature = "pkcs7")]
@@ -261,6 +271,8 @@ pub use craft_silhouette_studio as silhouette_studio;
pub use craft_sketch as sketch;
#[cfg(feature = "smime")]
pub use craft_smime as smime;
#[cfg(feature = "sniff")]
pub use craft_sniff as sniff;
#[cfg(feature = "stencilcraft-document")]
pub use craft_stencilcraft_document as stencilcraft_document;
#[cfg(feature = "stencilcraft-format")]
@@ -271,6 +283,8 @@ pub use craft_stream_codecs as stream_codecs;
pub use craft_svg as svg;
#[cfg(feature = "table")]
pub use craft_table as table;
#[cfg(feature = "tar")]
pub use craft_tar as tar;
#[cfg(feature = "tcraft")]
pub use craft_tcraft as tcraft;
#[cfg(feature = "text")]
+1
View File
@@ -11,6 +11,7 @@ pub const NOT_IN_FACADE: &[(&str, &str)] = &[
("fonts-build", "a build-script helper; apps call it from build.rs"),
("media-testkit", "test support: ffmpeg discovery and fixture locations for the media crates"),
("plate-test", "a test harness: reference PDFs and PDF structure checks"),
("render-parity", "test support: compares two renders of a photo against a stated tolerance"),
("story-render", "a tool: renders a component's stories to PNG files offscreen"),
("ui-bench", "a tool: the release-mode benchmark of the UI library"),
("ui-gallery", "a tool: the story gallery application"),
+16 -3
View File
@@ -69,9 +69,18 @@ impl Writer<'_> {
}
/// A shape's or text box's `wps:wsp`, at `off` in its group (or 0, 0). In a group it carries
/// its own `wps:cNvPr` with drawing id `id`.
pub(super) fn wsp(&mut self, w: &mut W, o: &InlineObject, id: Option<&str>, off: (f32, f32), rels: &mut PartRels, depth: usize) {
let InlineObject::Shape { kind, w: sw, h: sh, fill, stroke, stroke_width, story, effects, float, freeform, extra } = o else { return };
/// its own `wps:cNvPr` with drawing id `id`. Returns where in `w` its `w:txbxContent` was
/// written, for a VML fallback to repeat.
pub(super) fn wsp(
&mut self,
w: &mut W,
o: &InlineObject,
id: Option<&str>,
off: (f32, f32),
rels: &mut PartRels,
depth: usize,
) -> Option<std::ops::Range<usize>> {
let InlineObject::Shape { kind, w: sw, h: sh, fill, stroke, stroke_width, story, effects, float, freeform, extra } = o else { return None };
let geom = freeform.as_deref().filter(|_| *kind == ShapeKind::Freeform);
w.open("wps:wsp", &[]);
if let Some(id) = id {
@@ -181,6 +190,7 @@ impl Writer<'_> {
// its place in it instead.
let part = (*story).and_then(|id| self.doc.parts.get(&id).filter(|p| p.kind == craft_wordmodel::PartKind::TextBox).map(|p| (id, p)));
let link = part.and_then(|(id, _)| self.txbx_links.get(&id).copied());
let mut content = None;
if let Some((n, seq)) = link.filter(|(_, seq)| *seq > 0) {
w.empty("wps:linkedTxbx", &[("id", &n.to_string()), ("seq", &seq.to_string())]);
} else if let Some((id, part)) = part
@@ -190,6 +200,7 @@ impl Writer<'_> {
Some((n, _)) => w.open("wps:txbx", &[("id", &n.to_string())]),
None => w.open("wps:txbx", &[]),
}
let start = w.s.len();
w.open("w:txbxContent", &[]);
if self.written_boxes.insert(id) {
let blocks = part.blocks.clone();
@@ -198,6 +209,7 @@ impl Writer<'_> {
w.empty("w:p", &[]);
}
w.close("w:txbxContent");
content = Some(start..w.s.len());
w.close("wps:txbx");
self.boxes.leave();
}
@@ -231,6 +243,7 @@ impl Writer<'_> {
w.close("wps:bodyPr");
}
w.close("wps:wsp");
content
}
/// A `w:drawing` holding `graphic` (an `a:graphic`) in a frame of the object's size and place.
+190 -3
View File
@@ -1,7 +1,7 @@
use super::drawing::{DrawingInfo, shape_name, spin_attrs, write_fill, xfrm_spin_only};
use super::field::{FieldData, field_run};
use super::section::comment_reference_style;
use super::{CharProps, Fill, InlineObject, NoteKind, PartRels, TextColor, W, Wrap, Writer, emu, n, rpr};
use super::{Anchor, CharProps, Fill, Float, InlineObject, NoteKind, PartRels, ShapeKind, TextColor, W, Wrap, Writer, emu, n, rpr};
impl Writer<'_> {
/// Write an inline object (`Bookmark`, `Comment`, `Equation`, `Field`, `Shape`, etc.).
@@ -362,10 +362,24 @@ impl Writer<'_> {
}
fn write_shape(&mut self, w: &mut W, o: &InlineObject, props: &CharProps, rels: &mut PartRels, depth: usize) {
let InlineObject::Shape { kind, w: sw, h: sh, float, effects, freeform, extra, .. } = o else { return };
let InlineObject::Shape { kind, w: sw, h: sh, float, effects, freeform, extra, story, .. } = o else { return };
self.rev_open(w, props);
w.open("w:r", &[]);
self.rpr(w, props);
// A Word 2010 shape (`wps`) with a VML form goes in `mc:AlternateContent`, as Word writes
// it: a reader that doesn't know `wps` (Word 2007, the Office Compatibility Pack) takes the
// VML fallback instead of refusing the file. A shape without one stays bare: guarded with
// no fallback, that reader would open the file without the shape and drop it on its next
// save, which is worse than refusing the file.
// VML currently has no chain IDs or mso-next-textbox links. Keep every member of a
// linked story bare, including its head: an unlinked fallback would silently lose
// overflow text in older readers. Unlinked boxes still get the compatibility fallback.
let linked = story.is_some_and(|id| self.txbx_links.contains_key(&id));
let fallback = freeform.is_none() && vml_tag(*kind).is_some() && !linked;
if fallback {
w.open("mc:AlternateContent", &[]);
w.open("mc:Choice", &[("Requires", "wps")]);
}
w.open("w:drawing", &[]);
let docpr = self.shape_docpr(extra.id);
let link = Self::drawing_link(props, rels);
@@ -384,18 +398,164 @@ impl Writer<'_> {
}
let float = &float;
self.drawing_open(w, DrawingInfo { float, size: (*sw, *sh), docpr: &docpr, name: &name, alt: o.alt_text(), link: link.as_deref() });
// This shape's stacking order: floating objects in its text box move `self.z` on.
let z = self.z;
w.empty("wp:cNvGraphicFramePr", &[]);
w.open("a:graphic", &[]);
w.open("a:graphicData", &[("uri", "http://schemas.microsoft.com/office/word/2010/wordprocessingShape")]);
self.wsp(w, o, None, (0.0, 0.0), rels, depth);
let content = self.wsp(w, o, None, (0.0, 0.0), rels, depth);
w.close("a:graphicData");
w.close("a:graphic");
w.close(if float.wrap == Wrap::Inline { "wp:inline" } else { "wp:anchor" });
w.close("w:drawing");
if fallback {
w.close("mc:Choice");
// The same text again: the fallback is a copy of the choice, not more of the story.
let content = content.and_then(|r| w.s.get(r)).map(str::to_owned);
w.open("mc:Fallback", &[]);
self.vml_shape(w, o, float, z, content.as_deref());
w.close("mc:Fallback");
w.close("mc:AlternateContent");
}
w.close("w:r");
self.rev_close(w, props);
}
/// A shape as VML (`w:pict`), for readers older than Word 2010 shapes: its frame, place,
/// wrapping, fill, outline and text box. Only for a kind [`vml_tag`] names.
fn vml_shape(&self, w: &mut W, o: &InlineObject, float: &Float, z: u32, content: Option<&str>) {
let InlineObject::Shape { kind, w: sw, h: sh, fill, stroke, stroke_width, story, extra, .. } = o else { return };
let Some(tag) = vml_tag(*kind) else { return };
let pt = |v: f32| format!("{:.2}pt", craft_wordmodel::geom::finite(v));
let mut style = Vec::new();
if float.wrap != Wrap::Inline {
style.push("position:absolute".to_string());
style.push(format!("margin-left:{}", pt(float.x)));
style.push(format!("margin-top:{}", pt(float.y)));
}
style.push(format!("width:{}", pt(sw.max(0.0))));
style.push(format!("height:{}", pt(sh.max(0.0))));
let spin = float.spin();
if spin.deg != 0.0 {
style.push(format!("rotation:{}", spin.deg));
}
match (spin.flip_h, spin.flip_v) {
(true, true) => style.push("flip:x y".into()),
(true, false) => style.push("flip:x".into()),
(false, true) => style.push("flip:y".into()),
(false, false) => {}
}
if float.wrap != Wrap::Inline {
// The stacking order the choice was given, as a signed 32-bit `z-index`. Behind the text
// is negative, shifted down rather than negated, so a higher `z` still stacks higher.
let z = i64::from(z.min(i32::MAX as u32));
let z = if float.wrap == Wrap::BehindText { z - (i64::from(i32::MAX) + 1) } else { z };
style.push(format!("z-index:{z}"));
// The same clearances from the text as the choice's `distL`/`distR`/`distT`/`distB`.
let d = pt(float.dist.clamp(0.0, 1584.0));
style.push(format!("mso-wrap-distance-left:{d}"));
style.push(format!("mso-wrap-distance-right:{d}"));
style.push(format!("mso-wrap-distance-top:{}", pt(float.dist_top.clamp(0.0, 1584.0))));
style.push(format!("mso-wrap-distance-bottom:{}", pt(float.dist_bottom.clamp(0.0, 1584.0))));
if let Some(a) = float.h_align {
style.push(format!("mso-position-horizontal:{}", a.ooxml(true)));
}
style.push(format!("mso-position-horizontal-relative:{}", vml_rel(float.h_rel, true)));
if let Some(a) = float.v_align {
style.push(format!("mso-position-vertical:{}", a.ooxml(false)));
}
style.push(format!("mso-position-vertical-relative:{}", vml_rel(float.v_rel, false)));
}
let body = (*story).and_then(|id| self.doc.parts.get(&id)).map(|p| p.body).unwrap_or_default();
style.push(
match body.anchor {
craft_wordmodel::props::VAlign::Top => "v-text-anchor:top",
craft_wordmodel::props::VAlign::Center => "v-text-anchor:middle",
craft_wordmodel::props::VAlign::Bottom => "v-text-anchor:bottom",
}
.into(),
);
if float.hidden {
style.push("visibility:hidden".into());
}
let style = style.join(";");
let mut attrs: Vec<(&str, String)> = vec![("style", style)];
if *kind == ShapeKind::RoundedRectangle {
// The same fraction of the shorter side: VML's in 1/65536ths, DrawingML's in 1/100000ths.
let c = match extra.corner {
Some(c) => c.min(craft_wordmodel::connector::MAX_CORNER) as f32 / 100_000.0,
None => craft_wordmodel::connector::DEFAULT_CORNER,
};
attrs.push(("arcsize", format!("{}f", (c * 65_536.0).round() as u32)));
}
if float.wrap != Wrap::Inline {
attrs.push(("o:allowincell", (if float.outside_cell { "f" } else { "t" }).into()));
}
let opacity = match fill.as_ref() {
Some(f) => {
attrs.push(("fillcolor", format!("#{}", f.main_color().hex())));
match f {
Fill::Solid { transparency, .. } if *transparency > 0.0 => Some(format!("{:.4}", (1.0 - transparency / 100.0).clamp(0.0, 1.0))),
_ => None,
}
}
None => {
attrs.push(("filled", "f".into()));
None
}
};
match stroke {
Some(c) => {
attrs.push(("strokecolor", format!("#{}", c.hex())));
attrs.push(("strokeweight", pt(stroke_width.clamp(0.0, 100.0))));
}
None => attrs.push(("stroked", "f".into())),
}
let attrs: Vec<(&str, &str)> = attrs.iter().map(|(k, v)| (*k, v.as_str())).collect();
w.open("w:pict", &[]);
w.open(tag, &attrs);
if let Some(opacity) = &opacity {
w.empty("v:fill", &[("opacity", opacity)]);
}
if let Some(content) = content {
let inset = body.inset.map(|i| i.map(|v| pt(v.min(craft_wordmodel::MAX_TEXT_BOX_INSET) as f32 / 12_700.0)).join(","));
let mut tb: Vec<(&str, &str)> = Vec::new();
if let Some(inset) = &inset {
tb.push(("inset", inset));
}
// Text direction as the VML reader takes it (stacked letters have no VML form).
let mut tb_style = match body.vert {
craft_wordmodel::props::TextVert::Vert => vec!["layout-flow:vertical"],
craft_wordmodel::props::TextVert::Vert270 => vec!["layout-flow:vertical", "mso-layout-flow-alt:bottom-to-top"],
craft_wordmodel::props::TextVert::Horz | craft_wordmodel::props::TextVert::Stacked => Vec::new(),
};
if body.auto_fit {
tb_style.push("mso-fit-shape-to-text:t");
}
let tb_style = tb_style.join(";");
if !tb_style.is_empty() {
tb.push(("style", &tb_style));
}
w.open("v:textbox", &tb);
w.raw(content);
w.close("v:textbox");
}
let wrap = match float.wrap {
Wrap::Square => Some("square"),
Wrap::Tight => Some("tight"),
Wrap::Through => Some("through"),
Wrap::TopAndBottom => Some("topAndBottom"),
Wrap::Inline | Wrap::BehindText | Wrap::InFrontOfText => None,
};
if let Some(wrap) = wrap {
let side = float.wrap_text.ooxml();
let attrs: &[(&str, &str)] = if float.wrap_text.is_both() { &[("type", wrap)] } else { &[("type", wrap), ("side", side)] };
w.empty("w10:wrap", attrs);
}
w.close(tag);
w.close("w:pict");
}
fn write_group(&mut self, w: &mut W, o: &InlineObject, props: &CharProps, rels: &mut PartRels, depth: usize) {
let InlineObject::Group { w: gw, h: gh, float, ch_w, ch_h, children, canvas } = o else { return };
self.rev_open(w, props);
@@ -487,6 +647,33 @@ impl Writer<'_> {
}
}
/// The VML element that draws a shape kind, for the kinds VML has one for.
fn vml_tag(kind: ShapeKind) -> Option<&'static str> {
match kind {
ShapeKind::Rectangle | ShapeKind::TextBox => Some("v:rect"),
ShapeKind::RoundedRectangle => Some("v:roundrect"),
ShapeKind::Ellipse => Some("v:oval"),
_ => None,
}
}
/// A floating object's anchor as VML names it (`mso-position-horizontal-relative`, `-vertical-`).
fn vml_rel(a: Anchor, horiz: bool) -> &'static str {
match (a, horiz) {
(Anchor::Margin, _) => "margin",
(Anchor::Page, _) => "page",
(Anchor::InsideMargin, _) => "inner-margin-area",
(Anchor::OutsideMargin, _) => "outer-margin-area",
(Anchor::LeftMargin, true) => "left-margin-area",
(Anchor::RightMargin, true) => "right-margin-area",
(Anchor::Character, true) => "char",
(Anchor::TopMargin, false) => "top-margin-area",
(Anchor::BottomMargin, false) => "bottom-margin-area",
(Anchor::Line, false) => "line",
_ => "text",
}
}
pub(super) fn ink_name(tool: craft_wordmodel::freeform::InkTool) -> &'static str {
match tool {
craft_wordmodel::freeform::InkTool::Pen => "Pen",
+1
View File
@@ -49,3 +49,4 @@ mod undefined_style;
mod vml_shapes;
mod word_xml;
mod wordml2003_write;
mod wps_fallback;
+3 -2
View File
@@ -956,7 +956,8 @@ fn toc_field_skips_nested_stories() {
let mut xml = String::new();
std::io::Read::read_to_string(&mut zip.by_name("word/document.xml").unwrap(), &mut xml).unwrap();
let boxes: Vec<(usize, usize)> = xml.match_indices("<w:txbxContent>").map(|(a, _)| (a, a + xml[a..].find("</w:txbxContent>").unwrap())).collect();
assert_eq!(boxes.len(), 2, "{xml}");
// Each box twice: its shape and its VML fallback.
assert_eq!(boxes.len(), 4, "{xml}");
let in_box = |i: usize| boxes.iter().any(|(a, b)| (*a..*b).contains(&i));
for kind in ["begin", "separate", "end"] {
let at: Vec<usize> = xml.match_indices(&format!(r#"fldCharType="{kind}""#)).map(|(i, _)| i).collect();
@@ -964,7 +965,7 @@ fn toc_field_skips_nested_stories() {
assert!(!in_box(at[0]), "field {kind} written inside a text box: {xml}");
}
let (begin, end) = (xml.find(r#"fldCharType="begin""#).unwrap(), xml.find(r#"fldCharType="end""#).unwrap());
assert!(begin > boxes[0].1 && end > boxes[1].1 && end < xml.find("</w:sdt>").unwrap(), "{xml}");
assert!(begin > boxes[1].1 && end > boxes[3].1 && end < xml.find("</w:sdt>").unwrap(), "{xml}");
}
#[test]
+2 -1
View File
@@ -257,5 +257,6 @@ fn story_shared_by_many_shapes_is_written_once() {
}
let xml = document_xml(&craft_docx::write(&d).unwrap());
assert!(xml.len() < 2_000_000, "document.xml is {} bytes", xml.len());
assert_eq!(xml.matches(text.as_str()).count(), 1);
// Once in the first shape and once in its VML fallback; the other shapes don't repeat it.
assert_eq!(xml.matches(text.as_str()).count(), 2);
}
+188
View File
@@ -0,0 +1,188 @@
//! Word 2010 shapes (`wps:wsp`) are written inside `mc:AlternateContent` with a VML fallback
//! (ECMA-376 Part 3, Markup Compatibility; Part 4 §14.1, VML), so a reader that doesn't know
//! `wps` — Word 2007, or Word 2000-2003 with the Office Compatibility Pack — takes the fallback
//! instead of refusing the whole file. A shape with no VML form is left bare.
use std::io::Read;
use craft_wordmodel::para::{Anchor, FloatAlign, ShapeKind, Wrap};
use craft_wordmodel::props::TextVert;
use craft_wordmodel::{Document, InlineObject, PartKind, TextBody, para_block};
fn document_xml(bytes: &[u8]) -> String {
let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap();
let mut s = String::new();
z.by_name("word/document.xml").unwrap().read_to_string(&mut s).unwrap();
s
}
fn shape(kind: ShapeKind, story: Option<u32>) -> InlineObject {
let float = craft_wordmodel::para::Float {
wrap: Wrap::Square,
h_rel: Anchor::Margin,
h_align: Some(FloatAlign::End),
v_rel: Anchor::Paragraph,
y: 0.05,
..Default::default()
};
InlineObject::Shape {
kind,
w: 10.7,
h: 12.2,
fill: None,
stroke: None,
stroke_width: 0.75,
float,
story,
effects: Default::default(),
freeform: None,
extra: Default::default(),
}
}
/// A page number in a floating text box, as in a footer: the shape that made the
/// Compatibility Pack refuse the file.
#[test]
fn a_text_box_gets_a_vml_fallback_with_its_text() {
let mut d = Document::from_text("a");
let id = d.add_part(PartKind::TextBox, vec![para_block(craft_wordmodel::Paragraph::with_text("29", Default::default()))]);
d.parts.get_mut(&id).unwrap().body = TextBody { auto_fit: true, inset: Some([0; 4]), ..Default::default() };
let mut o = shape(ShapeKind::TextBox, Some(id));
if let InlineObject::Shape { float, .. } = &mut o {
(float.dist, float.dist_top, float.dist_bottom) = (18.0, 12.0, 6.0);
}
d.insert_object(&craft_wordmodel::Pos::body(0, 1), o, &Default::default()).unwrap();
let bytes = craft_docx::write(&d).unwrap();
let xml = document_xml(&bytes);
assert!(xml.contains(r#"<mc:AlternateContent><mc:Choice Requires="wps"><w:drawing>"#), "{xml}");
assert!(!xml.contains("<w:r><w:drawing>"), "no bare Word 2010 shape: {xml}");
let fallback = &xml[xml.find("<mc:Fallback>").expect("a fallback")..];
assert!(fallback.starts_with("<mc:Fallback><w:pict><v:rect "), "{fallback}");
for want in [
"position:absolute",
"mso-position-horizontal:right",
"mso-position-horizontal-relative:margin",
"mso-position-vertical-relative:text",
"mso-wrap-distance-left:18.00pt;mso-wrap-distance-right:18.00pt;mso-wrap-distance-top:12.00pt;mso-wrap-distance-bottom:6.00pt",
r#"filled="f""#,
r#"stroked="f""#,
r#"<v:textbox inset="0.00pt,0.00pt,0.00pt,0.00pt" style="mso-fit-shape-to-text:t"><w:txbxContent>"#,
r#"<w10:wrap type="square"/></v:rect></w:pict></mc:Fallback></mc:AlternateContent>"#,
] {
assert!(fallback.contains(want), "{want} in {fallback}");
}
assert_eq!(xml.matches(">29</w:t>").count(), 2, "the text in the choice and again in the fallback");
// Read back: the choice, once.
let r = craft_docx::read(&bytes).unwrap();
let mut boxes = Vec::new();
r.objects_in_reading_order(&mut |o| boxes.extend(o.text_boxes()));
boxes.retain(|id| r.parts.get(id).is_some_and(|p| p.kind == PartKind::TextBox));
assert_eq!(boxes.len(), 1, "one text box, not the fallback as well");
assert_eq!(r.plain_text(craft_wordmodel::StoryRef::Part(boxes[0])), "29");
}
/// The fallback keeps the box's own stacking order (not that of a floating object inside it),
/// its hidden flag and its text direction.
#[test]
fn the_fallback_keeps_order_visibility_and_direction() {
let mut d = Document::from_text("a");
let inner = d.add_part(PartKind::TextBox, vec![para_block(craft_wordmodel::Paragraph::with_text("inner", Default::default()))]);
let mut nested = shape(ShapeKind::TextBox, Some(inner));
if let InlineObject::Shape { float, .. } = &mut nested {
float.z = 100;
}
let mut p = craft_wordmodel::Paragraph::with_text("outer", Default::default());
p.insert_object(0, nested, &Default::default()).unwrap();
let outer = d.add_part(PartKind::TextBox, vec![para_block(p)]);
d.parts.get_mut(&outer).unwrap().body = TextBody { vert: TextVert::Vert270, ..Default::default() };
let mut o = shape(ShapeKind::TextBox, Some(outer));
if let InlineObject::Shape { float, .. } = &mut o {
float.z = 10;
float.hidden = true;
}
d.insert_object(&craft_wordmodel::Pos::body(0, 1), o, &Default::default()).unwrap();
let xml = document_xml(&craft_docx::write(&d).unwrap());
// The outer box's VML: the `v:rect` holding the vertical text box (the inner box's fallback
// is copied into the outer one's text, so it comes after it).
let tb = xml.find(r#"<v:textbox style="layout-flow:vertical;mso-layout-flow-alt:bottom-to-top">"#).expect("the outer box's text, turned");
let rect = &xml[xml[..tb].rfind("<v:rect ").expect("its v:rect")..tb];
assert!(rect.contains("z-index:10;"), "{rect}");
assert!(rect.contains("visibility:hidden"), "{rect}");
}
/// Shapes behind the text keep their order in the fallback: a higher `z` stacks higher, below
/// the text (a negative `z-index` in signed 32-bit range).
#[test]
fn behind_text_shapes_keep_their_order() {
let mut d = Document::from_text("a");
for z in [10, 20, u32::MAX] {
let mut o = shape(ShapeKind::Rectangle, None);
if let InlineObject::Shape { float, .. } = &mut o {
float.wrap = Wrap::BehindText;
float.z = z;
}
d.insert_object(&craft_wordmodel::Pos::body(0, 1), o, &Default::default()).unwrap();
}
let xml = document_xml(&craft_docx::write(&d).unwrap());
let mut got: Vec<i64> = xml
.match_indices("z-index:")
.map(|(i, m)| {
let v = &xml[i + m.len()..];
v[..v.find([';', '"']).unwrap()].parse().unwrap()
})
.collect();
got.sort_unstable();
let low = -(1i64 << 31);
assert_eq!(got, vec![low + 10, low + 20, -1], "{xml}");
}
/// A shape VML has no simple form for stays a bare `wps:wsp`: guarded without a fallback, an old
/// reader would open the file without it and drop it on its next save.
#[test]
fn a_shape_without_a_vml_form_is_not_guarded() {
let mut d = Document::from_text("a");
d.insert_object(&craft_wordmodel::Pos::body(0, 1), shape(ShapeKind::Star, None), &Default::default()).unwrap();
let bytes = craft_docx::write(&d).unwrap();
let xml = document_xml(&bytes);
assert!(xml.contains("<w:r><w:drawing>") && xml.contains(r#"<a:prstGeom prst="star5">"#), "{xml}");
assert!(!xml.contains("<mc:AlternateContent>"), "{xml}");
let r = craft_docx::read(&bytes).unwrap();
let mut kinds = Vec::new();
r.objects_in_reading_order(&mut |o| {
if let InlineObject::Shape { kind, .. } = o {
kinds.push(*kind);
}
});
assert_eq!(kinds, vec![ShapeKind::Star]);
}
/// A partial fallback must not turn a linked story into independent boxes in older readers.
#[test]
fn linked_boxes_stay_bare_while_unlinked_boxes_keep_their_fallback() {
let mut d = Document::from_text("a\nb\nc\nd");
let mut ids = Vec::new();
for (i, text) in ["Flows on", "", "", "Independent"].into_iter().enumerate() {
let id = d.add_part(PartKind::TextBox, vec![para_block(craft_wordmodel::Paragraph::with_text(text, Default::default()))]);
ids.push(id);
d.insert_object(&craft_wordmodel::Pos::body(i, 1), shape(ShapeKind::TextBox, Some(id)), &Default::default()).unwrap();
}
d.parts.get_mut(&ids[0]).unwrap().body.next = Some(ids[1]);
d.parts.get_mut(&ids[1]).unwrap().body.next = Some(ids[2]);
let bytes = craft_docx::write(&d).unwrap();
let xml = document_xml(&bytes);
assert_eq!(xml.matches("<mc:AlternateContent>").count(), 1, "{xml}");
assert_eq!(xml.matches("<mc:Fallback>").count(), 1, "{xml}");
let fallback = xml.split("<mc:Fallback>").nth(1).unwrap().split("</mc:Fallback>").next().unwrap();
assert!(fallback.contains(">Independent</w:t>"), "{fallback}");
assert!(!fallback.contains("Flows on"), "the chain head must not get an unlinked fallback");
assert_eq!(xml.matches(">Flows on</w:t>").count(), 1);
assert!(xml.contains(r#"<wps:txbx id="1">"#), "{xml}");
assert!(xml.contains(r#"<wps:linkedTxbx id="1" seq="1"/>"#), "{xml}");
assert!(xml.contains(r#"<wps:linkedTxbx id="1" seq="2"/>"#), "{xml}");
let read = craft_docx::read(&bytes).unwrap();
let chains = read.text_box_chains();
assert_eq!(chains.len(), 1);
assert_eq!(chains[0].len(), 3);
assert_eq!(read.plain_text(craft_wordmodel::StoryRef::Part(chains[0][0])), "Flows on");
}
+14 -2
View File
@@ -9,6 +9,14 @@ pub enum Rgba8EncodeError {
/// The output buffer could not be allocated.
OutOfMemory,
}
/// Reserves the DPX output up front; a failed reservation becomes [`Rgba8EncodeError::OutOfMemory`].
fn output_buffer(size: usize) -> std::result::Result<Vec<u8>, Rgba8EncodeError> {
let mut buf = Vec::new();
match buf.try_reserve_exact(size) {
Ok(()) => Ok(buf),
Err(_) => Err(Rgba8EncodeError::OutOfMemory),
}
}
fn checked_len(rgba: &[u8], w: u32, h: u32) -> std::result::Result<usize, Rgba8EncodeError> {
let n = (w as usize).checked_mul(h as usize).and_then(|n| n.checked_mul(4)).ok_or(Rgba8EncodeError::FrameSize)?;
if w == 0 || h == 0 || rgba.len() != n {
@@ -86,8 +94,7 @@ pub fn encode_rgba8_dpx10(rgba: &[u8], w: u32, h: u32, creator: &str) -> std::re
hd[1928] = 0;
// The caller already holds the frame; reserve the output fallibly so a large export reports
// an error instead of aborting the process.
let mut out = Vec::new();
out.try_reserve_exact(size).map_err(|_| Rgba8EncodeError::OutOfMemory)?;
let mut out = output_buffer(size)?;
out.extend_from_slice(&hd);
let ten = |v: u8| (u32::from(v) << 2) | (u32::from(v) >> 6);
for &[r, g, b, _] in rgba.as_chunks::<4>().0 {
@@ -107,4 +114,9 @@ mod tests {
assert_eq!(&bytes[2048..], &((1023u32 << 22) | (514u32 << 2)).to_be_bytes());
assert_eq!(encode_rgba8_dpx10(&[], 0, 1, ""), Err(Rgba8EncodeError::FrameSize));
}
#[test]
fn output_reservation_failure_maps_to_out_of_memory() {
assert!(matches!(output_buffer(usize::MAX), Err(Rgba8EncodeError::OutOfMemory)));
assert_eq!(output_buffer(DPX_HEADER).map(|b| b.capacity() >= DPX_HEADER), Ok(true));
}
}
+2 -2
View File
@@ -184,7 +184,7 @@ impl CropKept {
/// drawn one.
/// - Dismissal ([`dismiss`](Self::dismiss)): after Discard or an accepted crop the default frame
/// is still made but stays hidden until the next press, ratio change or size change
/// ([`show`](Self::show)), for that document and size only.
/// ([`show_frame`](Self::show_frame)), for that document and size only.
/// - [`CropTool`](super::CropTool) is the one writer: the session has no edit type of its own.
#[derive(Clone, Debug, Default)]
pub struct CropSession {
@@ -464,7 +464,7 @@ impl CropSession {
}
/// Discard or an accepted crop: the session is cleared and the document's default frame stays
/// hidden until [`show`](Self::show). The next [`ensure_frame`](Self::ensure_frame) still makes
/// hidden until [`show_frame`](Self::show_frame). The next [`ensure_frame`](Self::ensure_frame) still makes
/// the frame, so the hidden overlay has one to show.
pub fn dismiss(&mut self) {
let made_for = self.frame_for;
+1 -1
View File
@@ -109,7 +109,7 @@ pub(super) fn ruler_cursor(precise: bool, protractor: bool) -> Cursor {
/// Shift 10, Delete and Backspace clear the line.
/// - It paints a cross at each point, in the line's look; the app shows the line and the arm with `StraightLinePreview`
/// ([`RulerPass::show_line`](super::RulerPass::show_line)).
/// - [Read only](Self::read_only), it paints the crosses at rest and takes no press, key or focus: the ruler drawn
/// - [Read only](craft_ui::contract::Props::read_only), it paints the crosses at rest and takes no press, key or focus: the ruler drawn
/// under the other tools ([`show_ruler`](super::show_ruler)).
/// - **AccessKit:** the group's value reads "Length L, angle A°" in the ruler's units; each point is named and valued
/// "x, y" in the ruler's units.
+1 -3
View File
@@ -276,9 +276,7 @@ fn the_gear_enables_only_the_chosen_sizes_and_escape_keeps_the_pick() {
#[trial]
fn a_fixed_size_typed_past_its_range_keeps_the_end_and_reports_it_once_and_an_emptied_field_is_empty() {
let mut start = BoxConstraints::default();
start.constraint = BoxConstraint::FixedSize;
start.fixed = (Some(100.0), Some(50.0));
let start = BoxConstraints { constraint: BoxConstraint::FixedSize, fixed: (Some(100.0), Some(50.0)), ..Default::default() };
let (mut fixture, host) = gear(start);
type_into(&mut fixture, "Fixed Size W", "400000");
assert_eq!(host.borrow().value.fixed, (Some(300_000.0), Some(50.0)), "a width past 300,000 keeps 300,000");
+1 -2
View File
@@ -161,8 +161,7 @@ fn shift_squares_the_box_from_the_shorter_side() {
/// A Rectangle drag under the gear `constraint` with `change` applied to the gear's values.
fn gear(constraint: BoxConstraint, change: impl FnOnce(&mut BoxConstraints)) -> Shapes {
let mut constraints = BoxConstraints::default();
constraints.constraint = constraint;
let mut constraints = BoxConstraints { constraint, ..Default::default() };
change(&mut constraints);
Shapes::new(ShapeDragOptions::new(ShapeKind::Rectangle).constraints(constraints))
}
+1 -1
View File
@@ -776,7 +776,7 @@ fn a_secondary_click_on_the_box_opens_the_mode_menu() {
});
});
fixture.settle();
let mut right_click = |fixture: &mut Fixture, at: Pos2| {
let right_click = |fixture: &mut Fixture, at: Pos2| {
fixture.frame(vec![pointer::moved(at)]);
fixture.frame(vec![pointer::secondary(at, true)]);
fixture.frame(vec![pointer::secondary(at, false)]);
+20
View File
@@ -354,6 +354,10 @@ pub struct Chart {
pub legend: Option<String>,
/// `c:legend/c:overlay`: the legend is drawn over the plot instead of shrinking it.
pub legend_overlay: bool,
/// `c:legend/c:legendEntry`: entries whose text has its own size, weight or colour, over
/// the legend's.
#[serde(skip_serializing_if = "Vec::is_empty")]
pub legend_entries: Vec<LegendEntry>,
/// Text size in points of the axis labels and legend (from their `c:txPr`, else the chart's);
/// `None`: the file gives none.
#[serde(skip_serializing_if = "Option::is_none")]
@@ -431,6 +435,21 @@ pub struct Chart {
pub kept: ChartKept,
}
/// One legend entry's own text (`c:legendEntry/c:txPr`): what it sets wins over the legend's.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(default, rename_all = "camelCase")]
pub struct LegendEntry {
/// The entry's place in the legend, from 0 (`c:idx`).
pub idx: u32,
/// Text size in points.
#[serde(skip_serializing_if = "Option::is_none")]
pub size: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub bold: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub color: Option<ColorRef>,
}
/// One relationship of a chart part, with the target string as it was written.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
@@ -483,6 +502,7 @@ impl Default for Chart {
series: vec![],
legend: Some("b".into()),
legend_overlay: false,
legend_entries: vec![],
text_size: None,
title_size: None,
data_labels: false,
+47
View File
@@ -143,6 +143,7 @@ fn chart_within(root: &El, mut budget: usize) -> Option<Chart> {
c.legend = ch.child(Local("legend")).map(|l| l.child(Local("legendPos")).and_then(|c| c.attr_local("val")).unwrap_or("r").to_string());
c.legend_overlay =
ch.child(Local("legend")).and_then(|l| l.child(Local("overlay"))).and_then(|o| o.attr_local("val").and_then(value::to_bool)).unwrap_or(false);
c.legend_entries = legend_entries(ch.child(Local("legend")));
c.chart_fill = simple_fill(root.child(Local("spPr")));
c.plot_fill = simple_fill(pa.child(Local("spPr")));
@@ -552,6 +553,52 @@ fn text_color(e: Option<&El>) -> Option<crate::style::ColorRef> {
rec(e?, 0)
}
/// Legend entries read from one chart; a legend lists at most one per series or category.
const MAX_LEGEND_ENTRIES: usize = 1024;
/// The legend's entries with text of their own (`c:legendEntry/c:txPr`), over the legend's.
fn legend_entries(legend: Option<&El>) -> Vec<crate::chart::LegendEntry> {
legend
.into_iter()
.flat_map(|l| l.children(Local("legendEntry")))
.take(MAX_LEGEND_ENTRIES)
.filter_map(|e| {
// The paragraph's own run properties first, then the list style's defaults
// (`a:lstStyle` comes before `a:p`, and is the less specific of the two).
let tx = e.child(Local("txPr"));
let para = tx.and_then(|t| t.child(Local("p")));
let entry = crate::chart::LegendEntry {
idx: e.child(Local("idx")).and_then(|i| i.attr_local("val")).and_then(|v| v.parse().ok())?,
size: text_size(para).or_else(|| text_size(tx)),
bold: text_bold(para).or_else(|| text_bold(tx)),
color: text_color(para).or_else(|| text_color(tx)),
};
(entry.size.is_some() || entry.bold.is_some() || entry.color.is_some()).then_some(entry)
})
.collect()
}
/// The weight (`b`) of the first `a:defRPr` or `a:rPr` that sets one under `e`.
fn text_bold(e: Option<&El>) -> Option<bool> {
fn rec(e: &El, depth: usize) -> Option<bool> {
if depth > 8 {
return None;
}
for c in e.elements() {
if (c.is(Local("defRPr")) || c.is(Local("rPr")))
&& let Some(b) = c.attr_local("b").and_then(value::to_bool)
{
return Some(b);
}
if let Some(v) = rec(c, depth + 1) {
return Some(v);
}
}
None
}
rec(e?, 0)
}
/// The size in points of the first `a:defRPr` or `a:rPr` with one under `e` (`sz` in hundredths).
fn text_size(e: Option<&El>) -> Option<f64> {
fn rec(e: &El, depth: usize) -> Option<f64> {
+101
View File
@@ -101,6 +101,38 @@ fn tx_pr(w: &mut Writer, c: &Chart) {
w.close();
}
/// `c:legendEntry` with the entry's own text size, weight and colour.
fn legend_entry(w: &mut Writer, e: &crate::chart::LegendEntry) {
w.open("c:legendEntry", &[]);
w.tag("c:idx").attr("val", e.idx).empty();
w.open("c:txPr", &[]);
w.empty("a:bodyPr", &[]);
w.empty("a:lstStyle", &[]);
w.open("a:p", &[]);
w.open("a:pPr", &[]);
let mut run = w.tag("a:defRPr");
if let Some(sz) = e.size.filter(|v| v.is_finite() && *v > 0.0) {
// 1 to 400 pt, as the reader takes it, so a size reads back as written.
run = run.attr("sz", (sz * 100.0).round().clamp(100.0, 40_000.0) as u32);
}
if let Some(b) = e.bold {
run = run.attr("b", u8::from(b));
}
match &e.color {
Some(c) => {
run.open();
dml::solid(w, c);
w.close();
}
None => run.empty(),
}
w.close();
w.tag("a:endParaRPr").attr("lang", "en-US").empty();
w.close();
w.close();
w.close();
}
fn rich(w: &mut Writer, text: &str, color: Option<&crate::style::ColorRef>) {
w.open("c:tx", &[]);
w.open("c:rich", &[]);
@@ -781,6 +813,9 @@ pub fn chart_xml(c: &Chart) -> (Vec<u8>, bool) {
};
w.open("c:legend", &[]);
w.tag("c:legendPos").attr("val", p).empty();
for e in &c.legend_entries {
legend_entry(&mut w, e);
}
w.tag("c:overlay").attr("val", u8::from(c.legend_overlay)).empty();
tx_pr(&mut w, c);
w.close();
@@ -1058,6 +1093,72 @@ mod tests {
use crate::color::{ColorTransform, Rgba};
use crate::style::ColorRef;
/// A legend entry's own text (`c:legendEntry/c:txPr`: size, bold, colour) is read, and a
/// rebuilt part writes it back, in schema order before `c:overlay` (DeckCraft #2067).
#[test]
fn legend_entry_text_is_read_and_survives_an_edit() {
let read = |xml: &[u8]| crate::chart::read::read_chart_xml(xml).unwrap();
let entry = |idx: u32, rgb: &str| {
format!(
r#"<c:legendEntry><c:idx val="{idx}"/><c:txPr><a:bodyPr/><a:lstStyle/><a:p><a:pPr><a:defRPr sz="2000" b="1"><a:solidFill><a:srgbClr val="{rgb}"/></a:solidFill></a:defRPr></a:pPr><a:endParaRPr lang="en-US"/></a:p></c:txPr></c:legendEntry>"#
)
};
let xml = format!(
r#"<c:chartSpace xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"><c:chart><c:plotArea><c:lineChart><c:grouping val="standard"/><c:ser><c:idx val="0"/><c:val><c:numRef><c:numCache><c:ptCount val="1"/><c:pt idx="0"><c:v>1</c:v></c:pt></c:numCache></c:numRef></c:val></c:ser><c:axId val="1"/><c:axId val="2"/></c:lineChart><c:catAx><c:axId val="1"/><c:crossAx val="2"/></c:catAx><c:valAx><c:axId val="2"/><c:crossAx val="1"/></c:valAx></c:plotArea><c:legend><c:legendPos val="r"/>{}{}<c:legendEntry><c:idx val="2"/><c:txPr><a:bodyPr/><a:p><a:pPr><a:defRPr/></a:pPr></a:p></c:txPr></c:legendEntry><c:overlay val="0"/><c:txPr><a:bodyPr/><a:p><a:pPr><a:defRPr sz="1200"><a:solidFill><a:srgbClr val="7F7F7F"/></a:solidFill></a:defRPr></a:pPr></a:p></c:txPr></c:legend></c:chart></c:chartSpace>"#,
entry(0, "C00000"),
entry(1, "00B050")
);
let c = read(xml.as_bytes());
let want = |idx: u32, rgb: (u8, u8, u8)| crate::chart::LegendEntry {
idx,
size: Some(20.0),
bold: Some(true),
color: Some(ColorRef::rgb(Rgba::rgb(rgb.0, rgb.1, rgb.2))),
};
// The third entry sets nothing of its own and is not kept.
assert_eq!(c.legend_entries, vec![want(0, (0xC0, 0, 0)), want(1, (0, 0xB0, 0x50))]);
assert_eq!(c.text_size, Some(12.0), "the legend's own size stays the legend's");
let mut e = c.clone();
e.title = Some("Units".into());
e.raw = None;
let (out, kept) = super::chart_xml(&e);
assert!(!kept);
let s = String::from_utf8(out.clone()).unwrap();
let legend = &s[s.find("<c:legend>").unwrap()..];
let (pos, entry0, overlay) =
(legend.find("<c:legendPos").unwrap(), legend.find("<c:legendEntry>").unwrap(), legend.find("<c:overlay").unwrap());
assert!(pos < entry0 && entry0 < overlay, "{s}");
assert_eq!(read(&out).legend_entries, c.legend_entries);
// None: nothing written.
e.legend_entries.clear();
assert!(!String::from_utf8(super::chart_xml(&e).0).unwrap().contains("legendEntry"));
// The paragraph's own properties win over the list style's, which come first in the XML.
let listed = xml.replacen(
r#"<c:legendEntry><c:idx val="0"/><c:txPr><a:bodyPr/><a:lstStyle/>"#,
r#"<c:legendEntry><c:idx val="0"/><c:txPr><a:bodyPr/><a:lstStyle><a:lvl1pPr><a:defRPr sz="1200" b="0"><a:solidFill><a:srgbClr val="0000FF"/></a:solidFill></a:defRPr></a:lvl1pPr></a:lstStyle>"#,
1,
);
assert_ne!(listed, xml, "the list style was put in");
let c = read(listed.as_bytes());
assert_eq!(c.legend_entries[0], want(0, (0xC0, 0, 0)), "the paragraph's 20 pt, bold, C00000");
// The list style alone still counts.
let only_list =
listed.replacen(r#"<a:defRPr sz="2000" b="1"><a:solidFill><a:srgbClr val="C00000"/></a:solidFill></a:defRPr>"#, "<a:defRPr/>", 1);
let entry = &read(only_list.as_bytes()).legend_entries[0];
assert_eq!((entry.size, entry.bold), (Some(12.0), Some(false)));
// A size reads back as it was written, within the reader's 1-400 pt.
for (size, back) in [(20.5, 20.5), (400.0, 400.0), (500.0, 400.0)] {
let mut e = c.clone();
e.raw = None;
e.legend_entries = vec![crate::chart::LegendEntry { idx: 0, size: Some(size), bold: Some(false), color: None }];
let written = super::chart_xml(&e).0;
let sz = format!(r#"sz="{}""#, (back * 100.0) as u32);
assert!(String::from_utf8_lossy(&written).contains(&sz), "{size} pt is written as {sz}");
let back_entry = &read(&written).legend_entries[0];
assert_eq!((back_entry.size, back_entry.bold), (Some(back), Some(false)), "{size} pt");
}
}
#[test]
fn export_cost_counts_the_caches_each_group_writes() {
let s = Series { values: vec![Some(1.0); 600], ..Default::default() };
+22 -5
View File
@@ -234,16 +234,18 @@ fn sci(toks: &[Tok], v: f64, sep: char, dec: char) -> Option<String> {
m *= 10.0;
exp -= 1;
}
// Rounding the mantissa to its digits can reach 10 (9.999 as `0.00`).
if format!("{m:.mant_frac$}").starts_with("10") {
m /= 10.0;
exp += 1;
}
}
let step = mant.iter().take_while(|t| **t != Tok::Dot).filter(|t| matches!(t, Tok::Digit(_))).count().max(1) as i32;
let shift = exp.rem_euclid(step);
exp -= shift;
m *= 10f64.powi(shift);
// Round after forming the engineering mantissa: 999.94 as `##0.0` stays 999.9.
// Carry only when rounding exceeds all of the format's integer placeholders.
let carry = 10f64.powi(step);
if format!("{m:.mant_frac$}").parse::<f64>().is_ok_and(|rounded| rounded >= carry) {
m /= carry;
exp += step;
}
let mantissa = fixed(mant, mfirst, mlast, m, sep, dec);
let sign = if exp < 0 {
"-"
@@ -404,6 +406,21 @@ mod tests {
assert_eq!(format("##0.0E+0", 999.96, None), "1.0E+3");
}
#[test]
fn engineering_formats_round_the_full_mantissa() {
for (code, value, expected) in [
("##0.0E+00", 999.94, "999.9E+00"),
("##0.0E+00", 0.99994, "999.9E-03"),
("##0.0E+00", -999.94, "-999.9E+00"),
("#0.0E+00", 99.94, "99.9E+00"),
("##0.0E+00", 999.96, "1.0E+03"),
("#0.0E+00", 99.96, "1.0E+02"),
] {
assert_eq!(format(code, value, None), expected, "{code}, {value}");
}
assert_eq!(format("##0.0E+00", 999.94, Some("de-DE")), "999,9E+00");
}
#[test]
fn never_panics() {
for code in ["", ";", ";;;", "\"", "[", "\\", "0.", ".", "#,", "%", "_", "*", "@", "0E+00", "[$", "\"unterminated"] {
+23
View File
@@ -0,0 +1,23 @@
[package]
name = "craft-elf"
description = "ELF program headers and dynamic section entries from hostile bytes, from Craft Anything: 32/64-bit, either byte order, with bounded counts and overflow-checked offsets, plus segment, permission and dynamic tag names. Pure Rust, no I/O."
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
thiserror = { workspace = true }
# `serde`: `Serialize` on `ProgramHeader` (addresses as `0x…` strings), for apps that report
# segments as JSON.
serde = { workspace = true, optional = true }
[features]
serde = ["dep:serde"]
[dev-dependencies]
serde_json = { workspace = true }
+37
View File
@@ -0,0 +1,37 @@
//! Bounds-checked reads of fixed-width integers in both byte orders: out of range is `None`.
#[derive(Debug, Clone, Copy)]
pub(crate) struct Reader<'a> {
buf: &'a [u8],
}
macro_rules! int {
($le:ident, $be:ident, $ty:ty) => {
pub(crate) fn $le(&self, at: usize) -> Option<$ty> {
self.take(at).map(<$ty>::from_le_bytes)
}
pub(crate) fn $be(&self, at: usize) -> Option<$ty> {
self.take(at).map(<$ty>::from_be_bytes)
}
};
}
impl<'a> Reader<'a> {
pub(crate) fn new(buf: &'a [u8]) -> Self {
Self { buf }
}
pub(crate) fn u8(&self, at: usize) -> Option<u8> {
self.buf.get(at).copied()
}
fn take<const N: usize>(&self, at: usize) -> Option<[u8; N]> {
let end = at.checked_add(N)?;
self.buf.get(at..end)?.try_into().ok()
}
int!(u16_le, u16_be, u16);
int!(u32_le, u32_be, u32);
int!(u64_le, u64_be, u64);
}
+5
View File
@@ -0,0 +1,5 @@
//! Addresses serialised as `0x…` strings, as Craft Anything reports them.
pub(crate) fn serialize<S: serde::Serializer>(value: &u64, serializer: S) -> Result<S::Ok, S::Error> {
serializer.serialize_str(&format!("{value:#x}"))
}
+244
View File
@@ -0,0 +1,244 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
//! ELF program headers and dynamic section entries from untrusted bytes, 32- or 64-bit in either
//! byte order: [`program_headers`], [`dynamic_entries`], [`va_to_off`], and the segment type,
//! permission and dynamic tag names.
//!
//! From Craft Anything (`crates/formats/src/elf.rs`), where the mitigations, dynamic symbols,
//! symbol versions and core-file notes built on these stay. With the `serde` feature,
//! [`ProgramHeader`] is `Serialize` with `vaddr` as a `0x…` string.
mod bytes;
#[cfg(feature = "serde")]
mod hex;
use crate::bytes::Reader;
/// Program headers read from one table.
pub const MAX_PHDRS: usize = 4096;
/// Dynamic entries read from one segment.
pub const MAX_DYN: usize = 65_536;
/// Why a read gave up.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum ElfError {
/// The ELF header ends before the file does.
#[error("the file ends early ({0})")]
Truncated(&'static str),
/// No ELF magic, or an invalid class, encoding or field.
#[error("the file is malformed ({0})")]
Malformed(String),
}
/// Endian/class-aware reads over an ELF image.
#[derive(Clone, Copy)]
struct E<'a> {
r: Reader<'a>,
is64: bool,
le: bool,
}
impl<'a> E<'a> {
fn new(data: &'a [u8]) -> Result<Self, ElfError> {
if !data.starts_with(b"\x7fELF") {
return Err(ElfError::Malformed("no ELF magic".into()));
}
let r = Reader::new(data);
let class = r.u8(4).ok_or(ElfError::Truncated("ELF ident"))?;
let enc = r.u8(5).ok_or(ElfError::Truncated("ELF ident"))?;
if !matches!(class, 1 | 2) || !matches!(enc, 1 | 2) {
return Err(ElfError::Malformed("bad ELF class or data encoding".into()));
}
Ok(E { r, is64: class == 2, le: enc == 1 })
}
fn u16(&self, off: usize) -> Option<u16> {
if self.le { self.r.u16_le(off) } else { self.r.u16_be(off) }
}
fn u32(&self, off: usize) -> Option<u32> {
if self.le { self.r.u32_le(off) } else { self.r.u32_be(off) }
}
fn u64(&self, off: usize) -> Option<u64> {
if self.le { self.r.u64_le(off) } else { self.r.u64_be(off) }
}
fn word(&self, off: usize) -> Option<u64> {
if self.is64 { self.u64(off) } else { self.u32(off).map(u64::from) }
}
fn word_size(&self) -> usize {
if self.is64 { 8 } else { 4 }
}
}
/// One program header.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize))]
pub struct ProgramHeader {
/// [`segment_type_name`] of `p_type`.
pub kind: String,
/// [`permissions`] of `p_flags`.
pub flags: String,
pub offset: u64,
#[cfg_attr(feature = "serde", serde(serialize_with = "hex::serialize"))]
pub vaddr: u64,
pub file_size: u64,
pub mem_size: u64,
pub align: u64,
/// The raw segment type.
#[cfg_attr(feature = "serde", serde(skip))]
pub p_type: u32,
/// The raw `PF_R`/`PF_W`/`PF_X` flags.
#[cfg_attr(feature = "serde", serde(skip))]
pub p_flags: u32,
}
/// A segment type's `PT_` name without the prefix (`LOAD`, `GNU_RELRO`…), or its number in hex.
pub fn segment_type_name(t: u32) -> String {
match t {
0 => "NULL".into(),
1 => "LOAD".into(),
2 => "DYNAMIC".into(),
3 => "INTERP".into(),
4 => "NOTE".into(),
5 => "SHLIB".into(),
6 => "PHDR".into(),
7 => "TLS".into(),
0x6474_e550 => "GNU_EH_FRAME".into(),
0x6474_e551 => "GNU_STACK".into(),
0x6474_e552 => "GNU_RELRO".into(),
0x6474_e553 => "GNU_PROPERTY".into(),
0x7000_0001 => "ARM_EXIDX".into(),
other => format!("{other:#x}"),
}
}
/// `p_flags` as `rwx`, with `-` for each permission missing.
pub fn permissions(f: u32) -> String {
format!("{}{}{}", if f & 4 != 0 { 'r' } else { '-' }, if f & 2 != 0 { 'w' } else { '-' }, if f & 1 != 0 { 'x' } else { '-' })
}
/// The program headers of an ELF file, at most [`MAX_PHDRS`]. A table that runs past the file
/// ends at the last complete entry.
pub fn program_headers(data: &[u8]) -> Result<Vec<ProgramHeader>, ElfError> {
let e = E::new(data)?;
let (phoff, phentsize, phnum) = if e.is64 { (e.u64(32), e.u16(54), e.u16(56)) } else { (e.u32(28).map(u64::from), e.u16(42), e.u16(44)) };
let phoff = usize::try_from(phoff.ok_or(ElfError::Truncated("ELF header"))?).map_err(|_| ElfError::Malformed("phoff".into()))?;
let phentsize = usize::from(phentsize.unwrap_or(0));
let phnum = usize::from(phnum.unwrap_or(0)).min(MAX_PHDRS);
let min = if e.is64 { 56 } else { 32 };
if phnum > 0 && phentsize < min {
return Err(ElfError::Malformed("program header entry too small".into()));
}
let mut out = Vec::new();
for i in 0..phnum {
// The fields below sit within `phentsize` bytes of `o`, so their offsets can't overflow either.
let Some(o) = i.checked_mul(phentsize).and_then(|x| x.checked_add(phoff)).filter(|&o| o.checked_add(phentsize).is_some()) else { break };
let ph = if e.is64 {
(e.u32(o), e.u32(o + 4), e.u64(o + 8), e.u64(o + 16), e.u64(o + 32), e.u64(o + 40), e.u64(o + 48))
} else {
(
e.u32(o),
e.u32(o + 24),
e.u32(o + 4).map(u64::from),
e.u32(o + 8).map(u64::from),
e.u32(o + 16).map(u64::from),
e.u32(o + 20).map(u64::from),
e.u32(o + 28).map(u64::from),
)
};
let (Some(t), Some(f), Some(off), Some(va), Some(fs), Some(ms), Some(al)) = ph else { break };
out.push(ProgramHeader {
kind: segment_type_name(t),
flags: permissions(f),
offset: off,
vaddr: va,
file_size: fs,
mem_size: ms,
align: al,
p_type: t,
p_flags: f,
});
}
Ok(out)
}
/// A dynamic tag's `DT_` name without the prefix (`NEEDED`, `GNU_HASH`…), or its number in hex.
pub fn dynamic_tag_name(tag: u64) -> String {
match tag {
0 => "NULL",
1 => "NEEDED",
2 => "PLTRELSZ",
3 => "PLTGOT",
4 => "HASH",
5 => "STRTAB",
6 => "SYMTAB",
7 => "RELA",
8 => "RELASZ",
9 => "RELAENT",
10 => "STRSZ",
11 => "SYMENT",
12 => "INIT",
13 => "FINI",
14 => "SONAME",
15 => "RPATH",
16 => "SYMBOLIC",
17 => "REL",
18 => "RELSZ",
19 => "RELENT",
20 => "PLTREL",
21 => "DEBUG",
22 => "TEXTREL",
23 => "JMPREL",
24 => "BIND_NOW",
25 => "INIT_ARRAY",
26 => "FINI_ARRAY",
27 => "INIT_ARRAYSZ",
28 => "FINI_ARRAYSZ",
29 => "RUNPATH",
30 => "FLAGS",
0x6fff_fef5 => "GNU_HASH",
0x6fff_fffb => "FLAGS_1",
0x6fff_fff0 => "VERSYM",
0x6fff_fffe => "VERNEED",
0x6fff_ffff => "VERNEEDNUM",
_ => return format!("{tag:#x}"),
}
.into()
}
/// Map a virtual address to a file offset using `PT_LOAD` headers. Addresses in a segment's
/// zero-filled tail, or whose offset would overflow, map to nothing.
pub fn va_to_off(phdrs: &[ProgramHeader], va: u64) -> Option<u64> {
phdrs.iter().filter(|p| p.p_type == 1).find(|p| va >= p.vaddr && va - p.vaddr < p.file_size).and_then(|p| p.offset.checked_add(va - p.vaddr))
}
/// The (tag, value) pairs of the `PT_DYNAMIC` segment in `phdrs`, up to its `DT_NULL`, at most
/// [`MAX_DYN`]. Empty without a dynamic segment.
pub fn dynamic_entries(data: &[u8], phdrs: &[ProgramHeader]) -> Result<Vec<(u64, u64)>, ElfError> {
Ok(entries(&E::new(data)?, phdrs))
}
fn entries(e: &E<'_>, phdrs: &[ProgramHeader]) -> Vec<(u64, u64)> {
let mut out = Vec::new();
let Some(d) = phdrs.iter().find(|p| p.p_type == 2) else { return out };
let ent = 2 * e.word_size();
let n = usize::try_from(d.file_size).unwrap_or(0) / ent;
let start = usize::try_from(d.offset).unwrap_or(usize::MAX);
for i in 0..n.min(MAX_DYN) {
let Some(o) = i.checked_mul(ent).and_then(|x| start.checked_add(x)) else { break };
let (Some(tag), Some(val)) = (e.word(o), o.checked_add(e.word_size()).and_then(|v| e.word(v))) else { break };
if tag == 0 {
break;
}
out.push((tag, val));
}
out
}
#[cfg(test)]
mod tests;
+201
View File
@@ -0,0 +1,201 @@
//! Program header and dynamic section tests. `an_address_past_the_last_file_offset_maps_to_nothing`
//! is from Craft Anything's `elf.rs`; the image builder is a small stand-in for its testkit's.
use super::*;
/// Field writer in the image's class and byte order.
struct Image {
d: Vec<u8>,
is64: bool,
le: bool,
}
impl Image {
fn put(&mut self, at: usize, value: u64, width: usize) {
let bytes = value.to_le_bytes();
let mut field = bytes[..width].to_vec();
if !self.le {
field.reverse();
}
self.d[at..at + width].copy_from_slice(&field);
}
fn word(&mut self, at: usize, value: u64) {
self.put(at, value, if self.is64 { 8 } else { 4 });
}
}
/// An executable with PT_PHDR, PT_LOAD (r-x, whole file at 0x40_0000), PT_DYNAMIC at file 0x200
/// and PT_GNU_STACK (rw-). The dynamic section holds NEEDED, STRTAB and a DT_NULL, then junk.
fn elf(is64: bool, le: bool) -> Vec<u8> {
let mut img = Image { d: vec![0u8; 0x300], is64, le };
img.d[..4].copy_from_slice(b"\x7fELF");
img.d[4] = if is64 { 2 } else { 1 };
img.d[5] = if le { 1 } else { 2 };
img.d[6] = 1;
let (phoff_at, entsize_at, num_at, phoff, entsize) = if is64 { (32, 54, 56, 64, 56) } else { (28, 42, 44, 52, 32) };
img.word(phoff_at, phoff as u64);
img.put(entsize_at, entsize as u64, 2);
img.put(num_at, 4, 2);
let headers = [
(6u32, 4u32, phoff as u64, 0x40_0040, 4 * entsize as u64),
(1, 5, 0, 0x40_0000, 0x300),
(2, 6, 0x200, 0x40_0200, 0x60),
(0x6474_e551, 6, 0, 0, 0),
];
for (i, (t, f, off, va, size)) in headers.into_iter().enumerate() {
let o = phoff + i * entsize;
img.put(o, u64::from(t), 4);
if is64 {
img.put(o + 4, u64::from(f), 4);
img.word(o + 8, off);
img.word(o + 16, va);
img.word(o + 32, size);
img.word(o + 40, size);
img.word(o + 48, 0x1000);
} else {
img.word(o + 4, off);
img.word(o + 8, va);
img.word(o + 16, size);
img.word(o + 20, size);
img.put(o + 24, u64::from(f), 4);
img.word(o + 28, 0x1000);
}
}
let w = if is64 { 8 } else { 4 };
for (i, (tag, value)) in [(1u64, 1u64), (5, 0x40_0280), (0, 0), (1, 9)].into_iter().enumerate() {
img.word(0x200 + i * 2 * w, tag);
img.word(0x200 + i * 2 * w + w, value);
}
img.d
}
#[test]
fn program_headers_in_every_class_and_byte_order() {
for is64 in [false, true] {
for le in [false, true] {
let ph = program_headers(&elf(is64, le)).unwrap();
let kinds: Vec<(&str, &str)> = ph.iter().map(|p| (p.kind.as_str(), p.flags.as_str())).collect();
assert_eq!(kinds, [("PHDR", "r--"), ("LOAD", "r-x"), ("DYNAMIC", "rw-"), ("GNU_STACK", "rw-")], "is64={is64} le={le}");
assert_eq!((ph[1].offset, ph[1].vaddr, ph[1].file_size, ph[1].mem_size, ph[1].align), (0, 0x40_0000, 0x300, 0x300, 0x1000));
assert_eq!((ph[2].p_type, ph[2].p_flags), (2, 6));
let dynamic = dynamic_entries(&elf(is64, le), &ph).unwrap();
assert_eq!(dynamic, [(1, 1), (5, 0x40_0280)], "stops at DT_NULL");
assert_eq!(va_to_off(&ph, 0x40_0280), Some(0x280));
}
}
}
#[test]
fn bad_idents_and_headers_are_errors() {
assert_eq!(program_headers(b"\x7fELX\x02\x01"), Err(ElfError::Malformed("no ELF magic".into())));
assert_eq!(program_headers(b"\x7fELF"), Err(ElfError::Truncated("ELF ident")));
assert_eq!(program_headers(b"\x7fELF\x02"), Err(ElfError::Truncated("ELF ident")));
assert_eq!(program_headers(b"\x7fELF\x03\x01"), Err(ElfError::Malformed("bad ELF class or data encoding".into())));
assert_eq!(program_headers(b"\x7fELF\x01\x00"), Err(ElfError::Malformed("bad ELF class or data encoding".into())));
assert_eq!(program_headers(&elf(true, true)[..20]), Err(ElfError::Truncated("ELF header")));
assert_eq!(ElfError::Truncated("ELF header").to_string(), "the file ends early (ELF header)");
let mut small = elf(true, true);
small[54] = 40;
assert_eq!(program_headers(&small).unwrap_err().to_string(), "the file is malformed (program header entry too small)");
assert!(dynamic_entries(b"nope", &[]).is_err());
}
#[test]
fn a_table_cut_short_keeps_its_complete_entries() {
let d = elf(true, true);
assert_eq!(program_headers(&d[..64 + 56 * 2 + 10]).unwrap().len(), 2);
let mut none = d.clone();
none[56] = 0;
none[54] = 0;
assert!(program_headers(&none).unwrap().is_empty(), "no headers need no entry size");
let mut far = d.clone();
far[32..40].copy_from_slice(&u64::MAX.to_le_bytes());
assert!(program_headers(&far).unwrap().is_empty(), "offsets that overflow stop the walk");
}
#[test]
fn dynamic_entries_need_a_dynamic_segment_inside_the_file() {
let d = elf(false, true);
let mut ph = program_headers(&d).unwrap();
assert!(dynamic_entries(&d, &ph[..2]).unwrap().is_empty());
ph[2].offset = 0x2f8;
assert!(dynamic_entries(&d, &ph).unwrap().is_empty(), "an entry cut off by the end of the file");
ph[2].offset = u64::MAX;
assert!(dynamic_entries(&d, &ph).unwrap().is_empty());
}
fn load(offset: u64, vaddr: u64, file_size: u64) -> ProgramHeader {
ProgramHeader { kind: "LOAD".into(), flags: String::new(), offset, vaddr, file_size, mem_size: file_size, align: 0, p_type: 1, p_flags: 0 }
}
#[test]
fn an_address_past_the_last_file_offset_maps_to_nothing() {
assert_eq!(va_to_off(&[load(0x200, 0x1000, 0x100)], 0x1010), Some(0x210));
assert_eq!(va_to_off(&[load(u64::MAX, 0x1000, 0x100)], 0x1000), Some(u64::MAX));
assert_eq!(va_to_off(&[load(u64::MAX, 0x1000, 0x100)], 0x1001), None, "the offset would pass u64::MAX");
assert_eq!(va_to_off(&[load(0x200, 0x1000, 0x100)], 0x1100), None, "the zero-filled tail has no file bytes");
let mut dynamic = load(0, 0x1000, 0x100);
dynamic.p_type = 2;
assert_eq!(va_to_off(&[dynamic], 0x1010), None, "only PT_LOAD maps");
}
#[test]
fn names() {
let segments: Vec<String> =
[0, 1, 2, 3, 4, 5, 6, 7, 0x6474_e550, 0x6474_e551, 0x6474_e552, 0x6474_e553, 0x7000_0001, 0x99].into_iter().map(segment_type_name).collect();
assert_eq!(
segments,
[
"NULL",
"LOAD",
"DYNAMIC",
"INTERP",
"NOTE",
"SHLIB",
"PHDR",
"TLS",
"GNU_EH_FRAME",
"GNU_STACK",
"GNU_RELRO",
"GNU_PROPERTY",
"ARM_EXIDX",
"0x99"
]
);
assert_eq!([0, 1, 2, 4, 7].map(permissions), ["---", "--x", "-w-", "r--", "rwx"]);
let tags: Vec<String> =
(0..=30).chain([0x6fff_fef5, 0x6fff_fffb, 0x6fff_fff0, 0x6fff_fffe, 0x6fff_ffff, 0x7000_0000]).map(dynamic_tag_name).collect();
assert_eq!(tags[..3], ["NULL", "NEEDED", "PLTRELSZ"]);
assert_eq!(tags[29..], ["RUNPATH", "FLAGS", "GNU_HASH", "FLAGS_1", "VERSYM", "VERNEED", "VERNEEDNUM", "0x70000000"]);
assert!(tags.iter().all(|t| !t.is_empty()));
}
#[cfg(feature = "serde")]
#[test]
fn program_headers_serialize_with_hex_addresses() {
let ph = program_headers(&elf(true, true)).unwrap();
let value = serde_json::to_value(&ph[1]).unwrap();
assert_eq!(value["kind"], "LOAD");
assert_eq!(value["vaddr"], "0x400000");
assert!(value.get("p_type").is_none());
}
#[test]
fn damaged_input_never_panics() {
for sample in [elf(true, true), elf(false, false)] {
for i in 0..sample.len() {
if let Ok(ph) = program_headers(&sample[..i]) {
let _ = dynamic_entries(&sample[..i], &ph);
}
for v in [0x00, 0x7f, 0xff] {
let mut m = sample.clone();
m[i] = v;
if let Ok(ph) = program_headers(&m) {
let _ = dynamic_entries(&m, &ph);
let _ = va_to_off(&ph, 0x40_0280);
}
}
}
}
}
+263 -44
View File
@@ -7,7 +7,7 @@
//! - [`decode_part`] decodes the channels of one flat part that a filter keeps, in their stored
//! sample type, over the part's data window. Every compression the `exr` crate reads is read,
//! scan lines and tiles alike (the largest resolution level). Other parts' chunks are skipped
//! unread, and channels the filter drops are never allocated.
//! unread, and channels the filter drops are never allocated. Other parts may be deep.
//! - [`fn@write`] writes one scan-line part.
//!
//! The `exr` crate sizes buffers from headers before it reads pixels, so everything that reaches
@@ -18,7 +18,7 @@
use std::io::Cursor;
use exr::block::UncompressedBlock;
use exr::block::chunk::TileCoordinates;
use exr::block::chunk::{Chunk, CompressedBlock, CompressedScanLineBlock, CompressedTileBlock, TileCoordinates};
use exr::image::read::image::ReadImage;
use exr::image::read::layers::{ChannelsReader, ReadAllLayers, ReadChannels};
use exr::meta::MetaData;
@@ -245,11 +245,21 @@ fn window_size(size: exr::math::Vec2<usize>) -> (u32, u32) {
/// Reads the headers and checks every part's declared size, alone and together, before any pixel
/// allocation. Headers are read without the crate's validation, which refuses deep parts.
fn read_meta(bytes: &[u8], limits: &Limits) -> Result<MetaData> {
pub(super) fn read_meta(bytes: &[u8], limits: &Limits) -> Result<MetaData> {
check_previews(bytes)?;
let meta = MetaData::read_from_buffered(Cursor::new(bytes), false).map_err(map_exr)?;
let (mut total, mut dwa) = (0u64, 0u64);
for header in meta.headers.iter() {
let (mut total, mut flat, mut dwa) = (0u64, 0u64, 0u64);
for (index, header) in meta.headers.iter().enumerate() {
// The crate sizes a flat chunk's bound as a block's pixels times the bytes a pixel takes,
// without checking the product, before it reads the chunk.
let block_pixels = match header.blocks {
exr::meta::BlockDescription::Tiles(tiles) => (tiles.tile_size.0 as u64).checked_mul(tiles.tile_size.1 as u64),
exr::meta::BlockDescription::ScanLines => (header.compression.scan_lines_per_block() as u64).checked_mul(header.layer_size.0 as u64),
};
let block_bytes = block_pixels.and_then(|n| n.checked_mul(header.channels.bytes_per_pixel as u64)).and_then(|n| usize::try_from(n).ok());
if !header.deep && block_bytes.is_none() {
return Err(Error::LimitExceeded(format!("a block of part {index} holds more bytes than this platform can address")));
}
let (w, h) = window_size(header.layer_size);
if w > limits.max_width || h > limits.max_height {
return Err(Error::LimitExceeded(format!("a {w}x{h} part exceeds {}x{}", limits.max_width, limits.max_height)));
@@ -260,23 +270,24 @@ fn read_meta(bytes: &[u8], limits: &Limits) -> Result<MetaData> {
}
let bytes = pixels.saturating_mul(header.channels.list.len().max(1) as u64 * 4);
total = total.saturating_add(bytes);
if matches!(header.compression, ExrCompression::DWAA(_) | ExrCompression::DWAB(_)) {
dwa = dwa.saturating_add(bytes);
if !header.deep {
flat = flat.saturating_add(bytes);
if matches!(header.compression, ExrCompression::DWAA(_) | ExrCompression::DWAB(_)) {
dwa = dwa.saturating_add(bytes);
}
}
}
if total > limits.max_bytes {
return Err(Error::LimitExceeded(format!("{total} bytes across {} parts exceed {}", meta.headers.len(), limits.max_bytes)));
}
// The crate allocates a flat part's samples before reading its chunks, so the file must be
// able to hold them. DWA's run-length channels are zlib-packed on top and reach about
// 132000:1, so DWA parts are counted at a quarter. Deep parts size their samples from the
// file's own counts.
if !meta.requirements.has_deep_data {
let declared = total.saturating_sub(dwa).saturating_add(dwa / 4);
let capacity = (bytes.len() as u64).saturating_mul(limits.max_ratio).saturating_add(1 << 20);
if declared > capacity {
return Err(Error::LimitExceeded(format!("a {} byte file cannot hold {declared} bytes of samples", bytes.len())));
}
// A flat part's samples are allocated before its chunks are read, so the file must be able
// to hold the flat parts' samples. DWA's run-length channels are zlib-packed on top and reach
// about 132000:1, so DWA parts are counted at a quarter. Deep parts size their samples from
// the file's own counts and are not counted here.
let declared = flat.saturating_sub(dwa).saturating_add(dwa / 4);
let capacity = (bytes.len() as u64).saturating_mul(limits.max_ratio).saturating_add(1 << 20);
if declared > capacity {
return Err(Error::LimitExceeded(format!("a {} byte file cannot hold {declared} bytes of samples", bytes.len())));
}
Ok(meta)
}
@@ -459,19 +470,24 @@ pub fn decode_display_part(
}
let block_limit = std::cell::Cell::new(None);
let select = DisplaySelect { index, keep, fill, limits, seen: std::cell::Cell::new(0), block_limit: &block_limit };
let image = ReadImage::new(ReadAllLayers { read_channels: select }, |_| {})
.from_buffered(Cursor::new(bytes))
.map_err(|error| block_limit.take().map(Error::LimitExceeded).unwrap_or_else(|| map_exr(error)))?;
let channels = image
.layer_data
.into_iter()
.nth(index)
.ok_or_else(|| malformed("the part was not read"))?
.channel_data
.into_iter()
.flatten()
.map(|(name, samples)| DisplayChannel { name, samples })
.collect();
let exr_error = |error| block_limit.take().map(Error::LimitExceeded).unwrap_or_else(|| map_exr(error));
let planes = if meta.headers.iter().any(|header| header.deep) {
let header = meta.headers.get(index).ok_or_else(|| malformed(format!("part {index}")))?;
select.seen.set(index);
let mut reader = select.create_channels_reader(header).map_err(exr_error)?;
read_flat_part_chunks(bytes, &meta, index, &mut reader, limits)?;
reader.into_channels()
} else {
ReadImage::new(ReadAllLayers { read_channels: select }, |_| {})
.from_buffered(Cursor::new(bytes))
.map_err(exr_error)?
.layer_data
.into_iter()
.nth(index)
.ok_or_else(|| malformed("the part was not read"))?
.channel_data
};
let channels = planes.into_iter().flatten().map(|(name, samples)| DisplayChannel { name, samples }).collect();
Ok(DisplayPart { info, channels })
}
@@ -515,18 +531,23 @@ impl<'s> ReadChannels<'s> for DisplaySelect<'_> {
u32s: vec![],
});
}
// The EXR crate decompresses each accepted block before `read_block` can discard ignored
// channels. Bound that all-channel input separately from the selected output planes.
let Some(block_bytes) = layer_w
.checked_mul(layer_h)
.and_then(|pixels| pixels.checked_mul(header.channels.list.len().max(1)))
.and_then(|samples| samples.checked_mul(std::mem::size_of::<f32>()))
else {
// The EXR crate decompresses each accepted block, with every channel, before `read_block`
// can discard the ignored ones, so bound that all-channel block separately from the
// selected output planes. Blocks are decompressed one at a time, so the bound is the
// largest block (a tile within the data window, or a scan-line group across it), not the
// whole part: a part whose full channel set is larger than `max_bytes` still decodes when
// the kept planes and each block fit.
let block_pixels = match header.blocks {
exr::meta::BlockDescription::Tiles(tiles) => tiles.tile_size.0.min(layer_w).checked_mul(tiles.tile_size.1.min(layer_h)),
// a part shorter than its compression's scan-line group has only its own rows
exr::meta::BlockDescription::ScanLines => header.compression.scan_lines_per_block().min(layer_h).checked_mul(layer_w),
};
let Some(block_bytes) = block_pixels.and_then(|pixels| pixels.checked_mul(header.channels.bytes_per_pixel.max(1))) else {
self.block_limit.set(Some(format!("data-window block size exceeds {}", self.limits.max_bytes)));
return Err(exr::error::Error::Invalid("data-window block size overflow".into()));
};
if u64::try_from(block_bytes).map_or(true, |bytes| bytes > self.limits.max_bytes) || block_bytes > isize::MAX as usize {
self.block_limit.set(Some(format!("{block_bytes} data-window bytes exceed {}", self.limits.max_bytes)));
self.block_limit.set(Some(format!("{block_bytes} bytes in one data-window block exceed {}", self.limits.max_bytes)));
return Err(exr::error::Error::Invalid("data-window blocks exceed limit".into()));
}
let pixels = display_w.checked_mul(display_h).filter(|n| *n > 0).ok_or(exr::error::Error::Invalid("display window".into()))?;
@@ -727,22 +748,220 @@ impl ChannelsReader for SelectReader {
}
}
/// Where the headers end and the offset tables begin: [`MetaData::read_from_buffered`] consumes
/// exactly the version field and the headers.
fn tables_start(bytes: &[u8]) -> Result<u64> {
let mut cursor = Cursor::new(bytes);
MetaData::read_from_buffered(&mut cursor, false).map_err(map_exr)?;
Ok(cursor.position())
}
/// Stable dense ordinal without materializing every tile. The library's block iterator
/// internally allocates a Vec of all coordinates, which is unsuitable for cropped reads.
pub(super) fn flat_block_ordinal(header: &Header, tile: TileCoordinates) -> Result<usize> {
use exr::math::Vec2;
use exr::meta::attribute::LevelMode;
let mut start = 0usize;
let tile_size = header.max_block_pixel_size();
if tile_size.0 == 0 || tile_size.1 == 0 {
return Err(malformed("flat block has a zero dimension"));
}
let mut visit = |level: Vec2<usize>, size: Vec2<usize>| -> Result<Option<usize>> {
let nx = size.0.div_ceil(tile_size.0);
let ny = size.1.div_ceil(tile_size.1);
if tile.level_index == level {
if tile.tile_index.0 >= nx || tile.tile_index.1 >= ny {
return Err(malformed("flat tile lies outside its level"));
}
return tile
.tile_index
.1
.checked_mul(nx)
.and_then(|n| n.checked_add(tile.tile_index.0))
.and_then(|n| start.checked_add(n))
.map(Some)
.ok_or_else(|| malformed("block ordinal overflow"));
}
start = nx.checked_mul(ny).and_then(|n| start.checked_add(n)).ok_or_else(|| malformed("block count overflow"))?;
Ok(None)
};
match header.blocks {
exr::meta::BlockDescription::Tiles(tiles) => match tiles.level_mode {
LevelMode::Singular => {
if let Some(i) = visit(Vec2(0, 0), header.layer_size)? {
return Ok(i);
}
}
LevelMode::MipMap => {
for (level, size) in exr::meta::mip_map_levels(tiles.rounding_mode, header.layer_size) {
if let Some(i) = visit(Vec2(level, level), size)? {
return Ok(i);
}
}
}
LevelMode::RipMap => {
for (level, size) in exr::meta::rip_map_levels(tiles.rounding_mode, header.layer_size) {
if let Some(i) = visit(level, size)? {
return Ok(i);
}
}
}
},
exr::meta::BlockDescription::ScanLines => {
if let Some(i) = visit(Vec2(0, 0), header.layer_size)? {
return Ok(i);
}
}
}
Err(malformed("flat tile names a missing level"))
}
/// Reads the chunks of flat part `index` of a file that also holds deep parts into `reader`.
///
/// The `exr` crate's image reader validates every header before it reads a chunk and refuses
/// deep headers, so here only the flat headers are validated, the offset tables (one `u64` per
/// chunk, a table per part in header order) are walked from the end of the headers, and only the
/// selected part's chunks are read. Each chunk must start past the last offset table and inside
/// the file, carry the selected part's number, and hold one of the part's blocks that no other
/// chunk of the part holds, so the part's chunks hold every one of its blocks. A scan-line
/// chunk's y must be the first line of its block, and no chunk may declare more bytes than the
/// file has left after it.
fn read_flat_part_chunks(bytes: &[u8], meta: &MetaData, index: usize, reader: &mut impl ChannelsReader, limits: &Limits) -> Result<()> {
let header = meta.headers.get(index).ok_or_else(|| malformed(format!("part {index}")))?;
let multipart = meta.requirements.is_multilayer();
let mut long_names = false;
for flat in meta.headers.iter().filter(|h| !h.deep) {
flat.validate(multipart, &mut long_names, false).map_err(map_exr)?;
}
let truncated = || malformed("the offset tables run past the end of the file");
let (mut table, mut tables_end) = (0u64, tables_start(bytes)?);
for (i, h) in meta.headers.iter().enumerate() {
if i == index {
table = tables_end;
}
let len = u64::try_from(h.chunk_count).ok().and_then(|n| n.checked_mul(8)).ok_or_else(truncated)?;
tables_end = tables_end.checked_add(len).filter(|end| *end <= bytes.len() as u64).ok_or_else(truncated)?;
}
let offsets = usize::try_from(table)
.ok()
.and_then(|start| bytes.get(start..start.checked_add(header.chunk_count.checked_mul(8)?)?))
.ok_or_else(truncated)?;
// One bit per block, not a Vec/HashSet of every coordinate. A cropped display must not
// permit unbounded bookkeeping for an enormous data window of tiny tiles.
let tracking_bytes = header.chunk_count.div_ceil(8);
if tracking_bytes as u64 > limits.max_bytes || tracking_bytes > isize::MAX as usize {
return Err(Error::LimitExceeded("block tracking exceeds the byte budget".into()));
}
let mut seen = Vec::new();
seen.try_reserve_exact(tracking_bytes).map_err(|_| Error::LimitExceeded("cannot allocate block tracking".into()))?;
seen.resize(tracking_bytes, 0u8);
for &entry in offsets.as_chunks::<8>().0 {
let offset = u64::from_le_bytes(entry);
let chunk = usize::try_from(offset)
.ok()
.filter(|_| offset >= tables_end)
.and_then(|start| bytes.get(start..))
.filter(|rest| !rest.is_empty())
.ok_or_else(|| malformed(format!("chunk offset {offset} is outside the chunk area of the file")))?;
if multipart {
let part = chunk.get(..4).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(i32::from_le_bytes);
let part = part.ok_or_else(|| malformed(format!("the chunk at offset {offset} is cut off")))?;
if usize::try_from(part).ok() != Some(index) {
return Err(malformed(format!("the chunk at offset {offset} belongs to part {part}, not part {index}")));
}
}
// The crate allocates a chunk's declared payload before it reads it, so the payload must
// fit in the rest of the file. The length follows the part number and the block's y, or
// its four tile coordinates.
let fields = if matches!(header.blocks, exr::meta::BlockDescription::Tiles(_)) { 16 } else { 4 };
let size_at = if multipart { 4 } else { 0 } + fields;
let size = chunk.get(size_at..size_at + 4).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(i32::from_le_bytes);
let size = size.ok_or_else(|| malformed(format!("the chunk at offset {offset} is cut off")))?;
let left = chunk.len() - (size_at + 4);
if usize::try_from(size).is_ok_and(|size| size > left) {
return Err(malformed(format!("the chunk at offset {offset} declares {size} bytes, but {left} remain in the file")));
}
let size = usize::try_from(size).map_err(|_| malformed("negative packed chunk length"))?;
// Read coordinates without allocating the payload. Validate their real clipped block
// and budgets before Chunk::read uses the nominal (possibly huge) tile dimensions.
let coordinate_at = if multipart { 4 } else { 0 };
let shape = if matches!(header.blocks, exr::meta::BlockDescription::Tiles(_)) {
let coordinates = TileCoordinates::read(&mut Cursor::new(&chunk[coordinate_at..])).map_err(map_exr)?;
CompressedBlock::Tile(CompressedTileBlock { coordinates, compressed_pixels_le: Vec::new() })
} else {
let y = chunk
.get(coordinate_at..coordinate_at + 4)
.and_then(|b| <[u8; 4]>::try_from(b).ok())
.ok_or_else(|| malformed("truncated scan-line coordinate"))?;
CompressedBlock::ScanLine(CompressedScanLineBlock { y_coordinate: i32::from_le_bytes(y), compressed_pixels_le: Vec::new() })
};
let tile = header.get_block_data_indices(&shape).map_err(map_exr)?;
if let CompressedBlock::ScanLine(block) = &shape {
// The block index is rounded toward zero, so a y that is not the first line of a
// block, or lies just above the data window, would still name a block.
let lines = header.compression.scan_lines_per_block() as i64;
let first = i64::from(header.own_attributes.layer_position.y()) + tile.tile_index.y() as i64 * lines;
if i64::from(block.y_coordinate) != first {
return Err(malformed(format!(
"the scan-line chunk at offset {offset} starts at y {}, which is not the first line of a block",
block.y_coordinate
)));
}
}
let ordinal = flat_block_ordinal(header, tile)?;
let flag = 1u8 << (ordinal % 8);
let byte = seen.get_mut(ordinal / 8).ok_or_else(|| malformed("block is outside its chunk table"))?;
if *byte & flag != 0 {
return Err(malformed("duplicate flat block"));
}
*byte |= flag;
if reader.filter_block(tile) {
let area = header.get_absolute_block_pixel_coordinates(tile).map_err(map_exr)?.size;
let raw_size = area
.0
.checked_mul(area.1)
.and_then(|n| n.checked_mul(header.channels.bytes_per_pixel))
.ok_or_else(|| Error::LimitExceeded("flat block byte size overflows".into()))?;
if raw_size as u64 > limits.max_bytes || raw_size > isize::MAX as usize {
return Err(Error::LimitExceeded("flat block exceeds the byte budget".into()));
}
// Packed size need not be less than raw size for every accepted compression.
// Bound its allocation independently, rather than rejecting valid compressed data
// solely because a clipped edge block has fewer raw bytes.
if size as u64 > limits.max_bytes || size > isize::MAX as usize {
return Err(Error::LimitExceeded("packed flat block exceeds the byte budget".into()));
}
let chunk = Chunk::read(&mut Cursor::new(chunk), meta).map_err(map_exr)?;
let block = UncompressedBlock::decompress_chunk(chunk, meta, false).map_err(map_exr)?;
reader.read_block(header, block).map_err(map_exr)?;
}
}
Ok(())
}
/// Decodes the channels of flat part `index` that `keep` accepts (by the name as stored), in
/// their stored type. A file with deep parts, or a kept channel that is subsampled, is
/// [`Error::Unsupported`].
/// their stored type. A deep part, or a kept channel that is subsampled, is
/// [`Error::Unsupported`]; the file's other parts may be deep.
pub fn decode_part(bytes: &[u8], index: usize, keep: &dyn Fn(&str) -> bool, limits: &Limits) -> Result<Part> {
let meta = read_meta(bytes, limits)?;
if meta.requirements.has_deep_data {
let header = meta.headers.get(index).ok_or_else(|| malformed(format!("part {index} of {}", meta.headers.len())))?;
if header.deep {
return Err(Error::Unsupported("deep data (use the deep reader)".into()));
}
let header = meta.headers.get(index).ok_or_else(|| malformed(format!("part {index} of {}", meta.headers.len())))?;
if matches!(header.compression, ExrCompression::HTJ2K32 | ExrCompression::HTJ2K256) {
return Err(Error::Unsupported("HTJ2K compression".into()));
}
let info = part_info(index, header);
let select = Select { index, keep, seen: std::cell::Cell::new(0) };
let image = ReadImage::new(ReadAllLayers { read_channels: select }, |_| {}).from_buffered(Cursor::new(bytes)).map_err(map_exr)?;
let planes = image.layer_data.into_iter().nth(index).map(|l| l.channel_data).ok_or_else(|| malformed("the part was not read"))?;
let planes = if meta.headers.iter().any(|header| header.deep) {
select.seen.set(index);
let mut reader = select.create_channels_reader(header).map_err(map_exr)?;
read_flat_part_chunks(bytes, &meta, index, &mut reader, limits)?;
reader.into_channels()
} else {
let image = ReadImage::new(ReadAllLayers { read_channels: select }, |_| {}).from_buffered(Cursor::new(bytes)).map_err(map_exr)?;
image.layer_data.into_iter().nth(index).map(|l| l.channel_data).ok_or_else(|| malformed("the part was not read"))?
};
let channels = info.channels.iter().zip(planes).filter_map(|(c, p)| Some(Channel { name: c.name.clone(), samples: p? })).collect();
Ok(Part { info, channels })
}
+473 -4
View File
@@ -1,4 +1,5 @@
//! The preview check is PhotoCraft's test; the rest builds files with the `exr` crate.
//! The preview check is PhotoCraft's test; the rest builds files with the `exr` crate, except the
//! files that mix flat and deep parts, which are built by hand.
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use exr::meta::attribute::{Preview, Text};
@@ -129,10 +130,30 @@ fn decoded_display_part_places_selected_channels_and_checks_total_budget() {
let limits = Limits { max_bytes: 6 * 4 - 1, ..Limits::default() };
assert!(matches!(decode_display_part(&bytes, 0, &all, &|_| 0.0, &limits), Err(Error::LimitExceeded(_))));
// One selected display plane fits exactly, but the EXR decoder must decompress both data
// channels in its input block before discarding the ignored one.
// One selected display plane fits exactly. Both data channels together (2x2x2 f32, 32 bytes)
// don't, but they are decompressed one scan-line block (2 pixels x 2 channels) at a time.
let block_limits = Limits { max_bytes: 6 * 4, ..Limits::default() };
assert!(matches!(decode_display_part(&bytes, 0, &|name| name == "R", &|_| 0.0, &block_limits), Err(Error::LimitExceeded(_))));
let part = decode_display_part(&bytes, 0, &|name| name == "R", &|_| 9.0, &block_limits).unwrap();
assert_eq!(part.channels[0].samples, [0.0, 1.0, 9.0, 2.0, 3.0, 9.0]);
}
#[test]
fn decoded_display_part_bounds_each_all_channel_block() {
// The kept R plane is 4 pixels of f32 (16 bytes), but each scan-line block carries all four
// channels (64 bytes) before the ignored ones are discarded.
let bytes = file(vec![layer(None, 4, 1, &["R", "G", "B", "A"], false, exr::prelude::Compression::ZIP1)], (4, 1));
let r = |name: &str| name == "R";
assert!(matches!(decode_display_part(&bytes, 0, &r, &|_| 0.0, &Limits { max_bytes: 16, ..Limits::default() }), Err(Error::LimitExceeded(_))));
assert_eq!(decode_display_part(&bytes, 0, &r, &|_| 0.0, &Limits { max_bytes: 64, ..Limits::default() }).unwrap().channels.len(), 1);
// ZIP groups 16 scan lines, but a one-row part's block is that row: still 64 bytes, not 1024
let zip16 = file(vec![layer(None, 4, 1, &["R", "G", "B", "A"], false, exr::prelude::Compression::ZIP16)], (4, 1));
assert!(matches!(decode_display_part(&zip16, 0, &r, &|_| 0.0, &Limits { max_bytes: 63, ..Limits::default() }), Err(Error::LimitExceeded(_))));
assert_eq!(decode_display_part(&zip16, 0, &r, &|_| 0.0, &Limits { max_bytes: 64, ..Limits::default() }).unwrap().channels.len(), 1);
// A tile is the block of a tiled part: a 4x4 tile of four f32 channels is 256 bytes, while
// the whole 8x8 part is 1024 and the kept R plane of the 2x2 display window only 16.
let tiled = file(vec![layer(None, 8, 8, &["R", "G", "B", "A"], true, exr::prelude::Compression::ZIP1)], (2, 2));
assert!(matches!(decode_display_part(&tiled, 0, &r, &|_| 0.0, &Limits { max_bytes: 255, ..Limits::default() }), Err(Error::LimitExceeded(_))));
assert_eq!(decode_display_part(&tiled, 0, &r, &|_| 0.0, &Limits { max_bytes: 256, ..Limits::default() }).unwrap().channels[0].samples.len(), 4);
}
#[test]
@@ -238,3 +259,451 @@ fn hostile_files_never_panic() {
}
assert!(parts(b"not exr", &limits).is_err());
}
// A multi-part file with a flat part and a deep scan-line part, built by hand per the OpenEXR file
// layout (the `exr` crate's writer refuses deep headers): version 2 with the deep and multi-part
// flags, the headers and an empty header, one offset table per part, then chunks that each start
// with their part number. Both parts are uncompressed, two chunks a part: the deep part and a
// scan-line flat part hold one scan line a chunk, and a tiled flat part one 2 × 2 tile a chunk.
fn mixed_attr(out: &mut Vec<u8>, name: &str, ty: &str, payload: &[u8]) {
out.extend_from_slice(name.as_bytes());
out.push(0);
out.extend_from_slice(ty.as_bytes());
out.push(0);
out.extend_from_slice(&(payload.len() as i32).to_le_bytes());
out.extend_from_slice(payload);
}
/// The version field and the headers of a mixed file whose flat part is `w × h` with float
/// `B`, `G`, `R` channels, in scan lines or (with `tiled`) in one level of 2 × 2 tiles, and whose
/// deep part is 3 × 2 with one float `A` channel. Both parts are uncompressed.
fn mixed_headers(deep_first: bool, tiled: bool, (w, h): (usize, usize)) -> Vec<u8> {
mixed_headers_with(deep_first, tiled, (w, h), 0, &["B", "G", "R"], 2)
}
/// [`mixed_headers`] with the flat part's compression code (0 for none, or 3 for ZIP, which
/// holds 16 scan lines a block), its float channels, and the width and height of its tiles.
fn mixed_headers_with(deep_first: bool, tiled: bool, (w, h): (usize, usize), compression: u8, flat_channels: &[&str], tile: u32) -> Vec<u8> {
mixed_headers_with_deep_size(deep_first, tiled, (w, h), compression, flat_channels, tile, (3, 2))
}
fn mixed_headers_with_deep_size(
deep_first: bool,
tiled: bool,
(w, h): (usize, usize),
compression: u8,
flat_channels: &[&str],
tile: u32,
deep_size: (usize, usize),
) -> Vec<u8> {
let mut out = Vec::new();
out.extend_from_slice(&0x0131_2f76u32.to_le_bytes());
out.extend_from_slice(&(2u32 | 0x800 | 0x1000).to_le_bytes());
for deep in if deep_first { [true, false] } else { [false, true] } {
let (w, h, channels): (usize, usize, &[&str]) = if deep { (deep_size.0, deep_size.1, &["A"]) } else { (w, h, flat_channels) };
let mut chlist = Vec::new();
for name in channels {
chlist.extend_from_slice(name.as_bytes());
chlist.push(0);
chlist.extend_from_slice(&2i32.to_le_bytes()); // float
chlist.extend_from_slice(&[0u8; 4]); // pLinear and reserved
chlist.extend_from_slice(&1i32.to_le_bytes());
chlist.extend_from_slice(&1i32.to_le_bytes());
}
chlist.push(0);
let window: Vec<u8> = [0i32, 0, w as i32 - 1, h as i32 - 1].iter().flat_map(|v| v.to_le_bytes()).collect();
let tiled = tiled && !deep;
let compression = if deep { 0 } else { compression };
let lines = if compression == 3 { 16 } else { 1 };
let chunks = if tiled { w.div_ceil(tile as usize) * h.div_ceil(tile as usize) } else { h.div_ceil(lines) };
mixed_attr(&mut out, "channels", "chlist", &chlist);
mixed_attr(&mut out, "chunkCount", "int", &(chunks as i32).to_le_bytes());
mixed_attr(&mut out, "compression", "compression", &[compression]);
mixed_attr(&mut out, "dataWindow", "box2i", &window);
mixed_attr(&mut out, "displayWindow", "box2i", &window);
mixed_attr(&mut out, "lineOrder", "lineOrder", &[0]);
if deep {
mixed_attr(&mut out, "maxSamplesPerPixel", "int", &1i32.to_le_bytes());
}
mixed_attr(&mut out, "name", "string", if deep { b"deep" } else { b"flat" });
mixed_attr(&mut out, "pixelAspectRatio", "float", &1f32.to_le_bytes());
mixed_attr(&mut out, "screenWindowCenter", "v2f", &[0u8; 8]);
mixed_attr(&mut out, "screenWindowWidth", "float", &1f32.to_le_bytes());
if tiled {
// `tile` × `tile` tiles, one level, rounded down.
let mut desc: Vec<u8> = [tile, tile].iter().flat_map(|v| v.to_le_bytes()).collect();
desc.push(0);
mixed_attr(&mut out, "tiles", "tiledesc", &desc);
}
let ty: &[u8] = if deep {
b"deepscanline"
} else if tiled {
b"tiledimage"
} else {
b"scanlineimage"
};
mixed_attr(&mut out, "type", "string", ty);
if deep {
mixed_attr(&mut out, "version", "int", &1i32.to_le_bytes());
}
out.push(0); // end of this header
}
out.push(0); // end of the header list
out
}
/// Sample `i` of a flat channel: `R` is 1, 2, 3, …, `G` 10 more and `B` 100 more, top row first.
fn mixed_flat(channel: f32, i: usize) -> f32 {
channel + 1.0 + i as f32
}
/// A mixed file with a 3 × 2 flat part, and its chunk offsets as `offsets[part][chunk]`. The
/// tiled flat part's chunks hold the tiles at (0, 0) and (1, 0).
fn mixed_file(deep_first: bool, tiled: bool) -> (Vec<u8>, [[usize; 2]; 2]) {
let (w, h) = (3usize, 2usize);
let mut out = mixed_headers(deep_first, tiled, (w, h));
let chunks: Vec<Vec<Vec<u8>>> = (0..2i32)
.map(|part| {
let deep = (part == 0) == deep_first;
(0..h)
.map(|y| {
let mut c = Vec::new();
c.extend_from_slice(&part.to_le_bytes());
if tiled && !deep {
// Tile (y, 0) at level (0, 0): columns 2y and 2y + 1 that lie inside.
[y as i32, 0, 0, 0].iter().for_each(|v| c.extend_from_slice(&v.to_le_bytes()));
let xs = 2 * y..(2 * y + 2).min(w);
c.extend_from_slice(&((xs.len() * h * 3 * 4) as i32).to_le_bytes());
for row in 0..h {
for channel in [100.0, 10.0, 0.0] {
xs.clone().for_each(|x| c.extend_from_slice(&mixed_flat(channel, row * w + x).to_le_bytes()));
}
}
return c;
}
c.extend_from_slice(&(y as i32).to_le_bytes());
if deep {
// Packed count table, packed and unpacked sample data: one sample a pixel.
for size in [w * 4, w * 4, w * 4] {
c.extend_from_slice(&(size as u64).to_le_bytes());
}
(1..=w as u32).for_each(|n| c.extend_from_slice(&n.to_le_bytes()));
(0..w).for_each(|_| c.extend_from_slice(&0.5f32.to_le_bytes()));
} else {
c.extend_from_slice(&((w * 3 * 4) as i32).to_le_bytes());
for channel in [100.0, 10.0, 0.0] {
(0..w).for_each(|x| c.extend_from_slice(&mixed_flat(channel, y * w + x).to_le_bytes()));
}
}
c
})
.collect()
})
.collect();
let mut offsets = [[0usize; 2]; 2];
let mut at = out.len() + 2 * h * 8;
for (part, lines) in chunks.iter().enumerate() {
for (y, c) in lines.iter().enumerate() {
offsets[part][y] = at;
out.extend_from_slice(&(at as u64).to_le_bytes());
at += c.len();
}
}
chunks.into_iter().flatten().for_each(|c| out.extend_from_slice(&c));
(out, offsets)
}
#[test]
fn a_flat_part_beside_a_deep_part_decodes() {
let limits = Limits::default();
for (deep_first, tiled) in [(false, false), (true, false), (false, true), (true, true)] {
let (bytes, _) = mixed_file(deep_first, tiled);
let (flat, deep) = if deep_first { (1, 0) } else { (0, 1) };
let parts = parts(&bytes, &limits).unwrap();
assert_eq!((parts[flat].deep, parts[deep].deep, parts[flat].name.as_deref()), (false, true, Some("flat")));
let part = decode_part(&bytes, flat, &all, &limits).unwrap();
for (name, channel) in [("R", 0.0), ("G", 10.0), ("B", 100.0)] {
let expected: Vec<f32> = (0..6).map(|i| mixed_flat(channel, i)).collect();
assert_eq!(part.channel(name).unwrap().samples, Samples::F32(expected), "deep first {deep_first} tiled {tiled} {name}");
}
let only_g = decode_part(&bytes, flat, &|n| n == "G", &limits).unwrap();
assert_eq!(only_g.channels.iter().map(|c| c.name.as_str()).collect::<Vec<_>>(), ["G"]);
let display = decode_display_part(&bytes, flat, &|n| n == "flat.R", &|_| 9.0, &limits).unwrap();
assert_eq!(display.channels.len(), 1);
assert_eq!((display.channels[0].name.as_str(), display.channels[0].samples.as_slice()), ("flat.R", &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0][..]));
assert!(matches!(decode_part(&bytes, deep, &all, &limits), Err(Error::Unsupported(_))));
assert!(matches!(decode_display_part(&bytes, deep, &all, &|_| 0.0, &limits), Err(Error::Unsupported(_))));
assert!(matches!(decode_part(&bytes, 2, &all, &limits), Err(Error::Malformed(_))));
let deep_image = crate::deep::decode_deep(&bytes, &limits).unwrap();
assert_eq!((deep_image.width, deep_image.height), (3, 2));
}
}
#[test]
fn a_flat_part_s_chunks_beside_a_deep_part_are_checked_before_they_are_read() {
let limits = Limits::default();
for deep_first in [false, true] {
let (good, offsets) = mixed_file(deep_first, false);
let (flat, deep) = if deep_first { (1, 0) } else { (0, 1) };
let tables = mixed_headers(deep_first, false, (3, 2)).len();
let entry = tables + flat * 2 * 8;
let with_offset = |slot: usize, value: u64| {
let mut b = good.clone();
b[entry + slot * 8..entry + slot * 8 + 8].copy_from_slice(&value.to_le_bytes());
b
};
let refused = |b: &[u8]| {
for r in [decode_part(b, flat, &all, &limits).map(|_| ()), decode_display_part(b, flat, &all, &|_| 0.0, &limits).map(|_| ())] {
assert!(matches!(&r, Err(Error::Malformed(_))), "deep first {deep_first}: {r:?}");
}
};
// The deep part's chunk, the end of the file and past it, the offset tables, the same
// chunk twice.
refused(&with_offset(0, offsets[deep][0] as u64));
refused(&with_offset(0, good.len() as u64));
refused(&with_offset(0, u64::MAX));
refused(&with_offset(0, (tables + 8) as u64));
refused(&with_offset(1, offsets[flat][0] as u64));
// A chunk naming another part, and one declaring more bytes than a block holds.
let mut b = good.clone();
b[offsets[flat][1]..offsets[flat][1] + 4].copy_from_slice(&(deep as i32).to_le_bytes());
refused(&b);
let mut b = good.clone();
b[offsets[flat][0] + 8..offsets[flat][0] + 12].copy_from_slice(&i32::MAX.to_le_bytes());
refused(&b);
// Offset tables longer than the file.
refused(&good[..tables + 20]);
for n in 0..good.len() {
let _ = decode_part(&good[..n], flat, &all, &limits);
let _ = decode_display_part(&good[..n], flat, &all, &|_| 0.0, &limits);
}
for i in 0..good.len() {
for v in [0u8, 1, 0x7F, 0xFF] {
let mut b = good.clone();
b[i] = v;
if let Ok(p) = decode_part(&b, flat, &all, &limits) {
let n = p.info.data_window.width as usize * p.info.data_window.height as usize;
assert!(p.channels.iter().all(|c| c.samples.len() == n));
}
let _ = decode_display_part(&b, flat, &all, &|_| 0.0, &limits);
}
}
}
}
#[test]
fn a_tiled_flat_part_beside_a_deep_part_must_hold_each_of_its_tiles() {
let limits = Limits::default();
for deep_first in [false, true] {
let (good, offsets) = mixed_file(deep_first, true);
let flat = usize::from(deep_first);
// The second chunk names tile (0, 0) at level (1, 1), which a one-level part does not
// have, or tile (0, 0) again, so tile (1, 0) has no chunk.
for coordinates in [[0i32, 0, 1, 1], [0, 0, 0, 0]] {
let mut b = good.clone();
let at = offsets[flat][1] + 4;
coordinates.iter().enumerate().for_each(|(i, v)| b[at + i * 4..at + i * 4 + 4].copy_from_slice(&v.to_le_bytes()));
for r in [decode_part(&b, flat, &all, &limits).map(|_| ()), decode_display_part(&b, flat, &all, &|_| 0.0, &limits).map(|_| ())] {
assert!(matches!(&r, Err(Error::Malformed(_))), "deep first {deep_first} {coordinates:?}: {r:?}");
}
}
}
}
#[test]
fn a_scan_line_chunk_beside_a_deep_part_must_start_at_its_block_s_first_line() {
// A ZIP flat part (16 lines a block) holds its 3 × 2 pixels in one chunk, stored raw, as a
// block is when compressing it does not make it smaller; the deep part follows.
let limits = Limits::default();
let (w, h) = (3usize, 2usize);
let (plain, offsets) = mixed_file(false, false);
let deep = &plain[offsets[1][0]..];
let with_y = |y: i32| {
let mut out = mixed_headers_with(false, false, (w, h), 3, &["B", "G", "R"], 2);
let mut chunk = Vec::new();
chunk.extend_from_slice(&0i32.to_le_bytes());
chunk.extend_from_slice(&y.to_le_bytes());
chunk.extend_from_slice(&((w * h * 3 * 4) as i32).to_le_bytes());
for row in 0..h {
for channel in [100.0, 10.0, 0.0] {
(0..w).for_each(|x| chunk.extend_from_slice(&mixed_flat(channel, row * w + x).to_le_bytes()));
}
}
let flat_at = out.len() + 3 * 8;
let deep_at = flat_at + chunk.len();
for at in [flat_at, deep_at, deep_at + offsets[1][1] - offsets[1][0]] {
out.extend_from_slice(&(at as u64).to_le_bytes());
}
out.extend_from_slice(&chunk);
out.extend_from_slice(deep);
out
};
let part = decode_part(&with_y(0), 0, &all, &limits).unwrap();
assert_eq!(part.channel("R").unwrap().samples, Samples::F32((0..6).map(|i| mixed_flat(0.0, i)).collect()));
// Each of these y values rounds to block 0, but none is that block's first line.
for y in [-15, -1, 1, 15] {
let b = with_y(y);
for r in [decode_part(&b, 0, &all, &limits).map(|_| ()), decode_display_part(&b, 0, &all, &|_| 0.0, &limits).map(|_| ())] {
assert!(matches!(&r, Err(Error::Malformed(_))), "y {y}: {r:?}");
}
}
}
/// A mixed file whose flat part, first, is one pixel of these float channels (each 1.0) in one
/// `tile` × `tile` tile, held by a chunk that declares `size` payload bytes.
fn mixed_one_tile(channels: &[&str], tile: u32, size: i32) -> Vec<u8> {
let (plain, offsets) = mixed_file(false, false);
let mut out = mixed_headers_with(false, true, (1, 1), 0, channels, tile);
let mut chunk = Vec::new();
[0i32, 0, 0, 0, 0, size].iter().for_each(|v| chunk.extend_from_slice(&v.to_le_bytes()));
channels.iter().for_each(|_| chunk.extend_from_slice(&1f32.to_le_bytes()));
let flat_at = out.len() + 3 * 8;
let deep_at = flat_at + chunk.len();
for at in [flat_at, deep_at, deep_at + offsets[1][1] - offsets[1][0]] {
out.extend_from_slice(&(at as u64).to_le_bytes());
}
out.extend_from_slice(&chunk);
out.extend_from_slice(&plain[offsets[1][0]..]);
out
}
#[test]
fn a_chunk_beside_a_deep_part_must_fit_the_rest_of_the_file() {
// 2^14 × 2^14 tiles of three floats let a chunk hold up to 3 GiB (also on 32-bit), so the crate would
// allocate the 2 GiB this chunk declares before it found the file too short.
let limits = Limits::default();
let channels = ["B", "G", "R"];
let good = mixed_one_tile(&channels, 1 << 14, 12);
assert_eq!(decode_part(&good, 0, &all, &limits).unwrap().channel("R").unwrap().samples, Samples::F32(vec![1.0]));
let b = mixed_one_tile(&channels, 1 << 14, i32::MAX);
for r in [decode_part(&b, 0, &all, &limits).map(|_| ()), decode_display_part(&b, 0, &all, &|_| 0.0, &limits).map(|_| ())] {
assert!(matches!(&r, Err(Error::Malformed(m)) if m.contains("remain in the file")), "{r:?}");
}
}
#[test]
fn a_tile_too_large_to_address_is_refused_before_a_chunk_is_read() {
// Five float channels in 10^9 × 10^9 tiles: 20 bytes a pixel times 10^18 pixels does not fit
// in a 64-bit size, though the 1 × 1 part passes every size limit.
let limits = Limits::default();
let channels = ["A", "B", "C", "D", "E"];
let tile = 1_000_000_000u32;
let mixed = mixed_one_tile(&channels, tile, 20);
// A file of flat parts only, read by the crate's own reader, with its 4 × 4 tiles widened.
let mut flat_only = file(vec![layer(None, 1, 1, &channels, true, exr::prelude::Compression::Uncompressed)], (1, 1));
let at = flat_only.windows(9).position(|w| w == b"tiledesc\0").unwrap() + 9 + 4;
[tile, tile].iter().enumerate().for_each(|(i, v)| flat_only[at + i * 4..at + i * 4 + 4].copy_from_slice(&v.to_le_bytes()));
for (name, b) in [("mixed", &mixed), ("flat only", &flat_only)] {
let results = [
parts(b, &limits).map(|_| ()),
decode_part(b, 0, &all, &limits).map(|_| ()),
decode_display_part(b, 0, &all, &|_| 0.0, &limits).map(|_| ()),
];
for r in results {
assert!(matches!(&r, Err(Error::LimitExceeded(_))), "{name}: {r:?}");
}
}
}
#[test]
fn the_flat_parts_of_a_mixed_file_must_fit_the_file() {
// 1024 × 1024 × 3 float samples (12 MiB) declared in a file of a few hundred bytes: over a
// ratio of 1 plus the 1 MiB slack, though another part is deep.
let headers = mixed_headers(false, false, (1024, 1024));
assert!(parts(&headers, &Limits::default()).is_ok());
let tight = Limits { max_ratio: 1, ..Limits::default() };
for r in [parts(&headers, &tight).map(|_| ()), decode_part(&headers, 0, &all, &tight).map(|_| ())] {
assert!(matches!(&r, Err(Error::LimitExceeded(_))), "{r:?}");
}
// The deep part's samples are not counted, and a small flat part passes the same ratio.
assert!(parts(&mixed_file(false, false).0, &tight).is_ok());
}
#[test]
fn deep_samples_do_not_count_against_the_flat_expansion_ratio() {
// The 4 MiB deep plane would exceed max_ratio=1 plus the 1 MiB slack if counted.
// Header inspection does not allocate those samples; deep decoding bounds its own counts.
let headers = mixed_headers_with_deep_size(false, false, (1, 1), 0, &["R"], 2, (1024, 1024));
let limits = Limits { max_ratio: 1, max_bytes: 8 * 1024 * 1024, ..Limits::default() };
assert!(parts(&headers, &limits).is_ok());
}
#[test]
fn mixed_display_decode_bounds_each_block_not_the_whole_flat_part() {
for (tiled, block_bytes) in [(false, 36), (true, 48)] {
for deep_first in [false, true] {
let (bytes, _) = mixed_file(deep_first, tiled);
let flat = usize::from(deep_first);
let keep = |name: &str| name == "flat.R";
// The full flat part is 72 bytes; the selected plane is 24 bytes.
let limits = Limits { max_bytes: block_bytes, ..Limits::default() };
let part = decode_display_part(&bytes, flat, &keep, &|_| 0.0, &limits).unwrap();
assert_eq!(part.channels[0].samples, [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]);
let tight = Limits { max_bytes: block_bytes - 1, ..limits };
assert!(matches!(decode_display_part(&bytes, flat, &keep, &|_| 0.0, &tight), Err(Error::LimitExceeded(_))));
}
}
}
#[test]
fn present_packed_payload_is_bounded_before_allocation() {
let packed = 1024 * 1024;
let mut bytes = mixed_one_tile(&["R"], 1 << 15, packed);
// Make the declared payload present, not merely a truncated-file error. The nominal
// huge tile allows this size in exr::Chunk::read, but the actual flat image is 1x1.
bytes.resize(bytes.len() + packed as usize, 0);
let limits = Limits { max_bytes: 4, ..Limits::default() };
assert!(matches!(
decode_display_part(&bytes, 0, &|_| true, &|_| 0.0, &limits),
Err(Error::LimitExceeded(message)) if message.contains("packed flat block")
));
// decode_part also includes the deep header's 24 bytes in its aggregate header budget.
let limits = Limits { max_bytes: 32, ..limits };
assert!(matches!(decode_part(&bytes, 0, &all, &limits), Err(Error::LimitExceeded(message)) if message.contains("packed flat block")));
}
#[test]
fn cropped_mixed_display_bounds_block_tracking() {
let mut bytes = mixed_headers_with(false, true, (100, 100), 0, &["R"], 1);
// Both display windows are 1x1; the flat data window still has 10,000 one-pixel tiles.
let needle = b"displayWindow\0box2i\0";
let positions: Vec<usize> = bytes.windows(needle.len()).enumerate().filter_map(|(i, w)| (w == needle).then_some(i + needle.len() + 4)).collect();
for at in positions {
bytes[at..at + 16].fill(0);
}
// Supply all offset-table bytes so validation reaches bookkeeping, not truncation.
bytes.resize(bytes.len() + (10_000 + 2) * 8, 0);
let limits = Limits { max_bytes: 4, ..Limits::default() };
assert!(matches!(
decode_display_part(&bytes, 0, &|_| true, &|_| 0.0, &limits),
Err(Error::LimitExceeded(message)) if message.contains("block tracking")
));
}
#[test]
fn dense_block_ordinals_cover_scan_lines_and_all_tile_levels() {
use exr::block::chunk::TileCoordinates;
use exr::meta::attribute::LevelMode;
for tiled in [false, true] {
let bytes = file(vec![layer(None, 9, 7, &["R"], tiled, exr::prelude::Compression::ZIP1)], (9, 7));
let mut meta = read_meta(&bytes, &Limits::default()).unwrap();
let header = &mut meta.headers[0];
for mode in [LevelMode::Singular, LevelMode::MipMap, LevelMode::RipMap] {
if let exr::meta::BlockDescription::Tiles(tiles) = &mut header.blocks {
tiles.level_mode = mode;
}
for (i, block) in header.blocks_increasing_y_order().enumerate() {
assert_eq!(flat_block_ordinal(header, block.location).unwrap(), i);
}
let absent = TileCoordinates { tile_index: exr::math::Vec2(0, 0), level_index: exr::math::Vec2(100, 100) };
assert!(flat_block_ordinal(header, absent).is_err());
}
}
}
+22 -7
View File
@@ -391,19 +391,14 @@ fn decode_deep_meta(meta: &MetaData, bytes: &[u8], limits: &Limits) -> Result<De
let bytes_per_sample: usize = exr_types.iter().map(|t| sample_bytes(*t)).sum();
let multipart = meta.requirements.has_multiple_layers;
// The offset tables follow the headers: `chunk_count` u64 per header, in header order.
// Our part's table starts after the tables of all earlier parts.
let mut table_pos = header_end(bytes)?;
for h_ in meta.headers.iter().take(header_index) {
table_pos = table_pos.saturating_add((h_.chunk_count.min(1 << 24) as u64).saturating_mul(8));
}
// A chunk offset before the end of the last offset table points into table data.
let (table_pos, tables_end) = offset_tables(meta, bytes, header_index)?;
let mut cur = Cursor::new(bytes);
seek_to(&mut cur, table_pos)?;
let mut offsets = Vec::with_capacity(header.chunk_count.min(1 << 20));
for _ in 0..header.chunk_count {
offsets.push(u64(&mut cur)?);
}
let tables_end = cur.position();
// Block geometry.
let (tiles_x, tile_w, tile_h, slots) = match &header.blocks {
@@ -631,6 +626,26 @@ fn header_end(bytes: &[u8]) -> Result<u64> {
Ok(cur.position())
}
/// Where part `header_index`'s offset table starts and where the last part's table ends. The
/// tables follow the headers, `chunk_count` u64 per header in header order, and the chunks
/// follow the last table. Sizes that overflow, or tables that run past the end of the file,
/// are errors.
fn offset_tables(meta: &MetaData, bytes: &[u8], header_index: usize) -> Result<(u64, u64)> {
let mut table_pos = header_end(bytes)?;
let mut tables_end = table_pos;
for (i, h) in meta.headers.iter().enumerate() {
let len = u64::try_from(h.chunk_count).ok().and_then(|n| n.checked_mul(8));
tables_end = len.and_then(|len| tables_end.checked_add(len)).ok_or_else(|| err("the offset tables are too large"))?;
if i < header_index {
table_pos = tables_end;
}
}
if tables_end > bytes.len() as u64 {
return Err(err(format!("the offset tables end at byte {tables_end}, past the end of the file")));
}
Ok((table_pos, tables_end))
}
/// Composite the deep samples into a flat image: per pixel, samples sorted by Z (stable; the
/// file is assumed messy), front-to-back `over` with premultiplied colour, un-premultiplied
/// into straight alpha at the end. Without any alpha channel the front-most sample wins,
+100
View File
@@ -886,6 +886,106 @@ fn inconsistent_count_table_errors() {
assert!(matches!(decode_deep_exr(&bytes, &Limits::none()), Err(CodecError::Malformed(_))));
}
/// A two-part file made from the single-part deep file `deep` and a 1×1 flat scan-line part,
/// the deep part first when `deep_first`: each header gains `name` and `chunkCount`, the
/// version field the multipart bit, and each chunk its part number. The offset tables follow
/// the headers in header order, then the chunks. Returns the file, where the first table
/// starts, where the second table starts and where the last table ends.
fn two_part(deep: &[u8], deep_first: bool) -> (Vec<u8>, usize, usize, usize) {
let end = header_end_of(deep);
let first = u64::from_le_bytes(deep[end..end + 8].try_into().expect("8")) as usize;
let offsets: Vec<usize> =
(0..(first - end) / 8).map(|i| u64::from_le_bytes(deep[end + 8 * i..end + 8 * i + 8].try_into().expect("8")) as usize).collect();
let deep_chunks: Vec<&[u8]> = offsets.iter().enumerate().map(|(i, &o)| &deep[o..offsets.get(i + 1).copied().unwrap_or(deep.len())]).collect();
let mut deep_header = deep[8..end - 1].to_vec();
attr(&mut deep_header, "name", "string", b"deep");
attr(&mut deep_header, "chunkCount", "int", &(deep_chunks.len() as i32).to_le_bytes());
let mut flat_header = Vec::new();
let mut chlist = b"R\0".to_vec();
chlist.extend_from_slice(&1i32.to_le_bytes()); // half
chlist.extend_from_slice(&[0u8; 4]); // pLinear, reserved
chlist.extend_from_slice(&1i32.to_le_bytes());
chlist.extend_from_slice(&1i32.to_le_bytes());
chlist.push(0);
attr(&mut flat_header, "channels", "chlist", &chlist);
attr(&mut flat_header, "compression", "compression", &[0]);
let dw: Vec<u8> = [0i32; 4].iter().flat_map(|v| v.to_le_bytes()).collect();
attr(&mut flat_header, "dataWindow", "box2i", &dw);
attr(&mut flat_header, "displayWindow", "box2i", &dw);
attr(&mut flat_header, "lineOrder", "lineOrder", &[0]);
attr(&mut flat_header, "pixelAspectRatio", "float", &1f32.to_le_bytes());
attr(&mut flat_header, "screenWindowCenter", "v2f", &[0f32.to_le_bytes(), 0f32.to_le_bytes()].concat());
attr(&mut flat_header, "screenWindowWidth", "float", &1f32.to_le_bytes());
attr(&mut flat_header, "name", "string", b"flat");
attr(&mut flat_header, "type", "string", b"scanlineimage");
attr(&mut flat_header, "chunkCount", "int", &1i32.to_le_bytes());
// The flat part's one chunk: y, the data size, one half sample.
let mut flat_chunk = 0i32.to_le_bytes().to_vec();
flat_chunk.extend_from_slice(&2i32.to_le_bytes());
flat_chunk.extend_from_slice(&f16::from_f32(0.5).to_le_bytes());
let flat_chunks: Vec<&[u8]> = vec![&flat_chunk];
let parts: [(&[u8], &[&[u8]]); 2] = if deep_first {
[(&deep_header, &deep_chunks), (&flat_header, &flat_chunks)]
} else {
[(&flat_header, &flat_chunks), (&deep_header, &deep_chunks)]
};
let mut out = deep[..4].to_vec();
out.extend_from_slice(&(2u32 | 0x800 | 0x1000).to_le_bytes());
for (header, _) in &parts {
out.extend_from_slice(header);
out.push(0); // end of this header
}
out.push(0); // end of the headers
let first_table = out.len();
let second_table = first_table + parts[0].1.len() * 8;
let tables_end = second_table + parts[1].1.len() * 8;
let mut pos = tables_end;
for (_, chunks) in &parts {
for c in chunks.iter() {
out.extend_from_slice(&(pos as u64).to_le_bytes());
pos += 4 + c.len();
}
}
for (part, (_, chunks)) in parts.iter().enumerate() {
for c in chunks.iter() {
out.extend_from_slice(&(part as i32).to_le_bytes());
out.extend_from_slice(c);
}
}
(out, first_table, second_table, tables_end)
}
/// The deep part of a two-part file decodes as the single-part file does, first or second.
/// Its chunks start after every part's offset table: a chunk offset into the other part's
/// table, or one byte before the end of the last table, is refused as table data, and a file
/// that ends inside the last table is refused even when the deep part's own table is whole.
#[test]
fn multipart_deep_chunks_start_after_every_offset_table() {
let single = gen_deep(4, 4, &RGBAZ, 0, None, COUNTS_ONE, &sample_rgba);
let want = decode_deep_exr(&single, &Limits::none()).expect("single part");
for deep_first in [true, false] {
let (bytes, ..) = two_part(&single, deep_first);
let deep = decode_deep_exr(&bytes, &Limits::none()).expect("two parts");
assert_eq!(deep.counts, want.counts);
assert_eq!(deep.channel("Z").expect("Z").samples, want.channel("Z").expect("Z").samples);
}
let refused = |bytes: &[u8], what: &str| match decode_deep_exr(bytes, &Limits::none()) {
Err(e @ CodecError::Malformed(_)) => assert!(e.to_string().contains(what), "{e}"),
Err(e) => panic!("{e}, want an error about {what}"),
Ok(_) => panic!("decoded, want an error about {what}"),
};
let (bytes, first_table, second_table, tables_end) = two_part(&single, true);
for off in [second_table, tables_end - 1] {
let mut into_table = bytes.clone();
into_table[first_table..first_table + 8].copy_from_slice(&(off as u64).to_le_bytes());
refused(&into_table, "outside the chunk area");
}
refused(&bytes[..tables_end - 4], "past the end of the file");
}
/// `gen_deep` for a cropped render: the data window sits at `(dwx, dwy)` inside a larger
/// display window of `sw × sh` (the stage the renderer composed onto). Scanline chunk Y
/// coordinates are absolute display rows, so they carry the data-window offset.
+89
View File
@@ -0,0 +1,89 @@
//! Shared ownership of a font file and a face within it.
use std::sync::Arc;
/// Font file bytes and the index of a face in that file.
///
/// Clones share owned bytes, so collection faces and variable instances do not copy the file.
/// Static bytes let applications use embedded fonts without allocating a copy.
#[derive(Clone, Debug)]
pub struct FaceData {
bytes: Bytes,
index: u32,
}
#[derive(Clone, Debug)]
enum Bytes {
Static(&'static [u8]),
Owned(Arc<Vec<u8>>),
Shared(Arc<[u8]>),
}
impl FaceData {
/// A face in statically embedded font bytes.
pub fn from_static(bytes: &'static [u8], index: u32) -> Self {
Self { bytes: Bytes::Static(bytes), index }
}
/// A face in a shared vector, without copying its allocation.
pub fn from_owned(bytes: Arc<Vec<u8>>, index: u32) -> Self {
Self { bytes: Bytes::Owned(bytes), index }
}
/// A face in bytes shared by a catalog.
pub fn from_shared(bytes: Arc<[u8]>, index: u32) -> Self {
Self { bytes: Bytes::Shared(bytes), index }
}
/// The entire font file.
pub fn bytes(&self) -> &[u8] {
match &self.bytes {
Bytes::Static(bytes) => bytes,
Bytes::Owned(bytes) => bytes.as_slice(),
Bytes::Shared(bytes) => bytes,
}
}
/// The index of this face in the file.
pub fn index(&self) -> u32 {
self.index
}
/// Parse this face for font metadata and outlines.
pub fn skrifa(&self) -> Option<skrifa::FontRef<'_>> {
skrifa::FontRef::from_index(self.bytes(), self.index).ok()
}
/// Parse this face for shaping.
pub fn harfrust(&self) -> Option<harfrust::FontRef<'_>> {
harfrust::FontRef::from_index(self.bytes(), self.index).ok()
}
}
impl From<crate::FontData> for FaceData {
fn from(data: crate::FontData) -> Self {
Self::from_shared(data.bytes, data.index)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sources_keep_bytes_and_face_index_without_copying() {
static DATA: &[u8] = b"not a font";
let owned = Arc::new(DATA.to_vec());
let shared: Arc<[u8]> = Arc::from(DATA);
for data in [FaceData::from_static(DATA, 2), FaceData::from_owned(owned.clone(), 2), FaceData::from_shared(shared.clone(), 2)] {
let clone = data.clone();
assert_eq!(clone.bytes(), DATA);
assert_eq!(clone.bytes().as_ptr(), data.bytes().as_ptr());
assert_eq!(clone.index(), 2);
assert!(clone.skrifa().is_none());
assert!(clone.harfrust().is_none());
}
assert_eq!(Arc::strong_count(&owned), 1);
assert_eq!(Arc::strong_count(&shared), 1);
}
}
+4
View File
@@ -23,11 +23,13 @@
mod catalog;
mod embedding;
mod env;
mod face_data;
mod fallback;
mod fontconfig;
mod ligature;
mod line_break;
mod outline;
mod plan_cache;
mod registry;
mod scan;
mod segments;
@@ -41,11 +43,13 @@ mod visual_order;
pub use catalog::{Catalog, FontData, FontFileError, Limits, Lister, LoadedFace, Match, Watch};
pub use embedding::{SubsetEmbedding, subset_embedding};
pub use env::{FontEnv, Os, font_dirs, font_store_dir, office_font_dirs};
pub use face_data::FaceData;
pub use fallback::{cluster_face, drawable_characters};
pub use fontconfig::fontconfig_dirs;
pub use ligature::ligature_caret_stops;
pub use line_break::settle_line_break;
pub use outline::{FontOutline, GlyphPen, OutlineError};
pub use plan_cache::ShapePlanCache;
pub use scan::{FaceInfo, LocalizedName, Outlines, Source, is_font_file, style_italic, style_weight, style_width};
pub use segments::SegmentBoundary;
pub use shape::{RunGlyph, RunInput, ShapeRunError, shape_run};
+52
View File
@@ -0,0 +1,52 @@
//! Per-face bounded shaping plan reuse.
use std::sync::{Arc, Mutex};
/// A cache for one font face and variation instance. Do not share it across faces or instances.
/// Plans are created outside the lock, then checked again to avoid retaining duplicates.
#[derive(Default)]
pub struct ShapePlanCache {
plans: Mutex<Vec<Arc<harfrust::ShapePlan>>>,
}
impl ShapePlanCache {
/// Maximum number of plans retained; the oldest is evicted when this is reached.
pub const CAPACITY: usize = 32;
/// Returns a matching plan or builds and retains one for this face.
pub fn get_or_create(
&self,
shaper: &harfrust::Shaper<'_>,
instance: Option<&harfrust::ShaperInstance>,
direction: harfrust::Direction,
script: Option<harfrust::Script>,
language: Option<&harfrust::Language>,
features: &[harfrust::Feature],
) -> Arc<harfrust::ShapePlan> {
let key = harfrust::ShapePlanKey::new(script, direction).language(language).instance(instance).features(features);
let kept = |plans: &[Arc<harfrust::ShapePlan>]| plans.iter().rev().find(|p| key.matches(p)).cloned();
if let Some(plan) = kept(&self.plans.lock().unwrap_or_else(|e| e.into_inner())) {
return plan;
}
let plan = Arc::new(harfrust::ShapePlan::new(shaper, direction, script, language, features));
let mut plans = self.plans.lock().unwrap_or_else(|e| e.into_inner());
if let Some(made) = kept(&plans) {
return made;
}
if plans.len() >= Self::CAPACITY {
plans.remove(0);
}
plans.push(plan.clone());
plan
}
/// Number of retained plans.
pub fn len(&self) -> usize {
self.plans.lock().unwrap_or_else(|e| e.into_inner()).len()
}
/// Whether this cache contains no plans.
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
+19
View File
@@ -351,6 +351,25 @@ fn a_read_is_capped_where_the_file_has_grown() {
let _ = std::fs::remove_file(&path);
}
#[test]
fn shape_plan_cache_reuses_and_evicts_plans() {
let data = font(&spec("Test", "Regular", 400));
let face = harfrust::FontRef::from_index(&data, 0).unwrap();
let shaper_data = harfrust::ShaperData::new(&face);
let shaper = shaper_data.shaper(&face).build();
let cache = ShapePlanCache::default();
let make = |language: &harfrust::Language| cache.get_or_create(&shaper, None, harfrust::Direction::LeftToRight, None, Some(language), &[]);
let languages: Vec<harfrust::Language> = ('a'..='z').flat_map(|a| ['a', 'b'].map(move |b| format!("{a}{b}").parse().unwrap())).collect();
let first = make(&languages[0]);
assert!(Arc::ptr_eq(&first, &make(&languages[0])));
for language in languages.iter().skip(1) {
make(language);
}
assert_eq!(cache.len(), ShapePlanCache::CAPACITY);
assert!(!Arc::ptr_eq(&first, &make(&languages[0])));
assert_eq!(cache.len(), ShapePlanCache::CAPACITY);
}
#[test]
fn bounded_sfnt_reader_preserves_face_names_and_collection_indices() {
let mut bold = spec("Example", "Bold Italic", 700);
+20
View File
@@ -0,0 +1,20 @@
[package]
name = "craft-format"
description = "Versioned JSON file formats: the format tag, the registry of versions (hand-written steps, then one folder of changes per version, then the pending changes), the upgrade of an older file and the telling of a newer one"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
serde_json = { workspace = true }
# The container: content-addressed blobs (SHA-256) in a ZIP package.
sha2 = { version = "0.10", default-features = false, features = ["std"] }
craft-zip = { workspace = true }
[dev-dependencies]
proptest = { workspace = true }
+237
View File
@@ -0,0 +1,237 @@
//! For a build script: the generated lists of a family's changes. Each change is one `.rs` file in
//! the family's folder, in `v<N>/` once a release fixed its number, else in `pending/`; the lists
//! are generated so that a change adds a file and never edits a line another change also edits.
//!
//! The generated code, for the app to `include!`, is per family a `FOLDED` list (each folded
//! format and its parts) and a `PENDING` list, both prefixed by the family's `list_prefix`, then a
//! module per change file. It names the app's `Part` type, which the including module has in scope
//! (`type Part = craft_format::Part<AppError>;`), and a change file's step as `step` (a
//! `pub fn step(v: &mut Value) -> Result<(), AppError>`, when the file has one).
use std::fmt::Write as _;
use std::path::{Path, PathBuf};
/// A family's change files: their folder (relative to the crate's manifest), the last format
/// before them, and the prefix of the family's lists.
pub struct Family {
pub folder: &'static str,
pub base: u32,
pub list_prefix: &'static str,
}
/// What [`lists`] made: the code to write, and the folders the build script reruns on.
pub struct Lists {
pub code: String,
pub watch: Vec<PathBuf>,
}
/// The `FOLDED` and `PENDING` lists of every family, and a module for each change file. Refuses
/// versions that are not contiguous from a family's base, a change file not named with lowercase
/// letters, digits and dashes, and one that doesn't start with a `//!` note.
pub fn lists(manifest: &Path, families: &[Family]) -> Result<Lists, String> {
let mut code = String::new();
let mut mods = String::new();
let mut watch = Vec::new();
for family in families {
let root = manifest.join(family.folder);
family_lists(family, &root, &mut code, &mut mods)?;
watch.push(root);
}
code.push_str(&mods);
Ok(Lists { code, watch })
}
/// A family's `FOLDED` and `PENDING` lists; its folded formats follow on from its base.
fn family_lists(family: &Family, root: &Path, out: &mut String, mods: &mut String) -> Result<(), String> {
let Family { folder, base, list_prefix } = family;
let mut folded: Vec<(u32, PathBuf)> = Vec::new();
for entry in std::fs::read_dir(root).map_err(|e| format!("{}: {e}", root.display()))? {
// an entry that can't be read could be a version folder: fail rather than drop it
let path = entry.map_err(|e| format!("{}: {e}", root.display()))?.path();
let version = path.file_name().and_then(|n| n.to_str()).and_then(|n| n.strip_prefix('v')).and_then(|n| n.parse().ok());
if let Some(version) = version
&& path.is_dir()
{
folded.push((version, path));
}
}
folded.sort();
for (i, (version, _)) in folded.iter().enumerate() {
let expected = base + 1 + i as u32;
if *version != expected {
return Err(format!("{folder}/: expected v{expected}/ next, found v{version}/ (versions are contiguous)"));
}
}
// Module names start with the family's (`drawing_pending_…`), so two families' changes of one
// name never clash.
let module = list_prefix.to_ascii_lowercase();
let _ = writeln!(
out,
"/// The changes of each format after the base, oldest first (`{folder}/v<N>/`).\nconst {list_prefix}FOLDED: &[(u32, &[Part])] = &["
);
for (version, dir) in &folded {
let _ = writeln!(out, " ({version}, &[");
parts(dir, &format!("{module}v{version}"), false, out, mods)?;
out.push_str(" ]),\n");
}
let _ = writeln!(out, "];\n\n/// The changes no release has a number for yet (`{folder}/pending/`).\nconst {list_prefix}PENDING: &[Part] = &[");
parts(&root.join("pending"), &format!("{module}pending"), true, out, mods)?;
out.push_str("];\n\n");
Ok(())
}
/// Whether a change file declares its `step` function (`fn step(`, `pub fn step (` …) on a line of
/// code: a mention in a comment or a note is not a declaration.
fn declares_step(text: &str) -> bool {
let code = without_comments(text);
let ident = |c: char| c.is_alphanumeric() || c == '_';
// `fn`, whitespace, `step`, then `(` (or generics), across lines
code.match_indices("fn").any(|(at, _)| {
let before_ok = code[..at].chars().next_back().is_none_or(|c| !ident(c));
let rest = &code[at + 2..];
before_ok
&& rest.starts_with(char::is_whitespace)
&& rest
.trim_start()
.strip_prefix("step")
.is_some_and(|rest| matches!(rest.trim_start().chars().next(), Some('(' | '<')) && !rest.starts_with(ident))
})
}
/// `text` as code only: its `//` and (nested) `/* */` comments become a space, and the contents
/// of its string literals (plain, byte and raw) are dropped, line breaks kept throughout. A comment
/// marker inside a string is not a comment, and `fn/* note */step` still reads as `fn step`.
fn without_comments(text: &str) -> String {
let chars: Vec<char> = text.chars().collect();
let at = |i: usize| chars.get(i).copied();
let mut out = String::with_capacity(text.len());
let (mut i, mut depth) = (0usize, 0usize);
while let Some(c) = at(i) {
let next = at(i + 1);
if depth > 0 {
match (c, next) {
('/', Some('*')) => (depth, i) = (depth + 1, i + 2),
('*', Some('/')) => (depth, i) = (depth - 1, i + 2),
_ => {
if c == '\n' {
out.push('\n');
}
i += 1;
}
}
if depth == 0 {
out.push(' ');
}
continue;
}
let ident_before = i > 0 && at(i - 1).is_some_and(|p| p.is_alphanumeric() || p == '_');
match (c, next) {
('/', Some('/')) => {
while at(i).is_some_and(|c| c != '\n') {
i += 1;
}
out.push(' ');
}
('/', Some('*')) => (depth, i) = (1, i + 2),
('"', _) => {
out.push('"');
i += 1;
while let Some(c) = at(i) {
i += 1;
match c {
'\\' => i += 1,
'"' => break,
'\n' => out.push('\n'),
_ => {}
}
}
out.push('"');
}
// a raw string, `r"…"` or `r#"…"#`, after a byte prefix or on its own
('r', Some('"' | '#'))
if !ident_before || (i > 0 && at(i - 1) == Some('b') && !(i > 1 && at(i - 2).is_some_and(|p| p.is_alphanumeric() || p == '_'))) =>
{
let mut j = i + 1;
let mut hashes = 0;
while at(j) == Some('#') {
hashes += 1;
j += 1;
}
if at(j) != Some('"') {
out.push(c);
i += 1;
continue;
}
out.push('"');
i = j + 1;
while let Some(c) = at(i) {
if c == '"' && (1..=hashes).all(|k| at(i + k) == Some('#')) {
i += 1 + hashes;
break;
}
if c == '\n' {
out.push('\n');
}
i += 1;
}
out.push('"');
}
// a character literal (`'"'`, `'\''`), not a lifetime
('\'', Some('\\')) => {
i += 2;
while at(i).is_some_and(|c| c != '\'') {
i += 1;
}
i += 1;
out.push_str("' '");
}
('\'', Some(_)) if at(i + 2) == Some('\'') => {
i += 3;
out.push_str("' '");
}
_ => {
out.push(c);
i += 1;
}
}
}
out
}
/// The `Part`s of the change files in `dir`, in file-name order, and a module for each. A missing
/// folder is an empty list only where it may be absent (`optional`: `pending/`); any other error
/// reading it fails the build rather than leave a change out of the lists.
fn parts(dir: &Path, prefix: &str, optional: bool, out: &mut String, mods: &mut String) -> Result<(), String> {
let entries = match std::fs::read_dir(dir) {
Ok(entries) => entries,
Err(e) if optional && e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(e) => return Err(format!("{}: {e}", dir.display())),
};
let mut files: Vec<PathBuf> = Vec::new();
for entry in entries {
let path = entry.map_err(|e| format!("{}: {e}", dir.display()))?.path();
if path.extension().is_some_and(|x| x == "rs") {
files.push(path);
}
}
files.sort();
for path in files {
let name = path.file_stem().and_then(|s| s.to_str()).unwrap_or_default().to_string();
if name.is_empty() || !name.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') {
return Err(format!("{}: name a format change file with lowercase letters, digits and dashes", path.display()));
}
let text = std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?;
if !text.trim_start().starts_with("//!") {
return Err(format!("{}: start a format change with a `//!` note", path.display()));
}
let module = format!("{prefix}_{}", name.replace('-', "_"));
let step = if declares_step(&text) { format!("Some({module}::step)") } else { "None".into() };
let _ = writeln!(out, " Part {{ name: {name:?}, step: {step} }},");
let _ = writeln!(mods, "#[path = {:?}]\nmod {module};", path.display().to_string());
}
Ok(())
}
#[cfg(test)]
#[path = "build_tests.rs"]
mod tests;
+115
View File
@@ -0,0 +1,115 @@
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
/// A fresh, empty folder for one test.
fn scratch(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("craft-format-build-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
dir
}
fn write(path: &Path, text: &str) {
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, text).unwrap();
}
/// The error of a run that must fail.
fn refused(dir: &Path) -> String {
match lists(dir, &[DRAWING]) {
Ok(_) => panic!("expected an error"),
Err(e) => e,
}
}
const DRAWING: Family = Family { folder: "drawing", base: 3, list_prefix: "DRAWING_" };
#[test]
fn lists_folded_and_pending_changes_with_their_modules() {
let dir = scratch("ok");
write(&dir.join("drawing/v4/add-layers.rs"), "//! Layers.\npub fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n");
write(&dir.join("drawing/v4/notes.txt"), "not a change");
write(&dir.join("drawing/v5/rename.rs"), "\n//! A rename with no step.\n");
write(&dir.join("drawing/pending/next-1.rs"), "//! Pending.\nfn step(v: &mut Value) {}\n");
std::fs::create_dir_all(dir.join("drawing/v6x")).unwrap();
write(&dir.join("drawing/v7"), "a file, not a version folder");
let lists = lists(&dir, &[DRAWING]).unwrap();
assert_eq!(lists.watch, vec![dir.join("drawing")]);
let code = &lists.code;
assert!(code.contains("const DRAWING_FOLDED: &[(u32, &[Part])] = &["), "{code}");
assert!(code.contains(" (4, &[\n Part { name: \"add-layers\", step: Some(drawing_v4_add_layers::step) },\n ]),"), "{code}");
assert!(code.contains("Part { name: \"rename\", step: None }"), "{code}");
assert!(
code.contains("const DRAWING_PENDING: &[Part] = &[\n Part { name: \"next-1\", step: Some(drawing_pending_next_1::step) },"),
"{code}"
);
assert!(code.contains("mod drawing_v5_rename;"), "{code}");
assert!(!code.contains("notes"), "{code}");
assert!(code.find("const DRAWING_PENDING").unwrap() < code.find("#[path").unwrap(), "modules follow the lists");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn an_empty_family_has_empty_lists() {
let dir = scratch("empty");
std::fs::create_dir_all(dir.join("drawing")).unwrap();
let code = lists(&dir, &[DRAWING]).unwrap().code;
assert!(code.contains("const DRAWING_FOLDED: &[(u32, &[Part])] = &[\n];"), "{code}");
assert!(code.contains("const DRAWING_PENDING: &[Part] = &[\n];"), "{code}");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn refuses_gaps_bad_names_missing_notes_and_missing_folders() {
let dir = scratch("bad");
assert!(refused(&dir).contains("drawing"));
write(&dir.join("drawing/v5/a.rs"), "//! a");
assert!(refused(&dir).contains("expected v4/ next, found v5/"));
std::fs::rename(dir.join("drawing/v5"), dir.join("drawing/v4")).unwrap();
write(&dir.join("drawing/v4/Bad_Name.rs"), "//! b");
assert!(refused(&dir).contains("lowercase letters"));
std::fs::remove_file(dir.join("drawing/v4/Bad_Name.rs")).unwrap();
write(&dir.join("drawing/pending/no-note.rs"), "pub fn step() {}");
assert!(refused(&dir).contains("`//!` note"));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_step_is_a_declaration_not_a_mention() {
assert!(!declares_step("//! No fn step() is needed.\n"));
assert!(!declares_step("//! a note\n// fn step(v) was removed\nconst X: u8 = 1;\n"));
assert!(declares_step("//! a note\npub fn step (v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
// a declaration inside a block comment, nested or not, is not one
assert!(!declares_step("//! Documentation only.\n/*\nfn step() {}\n*/\n"));
assert!(!declares_step("/* outer /* inner */\nfn step() {}\n*/\nconst X: u8 = 1;\n"));
assert!(declares_step("/* a note */\nfn step(v: &mut Value) {}\n"));
// comment markers inside strings and character literals are not comments
assert!(declares_step("//! A change with a step.\nconst MARKER: &str = \"/*\";\npub fn step(_v: &mut Value) -> Result { Ok(()) }\n"));
assert!(declares_step("const A: &str = r#\"/* \"quoted\" \"#;\nconst B: &[u8] = br\"/*\";\nfn step(v: &mut Value) {}\n"));
assert!(declares_step("const Q: char = '\"';\nconst E: char = '\\'';\nfn step(v: &mut Value) { let _: &'static str = \"\"; }\n"));
assert!(declares_step("const S: &str = \"a \\\" /* b\";\nfn step(v: &mut Value) {}\n"));
assert!(!declares_step("const S: &str = \"\nfn step(v) \";\n"), "inside a string");
// a comment between the words is whitespace
assert!(declares_step("pub fn/* migration */step(v: &mut Value) {}\n"));
// across lines
assert!(declares_step("//! A change with a step.\npub fn step\n (_v: &mut Value) -> Result {\n Ok(())\n}\n"));
assert!(declares_step("pub fn\nstep(v: &mut Value) {}\n"));
assert!(!declares_step("fn stepper(v: &mut Value) {}\nfn prefn step() {}\n"), "other names");
assert!(declares_step("const X: u8 = 1; // fn step() is below\nfn step(v: &mut Value) {}\n"));
assert!(declares_step("//! a note\npub(crate) fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
assert!(declares_step("//! a note\n fn step(v: &mut Value) {}\n"));
assert!(!declares_step("//! a note\nfn stepper(v: &mut Value) {}\nfn fnstep() {}\n"));
}
#[test]
fn a_documentation_only_change_has_no_step_and_a_missing_version_folder_fails() {
let dir = scratch("doc-only");
write(&dir.join("drawing/v4/doc.rs"), "//! No fn step() is needed.\n");
let code = lists(&dir, &[DRAWING]).unwrap().code;
assert!(code.contains("Part { name: \"doc\", step: None }"), "{code}");
// a version folder that can't be listed is an error, not an empty format
let missing = parts(&dir.join("drawing/v9"), "x", false, &mut String::new(), &mut String::new());
assert!(missing.is_err());
assert!(parts(&dir.join("drawing/pending"), "x", true, &mut String::new(), &mut String::new()).is_ok());
let _ = std::fs::remove_dir_all(&dir);
}
+440
View File
@@ -0,0 +1,440 @@
//! The file container: a ZIP package of named parts (the file's JSON, binary payloads, optional
//! caches), never extracted to filesystem paths. Payloads are content-addressed blobs,
//! `blobs/<sha256>.<extension>`, stored once however often they are referred to. Every size is
//! bounded by the app's [`Limits`]: the file, each part, all parts expanded, the payloads resolved,
//! and the number of entries; a part is read no further than its declared size (a decompression
//! bomb stops there). What a part holds, and how a payload is encoded, is the app's.
use std::collections::{BTreeMap, BTreeSet};
use std::io::{Read, Seek, SeekFrom};
use std::sync::Arc;
use serde_json::Value;
use sha2::{Digest, Sha256};
use crate::FormatError;
/// How hard the payloads are compressed: Fast favours interactive file operations; Compact trades
/// CPU time for space.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum Compression {
#[default]
Fast,
Compact,
}
/// An app's budgets for its containers. The functions are read each time they are needed, so a
/// limit the app lets the user raise takes effect at once.
#[derive(Clone, Copy)]
pub struct Limits {
/// The most entries a container holds.
pub max_entries: usize,
/// The most bytes all parts may expand to, and all payloads may resolve to.
pub max_expanded: fn() -> usize,
/// The most bytes a part of this name may expand to.
pub part_limit: fn(&str) -> usize,
/// The Deflate level of a compressed part of this name and size.
pub level: fn(&str, usize) -> i64,
/// Whether a container of this many bytes may be read, or why not.
pub check_size: fn(u64) -> Result<(), String>,
/// Whether the work is cancelled (a save the person stopped): checked before each part, and
/// a long part is written a piece at a time and stops before the next piece when this says
/// so ([`FormatError::Cancelled`]). `None`: it cannot be cancelled here.
pub cancelled: Option<fn() -> bool>,
}
/// The most bytes of a part written between two cancellation checks.
const PIECE: usize = 1 << 20;
fn invalid(message: impl std::fmt::Display) -> FormatError {
FormatError::Invalid(message.to_string())
}
/// Whether the bytes start as a container does, whatever the file is called.
pub fn is_container(bytes: &[u8]) -> bool {
bytes.starts_with(b"PK\x03\x04")
}
/// The SHA-256 of the bytes.
pub fn sha256(bytes: &[u8]) -> [u8; 32] {
Sha256::digest(bytes).into()
}
fn hex(digest: &[u8; 32]) -> String {
use std::fmt::Write as _;
let mut out = String::with_capacity(64);
for byte in digest {
// Writing to a `String` cannot fail.
let _ = write!(&mut out, "{byte:02x}");
}
out
}
/// The SHA-256 of the bytes, in lowercase hexadecimal: a blob's name, and how a cache is keyed to
/// the file it was written for.
pub fn digest(bytes: &[u8]) -> String {
hex(&sha256(bytes))
}
/// One part a [`Writer`] wrote, with the SHA-256 of its uncompressed content. A caller that signs
/// a container uses these to bind the parts without reading the file back.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PartDigest {
pub name: String,
pub len: u64,
pub digest: [u8; 32],
}
/// A container being written.
pub struct Writer {
zip: craft_zip::Writer,
names: BTreeSet<String>,
expanded: usize,
resolved: usize,
limits: Limits,
/// Which written parts to record for a signature, and the records so far.
tracked: Option<TrackedParts>,
}
/// A tracking closure and the part digests collected so far.
type TrackedParts = (fn(&str) -> bool, Vec<PartDigest>);
impl Writer {
pub fn new(limits: Limits) -> Writer {
Writer { zip: craft_zip::Writer::new(), names: BTreeSet::new(), expanded: 0, resolved: 0, limits, tracked: None }
}
/// Record the parts written from now on whose name `include` accepts, with their SHA-256, for
/// [`Writer::take_parts`]. Used by a caller that signs the container.
pub fn track_parts(&mut self, include: fn(&str) -> bool) {
self.tracked = Some((include, Vec::new()));
}
/// The tracked parts written so far, oldest first, and stop tracking.
pub fn take_parts(&mut self) -> Option<Vec<PartDigest>> {
self.tracked.take().map(|(_, parts)| parts)
}
/// The container's bytes.
pub fn finish(self) -> Result<Vec<u8>, FormatError> {
self.zip.finish().map_err(invalid)
}
/// Add a part, compressed or stored. A name already written is written once.
pub fn part(&mut self, name: &str, bytes: &[u8], compressed: bool) -> Result<(), FormatError> {
self.part_with_digest(name, bytes, compressed, None)
}
/// [`Writer::part`] with a digest the caller already worked out (a blob's name is its digest),
/// so a signed save does not hash the same bytes twice.
fn part_with_digest(&mut self, name: &str, bytes: &[u8], compressed: bool, digest: Option<[u8; 32]>) -> Result<(), FormatError> {
if bytes.len() > (self.limits.part_limit)(name) {
return Err(invalid("part too large"));
}
if self.names.contains(name) {
return Ok(());
}
// Check the counters this part would leave, and commit them only once it is written: a
// refused part is not remembered as written.
let expanded = self.expanded.checked_add(bytes.len()).ok_or_else(|| invalid("expanded size overflow"))?;
if expanded > (self.limits.max_expanded)() || self.names.len().saturating_add(1) > self.limits.max_entries {
return Err(invalid("package too large"));
}
let cancelled = self.limits.cancelled;
if cancelled.is_some_and(|c| c()) {
return Err(FormatError::Cancelled);
}
// the configured level, clamped to the Deflate levels craft-zip writes (1 to 9)
let compression = if compressed {
craft_zip::Compression::Level((self.limits.level)(name, bytes.len()).clamp(1, 9) as u8)
} else {
craft_zip::Compression::Stored
};
match cancelled {
Some(cancelled) if bytes.len() > PIECE => {
let mut stopped = false;
let written = self.zip.add_with(name, compression, |out| {
for piece in bytes.chunks(PIECE) {
if cancelled() {
stopped = true;
return Err(std::io::Error::other("cancelled"));
}
out.write_all(piece)?;
}
Ok(())
});
if stopped {
return Err(FormatError::Cancelled);
}
written.map_err(invalid)?;
}
_ => self.zip.add(name, bytes, compression).map_err(invalid)?,
}
self.names.insert(name.to_owned());
self.expanded = expanded;
if let Some((include, parts)) = &mut self.tracked
&& include(name)
{
let digest = digest.unwrap_or_else(|| sha256(bytes));
parts.push(PartDigest { name: name.to_owned(), len: bytes.len() as u64, digest });
}
Ok(())
}
/// Would `entries` more optional parts of `extra` bytes, stored uncompressed, still keep the
/// file within `budget` and its expanded size within the reader's budget? The parts already
/// written count at their expanded size (an upper bound on what they take in the file).
pub fn room(&self, entries: usize, extra: usize, budget: usize) -> bool {
self.remaining(entries, budget).is_some_and(|left| extra <= left)
}
/// The bytes `entries` more optional parts may hold together ([`Writer::room`]).
pub fn space(&self, entries: usize, budget: usize) -> usize {
self.remaining(entries, budget).unwrap_or(0)
}
/// The bytes left for `entries` more parts, or `None` when the entries don't fit or what is
/// written already is past `budget` or the reader's budget.
fn remaining(&self, entries: usize, budget: usize) -> Option<usize> {
let count = self.names.len().checked_add(entries).filter(|&n| n <= self.limits.max_entries)?;
let overhead = count.saturating_mul(256).saturating_add(65_536);
let in_file = budget.checked_sub(overhead.saturating_add(self.expanded))?;
let expanded = (self.limits.max_expanded)().checked_sub(self.expanded)?;
Some(in_file.min(expanded))
}
/// Add a payload as a blob, `blobs/<sha256>.<extension>`, and say its name, for the JSON that
/// refers to it.
pub fn blob(&mut self, bytes: &[u8], extension: &str, compressed: bool) -> Result<Value, FormatError> {
let resolved = self
.resolved
.checked_add(bytes.len())
.filter(|&n| n <= (self.limits.max_expanded)())
.ok_or_else(|| invalid("resolved payloads too large"))?;
let digest = sha256(bytes);
let name = format!("blobs/{}.{extension}", hex(&digest));
self.part_with_digest(&name, bytes, compressed, Some(digest))?;
self.resolved = resolved;
Ok(Value::String(name))
}
}
/// A container being read. Opening reads all bytes into memory and checks the directory
/// (the entry count, unique and plain names, every part's
/// declared size), not the parts.
pub struct Reader {
archive: craft_zip::Archive<'static>,
blobs: BTreeMap<String, Vec<u8>>,
/// The bytes of payloads resolved so far, repeats counted: bounded by the expanded budget.
pub hydrated: usize,
document: Option<Arc<[u8]>>,
limits: Limits,
/// Every byte part reads have expanded to, whatever the budget was at the time.
read_total: u64,
}
impl Reader {
pub fn new(mut source: impl Read + Seek, limits: Limits) -> Result<Self, FormatError> {
let length = source.seek(SeekFrom::End(0)).map_err(invalid)?;
(limits.check_size)(length).map_err(invalid)?;
// Read all bytes into memory for the archive, no more than the length just approved: a file
// that grew since is refused rather than read past its budget.
source.seek(SeekFrom::Start(0)).map_err(invalid)?;
let capacity = usize::try_from(length).map_err(|_| invalid("file too large"))?;
let mut bytes = Vec::new();
bytes.try_reserve_exact(capacity).map_err(|_| invalid("file too large"))?;
source.take(length.saturating_add(1)).read_to_end(&mut bytes).map_err(invalid)?;
if bytes.len() as u64 != length {
return Err(invalid("the file changed while it was read"));
}
let mut archive = craft_zip::Archive::open_owned(
bytes,
craft_zip::Limits { max_entries: limits.max_entries as u64, max_total_bytes: (limits.max_expanded)() as u64 },
)
.map_err(invalid)?;
// the directory lists distinct names, so a repeated name is caught here
if archive.has_duplicate_names() {
return Err(invalid("duplicate or invalid archive entry"));
}
// Validate entry names match format rules: no empty, no backslashes,
// no path components that are "." or "..", and no duplicates; and every part's declared
// size within its configured limit.
let mut name_set = BTreeSet::new();
for entry_idx in 0..archive.len() {
// an entry whose name can't be read would be left out of the package unnoticed
let Ok(Some(name)) = archive.entry_name(entry_idx) else {
return Err(invalid("duplicate or invalid archive entry"));
};
{
if name.is_empty()
|| name.contains('\\')
|| name.split('/').any(|c| c.is_empty() || c == "." || c == "..")
|| !name_set.insert(name.clone())
{
return Err(invalid("duplicate or invalid archive entry"));
}
if archive.declared_size(&name).is_some_and(|size| size > (limits.part_limit)(&name) as u64) {
return Err(invalid("part too large"));
}
}
}
Ok(Self { archive, blobs: BTreeMap::new(), hydrated: 0, document: None, limits, read_total: 0 })
}
/// The file's own part, `document.json`, read once and kept: provisional caches and the
/// authoritative reading are bound to the same bytes, even if another process overwrites the
/// open file in place. Blobs remain hash-checked.
pub fn document(&mut self, limit: usize) -> Result<Arc<[u8]>, FormatError> {
if self.document.is_none() {
let bytes = self.limited_part("document.json", limit)?;
self.document = Some(bytes.into());
}
// a cached document still keeps the current part limit
let limit = limit.min((self.limits.part_limit)("document.json"));
self.document.as_ref().filter(|d| d.len() <= limit).cloned().ok_or_else(|| invalid("metadata budget"))
}
/// Whether the container has a part of this name.
pub fn has_part(&self, name: &str) -> bool {
self.archive.contains(name)
}
/// The names of every part, in the container's order.
pub fn part_names(&self) -> Vec<String> {
self.archive.names().map(|n| n.into_owned()).collect()
}
/// The part of this name, at most `limit` bytes and never more than its configured limit.
pub fn limited_part(&mut self, name: &str, limit: usize) -> Result<Vec<u8>, FormatError> {
let limit = limit.min((self.limits.part_limit)(name));
let expected = self.archive.declared_size(name).ok_or_else(|| invalid("part not found"))?;
if expected > limit as u64 {
return Err(invalid("part too large"));
}
// the expanded budget is read each time: a raised (or lowered) one applies at once, the
// bytes already read still counting against it
let budget = self.archive.budget_mut();
*budget = craft_zip::Budget::new(budget.what(), (self.limits.max_expanded)() as u64);
if budget.charge(self.read_total).is_err() {
return Err(invalid("package too large"));
}
// Read within budget. craft-zip's read() handles decompression bombs via the limit.
let bytes = self.archive.read(name, limit as u64).map_err(invalid)?.ok_or_else(|| invalid("part not found"))?;
if bytes.len() as u64 != expected {
return Err(invalid("part length mismatch"));
}
self.read_total = self.read_total.saturating_add(expected);
Ok(bytes)
}
/// The blob `value`'s `field` names, of this extension, within its part limit.
pub fn blob(&mut self, value: &Value, field: &str, extension: &str) -> Result<&[u8], FormatError> {
let limit = (self.limits.part_limit)(&format!("blob.{extension}"));
self.limited_blob(value, field, extension, limit)
}
/// [`Reader::blob`] within `limit` bytes. Its hash is checked when it is first read, and every
/// reference counts against the resolved budget.
pub fn limited_blob(&mut self, value: &Value, field: &str, extension: &str, limit: usize) -> Result<&[u8], FormatError> {
let name = value.get(field).and_then(Value::as_str).ok_or_else(|| invalid(format!("missing `{field}` reference")))?;
// a cached blob still keeps the part's current limit
let limit = limit.min((self.limits.part_limit)(name));
let hash = name
.strip_prefix("blobs/")
.and_then(|n| n.strip_suffix(&format!(".{extension}")))
.filter(|h| h.len() == 64 && h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))
.ok_or_else(|| invalid("invalid blob reference"))?;
if !self.blobs.contains_key(name) {
let bytes = self.limited_part(name, limit)?;
if digest(&bytes) != hash {
return Err(invalid(format!("blob hash mismatch: {name}")));
}
self.blobs.insert(name.to_owned(), bytes);
}
let bytes = self.blobs.get(name).ok_or_else(|| invalid("missing blob"))?;
if bytes.len() > limit {
return Err(invalid("blob exceeds its resource budget"));
}
// Count repeated references too: a tiny archive must not expand into unbounded owned
// strings or arrays when the same blob is referenced repeatedly.
self.hydrated = self
.hydrated
.checked_add(bytes.len())
.filter(|&n| n <= (self.limits.max_expanded)())
.ok_or_else(|| invalid("resolved payloads too large"))?;
Ok(bytes)
}
}
/// A small PNG picture of the file, kept in the container as a disposable cache keyed to the
/// exact file it shows: `cache/thumbnail.json` (its key) and `cache/thumbnail.png`. Reading it
/// never resolves a payload; a picture that isn't one, or is stale, is a miss.
#[derive(Clone, Copy, Debug)]
pub struct Thumbnail {
/// The key's format tag.
pub format: &'static str,
/// The build that wrote it: only the same build reads it back.
pub app: &'static str,
/// Its width and height in pixels.
pub size: [usize; 2],
/// The most bytes it may have.
pub max_bytes: usize,
}
impl Thumbnail {
/// Whether the bytes start as a PNG of this size within the byte budget (not a decode: a
/// consumer decodes with its own dimension and allocation limits).
pub fn valid(&self, bytes: &[u8]) -> bool {
bytes.len() <= self.max_bytes
&& bytes.get(..8) == Some(b"\x89PNG\r\n\x1a\n")
&& bytes.get(8..16) == Some(b"\0\0\0\rIHDR")
&& bytes.get(16..20) == Some((self.size[0] as u32).to_be_bytes().as_slice())
&& bytes.get(20..24) == Some((self.size[1] as u32).to_be_bytes().as_slice())
}
/// Add the picture, keyed to `document` (the file's own part), when it is one and there is
/// room for it within `budget`; otherwise leave it out, never failing the save.
pub fn write(&self, writer: &mut Writer, png: &[u8], document: &[u8], budget: usize) -> Result<(), FormatError> {
if !self.valid(png) {
return Ok(());
}
let key =
serde_json::to_vec(&serde_json::json!({"format": self.format, "app": self.app, "document": digest(document), "payload": digest(png)}))
.map_err(invalid)?;
let fits = |name: &str, len: usize| len <= (writer.limits.part_limit)(name);
if !fits("cache/thumbnail.json", key.len())
|| !fits("cache/thumbnail.png", png.len())
|| !writer.room(2, key.len().saturating_add(png.len()), budget)
{
return Ok(());
}
writer.part("cache/thumbnail.json", &key, false)?;
writer.part("cache/thumbnail.png", png, false)
}
/// The picture, when this build wrote it for exactly the file's own part (`document.json`,
/// read within `document_limit`).
pub fn read(&self, reader: &mut Reader, document_limit: usize) -> Result<Vec<u8>, FormatError> {
let key: Value = serde_json::from_slice(&reader.limited_part("cache/thumbnail.json", 4096)?).map_err(invalid)?;
let document = reader.document(document_limit)?;
if key.get("format").and_then(Value::as_str) != Some(self.format)
|| key.get("app").and_then(Value::as_str) != Some(self.app)
|| key.get("document").and_then(Value::as_str) != Some(digest(&document).as_str())
{
return Err(invalid("thumbnail key"));
}
let png = reader.limited_part("cache/thumbnail.png", self.max_bytes)?;
if !self.valid(&png) || key.get("payload").and_then(Value::as_str) != Some(digest(&png).as_str()) {
return Err(invalid("thumbnail payload"));
}
Ok(png)
}
}
#[cfg(test)]
#[path = "container_tests.rs"]
mod tests;
+260
View File
@@ -0,0 +1,260 @@
use super::*;
use std::io::Cursor;
const LIMITS: Limits = Limits {
max_entries: 8,
max_expanded: || 1 << 20,
part_limit: |name| if name.ends_with(".big") { 1 << 20 } else { 1024 },
level: |_, _| 1,
check_size: |length| if length <= 1 << 20 { Ok(()) } else { Err("too large".into()) },
cancelled: None,
};
/// Parts and blobs written are read back as they were; a blob is stored once, named by its hash,
/// and checked against it.
#[test]
fn parts_and_blobs_read_back() {
let mut writer = Writer::new(LIMITS);
writer.part("document.json", br#"{"a":1}"#, true).unwrap();
let first = writer.blob(b"payload", "bin", false).unwrap();
let again = writer.blob(b"payload", "bin", true).unwrap();
assert_eq!(first, again, "the same payload is one blob");
assert_eq!(first, Value::String(format!("blobs/{}.bin", digest(b"payload"))));
let bytes = writer.finish().unwrap();
assert!(is_container(&bytes));
let mut reader = Reader::new(Cursor::new(bytes), LIMITS).unwrap();
assert_eq!(reader.part_names().len(), 2);
assert!(reader.has_part("document.json") && !reader.has_part("other.json"));
assert_eq!(&*reader.document(64).unwrap(), br#"{"a":1}"#);
let reference = serde_json::json!({"blob": first});
assert_eq!(reader.blob(&reference, "blob", "bin").unwrap(), b"payload");
assert!(reader.blob(&reference, "blob", "png").is_err(), "another extension");
assert_eq!(reader.hydrated, 7);
reader.hydrated = (LIMITS.max_expanded)();
assert!(reader.blob(&reference, "blob", "bin").unwrap_err().to_string().contains("resolved payloads too large"));
}
/// A part past its limit, or more entries than the limit, are refused as they are written.
#[test]
fn budgets_are_kept_when_writing() {
let mut writer = Writer::new(LIMITS);
assert_eq!(writer.part("small.json", &[0; 1025], false), Err(FormatError::Invalid("part too large".into())));
assert!(writer.part("large.big", &[0; 1025], true).is_ok());
for i in 0..7 {
writer.part(&format!("{i}.json"), b"x", false).unwrap();
}
assert_eq!(writer.part("one-more.json", b"x", false), Err(FormatError::Invalid("package too large".into())));
assert!(!writer.room(1, 0, 1 << 20), "no room for an entry past the limit");
}
/// A thumbnail is kept only for the file it shows.
#[test]
fn a_thumbnail_is_keyed_to_its_file() {
let picture = Thumbnail { format: "app-thumbnail/1", app: "1.0", size: [2, 1], max_bytes: 1024 };
let mut png = b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR".to_vec();
png.extend_from_slice(&2u32.to_be_bytes());
png.extend_from_slice(&1u32.to_be_bytes());
assert!(picture.valid(&png));
let mut writer = Writer::new(LIMITS);
writer.part("document.json", b"{}", false).unwrap();
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
assert_eq!(picture.read(&mut reader, 64).unwrap(), png);
let mut writer = Writer::new(LIMITS);
writer.part("document.json", b"{\"changed\":1}", false).unwrap();
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
assert!(picture.read(&mut reader, 64).is_err(), "stale");
}
/// A refused part is not remembered as written: once the budget is raised it is written.
#[test]
fn a_refused_part_can_be_written_later() {
static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(4);
let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::SeqCst), ..LIMITS };
let mut writer = Writer::new(limits);
assert_eq!(writer.part("a.json", b"12345678", false), Err(FormatError::Invalid("package too large".into())));
assert!(writer.blob(b"12345678", "bin", false).is_err());
BUDGET.store(1 << 20, std::sync::atomic::Ordering::SeqCst);
writer.part("a.json", b"12345678", false).unwrap();
writer.blob(b"12345678", "bin", false).unwrap();
let reader = Reader::new(Cursor::new(writer.finish().unwrap()), limits).unwrap();
assert_eq!(reader.part_names().len(), 2, "both parts were written on the retry");
}
/// A cancelled save writes no more parts, whether the part is small or written in pieces.
#[test]
fn cancellation_stops_the_save() {
let cancelled = Limits { cancelled: Some(|| true), ..LIMITS };
let mut writer = Writer::new(cancelled);
assert_eq!(writer.part("a.json", b"a", false), Err(FormatError::Cancelled));
// cancelled after the first piece of a long part: checks before the part, then per piece
static CHECKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
let midway = Limits {
cancelled: Some(|| CHECKS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) >= 2),
max_expanded: || 1 << 22,
part_limit: |_| 1 << 22,
..LIMITS
};
let mut writer = Writer::new(midway);
assert_eq!(writer.part("big.big", &vec![0; PIECE + 1], true), Err(FormatError::Cancelled));
assert_eq!(CHECKS.load(std::sync::atomic::Ordering::SeqCst), 3);
let going = Limits { cancelled: Some(|| false), max_expanded: || 1 << 22, part_limit: |_| 1 << 22, ..LIMITS };
let mut writer = Writer::new(going);
writer.part("big.big", &vec![7; PIECE + 1], true).unwrap();
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), going).unwrap();
assert_eq!(reader.limited_part("big.big", 1 << 22).unwrap().len(), PIECE + 1);
}
/// Room counts what is already written, and the reader keeps every part's configured limit.
#[test]
fn room_counts_written_parts_and_reads_keep_part_limits() {
let mut writer = Writer::new(LIMITS);
writer.part("large.big", &[0; 500 * 1024], false).unwrap();
assert!(!writer.room(1, 200 * 1024, 600 * 1024));
assert!(writer.room(1, 1024, 1 << 20));
// what is written already is past the budget: not even an empty part fits
assert!(!writer.room(1, 0, 400 * 1024));
assert_eq!(writer.space(1, 400 * 1024), 0);
let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS };
let mut writer = Writer::new(wide);
writer.part("document.json", &[b' '; 2048], false).unwrap();
let bytes = writer.finish().unwrap();
assert!(Reader::new(Cursor::new(bytes.clone()), LIMITS).is_err(), "declared past its configured limit");
let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap();
assert!(reader.limited_part("document.json", 4096).is_ok());
}
/// A thumbnail past a cache part's own limit is left out, never failing the save.
#[test]
fn a_thumbnail_past_its_part_limit_is_left_out() {
let picture = Thumbnail { format: "app-thumbnail/1", app: "1.0", size: [2, 1], max_bytes: 2048 };
let mut png = b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR".to_vec();
png.extend_from_slice(&2u32.to_be_bytes());
png.extend_from_slice(&1u32.to_be_bytes());
png.resize(1500, 0);
let mut writer = Writer::new(LIMITS);
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
let reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
assert!(!reader.has_part("cache/thumbnail.json") && !reader.has_part("cache/thumbnail.png"));
}
/// A source that grows after its length was approved is refused.
#[test]
fn a_source_that_grows_while_read_is_refused() {
struct Growing(Cursor<Vec<u8>>, bool);
impl Read for Growing {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
if !self.1 {
self.1 = true;
let end = self.0.get_ref().len();
self.0.get_mut().extend_from_slice(&[0; 16]);
self.0.set_position(self.0.position().min(end as u64));
}
self.0.read(buf)
}
}
impl Seek for Growing {
fn seek(&mut self, to: SeekFrom) -> std::io::Result<u64> {
self.0.seek(to)
}
}
let mut writer = Writer::new(LIMITS);
writer.part("document.json", b"{}", false).unwrap();
let bytes = writer.finish().unwrap();
assert!(Reader::new(Growing(Cursor::new(bytes), false), LIMITS).err().is_some_and(|e| e.to_string().contains("changed")));
}
/// Compressed parts use the configured Deflate level.
#[test]
fn compressed_parts_use_the_configured_level() {
// words in a varied order, so the levels compress them differently
let words = ["alpha ", "beta ", "gamma ", "delta ", "epsilon ", "zeta ", "eta ", "theta "];
let mut seed = 7u32;
let text: Vec<u8> = (0..150)
.flat_map(|_| {
seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345);
words[(seed >> 16) as usize % words.len()].bytes()
})
.take(1000)
.collect();
let size = |level: fn(&str, usize) -> i64| {
let mut writer = Writer::new(Limits { level, ..LIMITS });
writer.part("document.json", &text, true).unwrap();
writer.finish().unwrap().len()
};
assert_ne!(size(|_, _| 1), size(|_, _| 9), "the level is used");
assert_eq!(size(|_, _| 42), size(|_, _| 9), "past 9 is clamped");
assert_eq!(size(|_, _| -3), size(|_, _| 1), "below 1 is clamped");
}
/// An archive naming one part twice is refused.
#[test]
fn a_repeated_part_name_is_refused() {
let mut writer = Writer::new(LIMITS);
writer.part("document.json", b"{}", false).unwrap();
writer.part("documenX.json", b"[]", false).unwrap();
let mut bytes = writer.finish().unwrap();
let (from, to) = (b"documenX.json", b"document.json");
let mut at = 0;
while let Some(i) = bytes[at..].windows(from.len()).position(|w| w == from) {
bytes[at + i..at + i + to.len()].copy_from_slice(to);
at += i + to.len();
}
assert!(Reader::new(Cursor::new(bytes), LIMITS).is_err());
}
/// Cached documents and blobs keep the current part limits.
#[test]
fn cached_parts_keep_the_current_part_limits() {
let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS };
let mut writer = Writer::new(wide);
writer.part("document.json", &[b' '; 512], false).unwrap();
let blob = writer.blob(&[7; 512], "bin", false).unwrap();
let bytes = writer.finish().unwrap();
let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap();
let reference = serde_json::json!({"blob": blob});
assert!(reader.document(4096).is_ok());
assert!(reader.blob(&reference, "blob", "bin").is_ok());
reader.limits = Limits { part_limit: |_| 128, ..wide };
assert!(reader.document(4096).is_err());
assert!(reader.limited_blob(&reference, "blob", "bin", 4096).is_err());
}
/// A raised expanded budget applies to the open reader, and what was read still counts.
#[test]
fn a_raised_expanded_budget_applies_at_once() {
static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(600);
let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::Relaxed), part_limit: |_| 1 << 20, ..LIMITS };
let wide = Limits { max_expanded: || 1 << 20, ..limits };
let mut writer = Writer::new(wide);
writer.part("document.json", &[b' '; 256], false).unwrap();
writer.part("other.big", &[1; 512], false).unwrap();
let bytes = writer.finish().unwrap();
BUDGET.store(1 << 20, std::sync::atomic::Ordering::Relaxed);
let mut reader = Reader::new(Cursor::new(bytes), limits).unwrap();
BUDGET.store(600, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("document.json", 4096).is_ok());
assert!(reader.limited_part("other.big", 4096).is_err(), "256 read + 512 is past 600");
BUDGET.store(1024, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("other.big", 4096).is_ok(), "768 fits the raised 1024");
// a lowered budget does not forget what was read: 768 + 256 is past 1000 after a dip to 100
BUDGET.store(100, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("document.json", 4096).is_err());
BUDGET.store(1000, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("document.json", 4096).is_err(), "1024 is past 1000");
BUDGET.store(1024, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("document.json", 4096).is_ok());
}
/// An entry whose local header can't be read is refused at open, not skipped.
#[test]
fn an_unreadable_entry_is_refused() {
let mut writer = Writer::new(LIMITS);
writer.part("document.json", b"{}", false).unwrap();
writer.part("other.json", b"[]", false).unwrap();
let mut bytes = writer.finish().unwrap();
let second = bytes.windows(4).enumerate().filter(|(_, w)| *w == b"PK\x03\x04").map(|(i, _)| i).nth(1).unwrap();
bytes[second + 2] = 0;
assert!(Reader::new(Cursor::new(bytes), LIMITS).is_err());
}
+257
View File
@@ -0,0 +1,257 @@
//! Versioned file formats: what an app needs to read every file it ever wrote, and to open a file
//! a newer build of it wrote.
//!
//! A file starts with `"format": "<prefix>N"`. A [`Family`] is one kind of file (an app's designs,
//! its drawings): its tag, its own numbers, and how a file of an older number is brought up to
//! this build's, step by step (N → N+1 → … → current). The app's code only knows the current
//! format; all translation lives in the steps:
//! - the steps written out by hand for the first formats, one per format ([`Formats::hand`]);
//! - then one folder per format, `v<N>/`, one file per change in it ([`Part`]), each with a step
//! when files from before the change are rewritten as they open ([`Formats::folded`]);
//! - then the changes no release has a number for yet, in `pending/` ([`Formats::pending`]):
//! while there are any, the build writes the next number, with their names in the file's
//! `format_pending`, so a later build runs just the steps a file is missing. A release moves
//! them into the next `v<N>/`. [`build`] lists the folders for a build script.
//!
//! A file a newer build wrote says so by a number past this build's or a change name this build
//! doesn't have ([`Family::newer`]); the app then reads what it can and keeps the rest
//! ([`newer`]). A file is saved as JSON, or in a [`container`] with its binary payloads.
//!
//! Steps are JSON-level (the file's shape) and return the app's own error type; every error this
//! crate makes is a [`FormatError`], which the app's error type takes `From`.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
use serde_json::Value;
pub mod build;
pub mod container;
pub mod newer;
/// Why a file is refused: its text says what is wrong, as the user reads it.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum FormatError {
/// The file isn't one of the family's, or it is damaged.
Invalid(String),
/// The work was cancelled (a save the person stopped): nothing more was written.
Cancelled,
}
impl std::fmt::Display for FormatError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
FormatError::Invalid(why) => f.write_str(why),
FormatError::Cancelled => f.write_str("cancelled"),
}
}
}
impl std::error::Error for FormatError {}
/// One JSON-level upgrade step: rewrites a file in place, or says why it can't.
pub type Step<E> = fn(&mut Value) -> Result<(), E>;
/// One change to a family's file format: a file in its `v<N>/` or `pending/` folder.
pub struct Part<E> {
/// The file's name, which a file written while the change was pending lists.
pub name: &'static str,
/// How a file from before the change is rewritten as it opens, if it is.
pub step: Option<Step<E>>,
}
/// Another spelling of a change name that a file of one format may carry (a build that named the
/// change otherwise): read as the change it stands for, never written.
#[derive(Clone, Copy, Debug)]
pub struct Alias {
/// The format whose files may say it.
pub format: u32,
/// The spelling in the file.
pub name: &'static str,
/// The change it stands for.
pub known_as: &'static str,
}
/// A family's upgrade path: the steps written out by hand, the folded formats after them (each a
/// list of changes) and the pending changes, which make the next format.
pub struct Formats<'a, E> {
/// Entry `i` turns format `i + 1` into `i + 2`.
pub hand: &'a [Step<E>],
/// Each format after the hand-written ones, oldest first, and its changes.
pub folded: &'a [(u32, &'a [Part<E>])],
/// The changes no release has a number for yet.
pub pending: &'a [Part<E>],
}
/// A kind of file whose format is versioned: its own format tag, its own numbers and its own
/// changes. Reading a file's format, telling a file a newer build saved and the upgrade are the
/// same for every family.
pub struct Family<'a, E> {
/// The app, as messages name it.
pub app: &'static str,
/// What a file of the family is called in messages.
pub noun: &'static str,
/// The tag before the number in a file's `format`.
pub prefix: &'static str,
/// The tag files saved under an older name carry: read, never written.
pub legacy_prefix: Option<&'static str>,
/// The format of a file that names none, if such files exist.
pub untagged: Option<u32>,
/// Other spellings of change names that files may carry.
pub aliases: &'a [Alias],
/// The upgrade path.
pub formats: Formats<'a, E>,
}
/// How a file shows a newer build saved it ([`Family::newer`]).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Newer {
/// The number the file says.
pub format: u32,
/// The number this build reads up to.
pub reads_up_to: u32,
/// The changes the file names that this build doesn't have.
pub unknown_changes: Vec<String>,
}
impl<E> Formats<'_, E> {
/// The format this build writes: the last folded one, or the next while changes are pending.
pub const fn current(&self) -> u32 {
self.hand.len() as u32 + 1 + self.folded.len() as u32 + if self.pending.is_empty() { 0 } else { 1 }
}
/// The changes format `version` is made of, if it is made of changes.
fn parts(&self, version: u32) -> Option<&[Part<E>]> {
let after = version.checked_sub(self.hand.len() as u32 + 2)? as usize;
match self.folded.get(after) {
Some((_, parts)) => Some(parts),
None if after == self.folded.len() && !self.pending.is_empty() => Some(self.pending),
None => None,
}
}
/// Whether a change of this name is anywhere in this build's history.
fn knows_part_name(&self, name: &str) -> bool {
self.folded.iter().flat_map(|(_, parts)| *parts).chain(self.pending).any(|part| part.name == name)
}
}
impl<E> Family<'_, E> {
/// The format this build writes: the last folded one, or the next while changes are pending.
pub const fn current(&self) -> u32 {
self.formats.current()
}
/// What a file of the family is called in messages.
pub const fn noun(&self) -> &'static str {
self.noun
}
/// The names of the pending changes this build writes into a file (`format_pending`).
pub fn pending_names(&self) -> Vec<String> {
self.formats.pending.iter().map(|p| p.name.to_string()).collect()
}
/// The format tag of format `n`.
pub fn format_string(&self, n: u32) -> String {
format!("{}{n}", self.prefix)
}
/// The version in one of this family's format tags; `None` for any other tag, another
/// family's included.
pub fn version_of(&self, s: &str) -> Option<u32> {
// each tag form parses on its own: a legacy prefix may start with the primary one
// (`app/` and `app/design/`)
let parse = |prefix: &str| s.strip_prefix(prefix).and_then(|n| n.parse().ok());
parse(self.prefix).or_else(|| self.legacy_prefix.and_then(parse))
}
/// The change a name a file of format `version` lists stands for: its alias's, or itself.
/// An alias counts only while this build has the change it stands for, so recognising it
/// never grants support to another reader.
fn canonical_applied_name<'name>(&self, version: u32, name: &'name str) -> &'name str {
match self.aliases.iter().find(|a| a.format == version && a.name == name) {
Some(alias) if self.formats.knows_part_name(alias.known_as) => alias.known_as,
_ => name,
}
}
/// What says a file of format `version` with the changes `applied` came from a newer build:
/// a number past this build's, or a change this build doesn't have anywhere in its history
/// (a branch's pending change may have landed at a later number). A list beside a format that
/// isn't made of changes means nothing (a tool or a test set an older number on a newer file).
/// `None` for a file this build fully knows.
pub fn newer(&self, version: u32, applied: Option<&[String]>) -> Option<Newer> {
let current = self.current();
let listed = applied.filter(|_| version > current || self.formats.parts(version).is_some()).unwrap_or_default();
let unknown_changes: Vec<String> =
listed.iter().filter(|name| !self.formats.knows_part_name(self.canonical_applied_name(version, name))).cloned().collect();
(version > current || !unknown_changes.is_empty()).then_some(Newer { format: version, reads_up_to: current, unknown_changes })
}
}
impl<E: From<FormatError>> Family<'_, E> {
/// Read a file's format and take its `format_pending` out of its JSON, before anything else
/// in it is read: a file that isn't this family's is refused here. One a newer build saved is
/// not: [`Family::newer`] says so.
pub fn take_version(&self, v: &mut Value) -> Result<(u32, Option<Vec<String>>), E> {
let (app, noun) = (self.app, self.noun);
let not_ours = |what: String| E::from(FormatError::Invalid(format!("not a {app} {noun} ({what})")));
let Some(obj) = v.as_object() else { return Err(not_ours("expected a JSON object".into())) };
let version = match obj.get("format") {
None => self.untagged.ok_or_else(|| not_ours("no format".into()))?,
Some(Value::String(f)) => self.version_of(f).ok_or_else(|| not_ours(format!("format `{f}`")))?,
Some(_) => return Err(not_ours("format is not a string".into())),
};
let applied = take_names(v, "format_pending", noun).map_err(E::from)?;
if version == 0 {
return Err(not_ours("format 0".into()));
}
Ok((version, applied))
}
/// Bring a file of format `version` up to this build's; `applied` is its `format_pending`
/// (both from [`Family::take_version`]): the changes of its format it already has, when it
/// was written while they were pending. The others' steps run first, and already-applied
/// changes never run again at their later number. A file past this build's number has no
/// steps to run.
pub fn upgrade(&self, v: &mut Value, version: u32, applied: Option<&[String]>) -> Result<(), E> {
let formats = &self.formats;
if version == 0 {
return Err(E::from(FormatError::Invalid(format!("not a {} {} (format 0)", self.app, self.noun))));
}
// A list beside a format that isn't made of changes is ignored, like any field an upgrade
// drops.
let applied = applied.filter(|_| formats.parts(version).is_some());
let already_applied =
|part: &Part<E>| applied.is_some_and(|names| names.iter().any(|name| self.canonical_applied_name(version, name) == part.name));
if applied.is_some() {
for part in formats.parts(version).unwrap_or_default().iter().filter(|part| !already_applied(part)) {
part.step.map_or(Ok(()), |step| step(v))?;
}
}
for from in version..self.current() {
match formats.hand.get(from as usize - 1) {
Some(step) => step(v)?,
None => {
for part in formats.parts(from + 1).unwrap_or_default().iter().filter(|part| !already_applied(part)) {
part.step.map_or(Ok(()), |step| step(v))?;
}
}
}
}
Ok(())
}
}
/// Take a short list of names out of a file's JSON: its `format_pending` (the pending changes it
/// has).
fn take_names(v: &mut Value, key: &str, noun: &str) -> Result<Option<Vec<String>>, FormatError> {
let Some(list) = v.as_object_mut().and_then(|o| o.remove(key)) else { return Ok(None) };
let invalid = || FormatError::Invalid(format!("damaged {noun}: `{key}` is not a short list of names"));
let list = list.as_array().filter(|l| l.len() <= 1000).ok_or_else(invalid)?;
list.iter().map(|n| n.as_str().filter(|n| n.len() <= 128).map(str::to_string).ok_or_else(invalid)).collect::<Result<_, _>>().map(Some)
}
#[cfg(test)]
#[path = "lib_tests.rs"]
mod tests;
+142
View File
@@ -0,0 +1,142 @@
use super::*;
use serde_json::json;
/// The app's error, as an app has one.
#[derive(Debug, PartialEq)]
struct AppError(String);
impl From<FormatError> for AppError {
fn from(e: FormatError) -> Self {
AppError(e.to_string())
}
}
/// Each step appends its name to the file's `log`, so the order steps ran in can be read back.
fn log(v: &mut Value, step: &str) {
if let Some(list) = v.get_mut("log").and_then(Value::as_array_mut) {
list.push(json!(step));
}
}
const HAND: [Step<AppError>; 2] = [
|v| {
log(v, "1to2");
Ok(())
},
|v| {
log(v, "2to3");
Ok(())
},
];
const V4: [Part<AppError>; 2] = [
Part {
name: "a",
step: Some(|v| {
log(v, "a");
Ok(())
}),
},
Part { name: "b", step: None },
];
const V5: [Part<AppError>; 1] = [Part {
name: "c",
step: Some(|v| {
log(v, "c");
Ok(())
}),
}];
const FOLDED: [(u32, &[Part<AppError>]); 2] = [(4, &V4), (5, &V5)];
const PENDING: [Part<AppError>; 2] = [
Part {
name: "d",
step: Some(|v| {
log(v, "d");
Ok(())
}),
},
Part {
name: "e",
step: Some(|v| {
log(v, "e");
Ok(())
}),
},
];
const ALIASES: [Alias; 1] = [Alias { format: 5, name: "old-c", known_as: "c" }];
/// Format 1 untagged; hand steps to 3; folded 4 and 5; pending changes make 6.
const FAMILY: Family<'static, AppError> = Family {
app: "App",
noun: "design",
prefix: "app/",
legacy_prefix: Some("old/"),
untagged: Some(1),
aliases: &ALIASES,
formats: Formats { hand: &HAND, folded: &FOLDED, pending: &PENDING },
};
fn steps_run(version: u32, applied: Option<&[String]>) -> Vec<String> {
let mut v = json!({"log": []});
FAMILY.upgrade(&mut v, version, applied).unwrap();
v["log"].as_array().unwrap().iter().map(|s| s.as_str().unwrap().to_string()).collect()
}
#[test]
fn tags_and_numbers() {
assert_eq!(FAMILY.current(), 6);
assert_eq!(FAMILY.format_string(6), "app/6");
assert_eq!(FAMILY.version_of("app/4"), Some(4));
assert_eq!(FAMILY.version_of("old/3"), Some(3), "the legacy tag is read");
assert_eq!(FAMILY.version_of("other/3"), None);
assert_eq!(FAMILY.pending_names(), vec!["d".to_string(), "e".to_string()]);
}
#[test]
fn a_file_is_taken_apart_or_refused_with_the_apps_words() {
let mut v = json!({"format": "app/6", "format_pending": ["d"], "x": 1});
assert_eq!(FAMILY.take_version(&mut v), Ok((6, Some(vec!["d".to_string()]))));
assert!(v.get("format_pending").is_none(), "taken out");
assert_eq!(FAMILY.take_version(&mut json!({})), Ok((1, None)), "untagged");
let refused = |mut v: Value| FAMILY.take_version(&mut v).unwrap_err().0;
assert_eq!(refused(json!([])), "not a App design (expected a JSON object)");
assert_eq!(refused(json!({"format": "other/2"})), "not a App design (format `other/2`)");
assert_eq!(refused(json!({"format": 2})), "not a App design (format is not a string)");
assert_eq!(refused(json!({"format": "app/0"})), "not a App design (format 0)");
assert_eq!(refused(json!({"format": "app/2", "format_pending": [1]})), "damaged design: `format_pending` is not a short list of names");
}
#[test]
fn every_step_runs_once_in_order() {
assert_eq!(steps_run(1, None), ["1to2", "2to3", "a", "c", "d", "e"]);
assert_eq!(steps_run(4, None), ["c", "d", "e"]);
assert_eq!(steps_run(6, None), Vec::<String>::new());
// A file written while `d` was pending has it: only `e` runs.
assert_eq!(steps_run(6, Some(&["d".to_string()])), ["e"]);
// A file of format 5 written while `c` was still pending, without it: `c` runs first.
assert_eq!(steps_run(5, Some(&[])), ["c", "d", "e"]);
// Under its other spelling `c` counts as had.
assert_eq!(steps_run(5, Some(&["old-c".to_string()])), ["d", "e"]);
// A list beside a hand-written format means nothing.
assert_eq!(steps_run(2, Some(&["d".to_string()])), ["2to3", "a", "c", "d", "e"]);
let mut v = json!({});
assert_eq!(FAMILY.upgrade(&mut v, 0, None), Err(AppError("not a App design (format 0)".into())));
}
#[test]
fn a_newer_file_is_told_by_its_number_or_an_unknown_change() {
assert_eq!(FAMILY.newer(6, None), None);
assert_eq!(FAMILY.newer(6, Some(&["d".to_string()])), None);
assert_eq!(FAMILY.newer(7, None), Some(Newer { format: 7, reads_up_to: 6, unknown_changes: Vec::new() }));
assert_eq!(FAMILY.newer(6, Some(&["z".to_string()])), Some(Newer { format: 6, reads_up_to: 6, unknown_changes: vec!["z".to_string()] }));
assert_eq!(FAMILY.newer(5, Some(&["old-c".to_string()])), None, "an alias of a known change");
assert_eq!(FAMILY.newer(4, Some(&["old-c".to_string()])), Some(Newer { format: 4, reads_up_to: 6, unknown_changes: vec!["old-c".to_string()] }));
assert_eq!(FAMILY.newer(2, Some(&["z".to_string()])), None, "a list beside a hand-written format means nothing");
}
#[test]
fn a_legacy_prefix_that_extends_the_primary_one_is_read() {
let nested = Family { prefix: "app/", legacy_prefix: Some("app/design/"), ..FAMILY };
assert_eq!(nested.version_of("app/design/3"), Some(3));
assert_eq!(nested.version_of("app/4"), Some(4));
assert_eq!(nested.version_of("app/design/x"), None);
}
+470
View File
@@ -0,0 +1,470 @@
//! A file a newer build saved: it opens, with a warning, and saving it loses nothing.
//!
//! A newer format number, or a change the file names that this build doesn't have, says a newer
//! build saved the file ([`crate::Family::newer`]). It may use nothing this build lacks, so the app
//! reads it as usual. When part of it can't be read, that part stays out of the way and the rest
//! opens: the app reads the elements of its lists it can ([`NewerFile::keep_element`]), then any
//! other top-level section it can't read is left out whole ([`leave_out_unreadable`],
//! [`read_around`]), and a field it doesn't know, which typed reading would drop, is noticed by
//! comparing the file with what was read ([`NewerFile::compare`]).
//!
//! All of these are named in [`NewerFile::not_understood`] and go back into the file when it is
//! saved ([`NewerFile::restore_into`]), as they came. What can't be put back because the file was
//! changed around it is what `restore_into` returns.
use std::collections::{BTreeMap, HashMap};
use std::sync::Arc;
use serde_json::{Map, Value, json};
use crate::Newer;
/// The most sections left out one at a time to find the one a file can't be read with.
const MAX_SECTION_TRIES: usize = 64;
/// What the file says and holds that this build doesn't read.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct NewerFile {
/// The app, as the notice names it.
pub app: &'static str,
/// The number the file says.
pub format: u32,
/// The number this build reads up to.
pub reads_up_to: u32,
/// The changes the file names that this build doesn't have.
pub unknown_changes: Vec<String>,
/// What this build couldn't read, by name.
pub not_understood: Vec<String>,
/// Fields known records lost in typed reading.
kept: Option<Kept>,
/// Top-level sections left out whole.
sections: Vec<Section>,
/// Elements left out of a top-level list: its key and the element.
elements: Vec<(String, Value)>,
/// The parts of a container this build has no use for: name and bytes. A blob only one of the
/// kept pieces refers to is one of them.
pub parts: Vec<(String, Arc<[u8]>)>,
/// The file's `container.json`, when this build doesn't know what it says.
pub container: Option<Value>,
}
#[derive(Clone, Debug, PartialEq)]
struct Section {
key: String,
value: Value,
/// This build's record of the section, without the file's: while it is unchanged, the file's
/// goes back.
empty: Option<Value>,
}
/// The fields of a record that typed reading dropped.
#[derive(Clone, Debug, PartialEq)]
enum Kept {
Field(Value),
Fields(BTreeMap<String, Kept>),
Items(Vec<(Anchor, Kept)>),
}
/// Which element of a list some kept fields belong to.
#[derive(Clone, Debug, PartialEq)]
enum Anchor {
Id(u64),
Name(String),
/// Its place, good only while the list keeps its length.
Index {
at: usize,
of: usize,
},
}
impl NewerFile {
/// A file of `app` that `newer` says a newer build saved, nothing left out of it yet.
pub fn new(app: &'static str, newer: &Newer) -> NewerFile {
NewerFile {
app,
format: newer.format,
reads_up_to: newer.reads_up_to,
unknown_changes: newer.unknown_changes.clone(),
..NewerFile::default()
}
}
/// What to tell the user, whatever else is said.
pub fn notice(&self) -> String {
let (app, reads, format) = (self.app, self.reads_up_to, self.format);
match self.unknown_changes.first() {
Some(change) if format <= reads => {
format!("Saved by a newer {app} (format {format} with `{change}`, which this version doesn't know). If something's missing, update.")
}
_ => format!("Saved by a newer {app} (format {format}, this version reads up to {reads}). If something's missing, update."),
}
}
/// Every line to report: the notice, then what couldn't be read, by name.
pub fn warnings(&self) -> Vec<String> {
let unread = self.not_understood.iter().map(|what| format!("Not read: {what}. Saving keeps it."));
std::iter::once(self.notice()).chain(unread).collect()
}
/// What a reply says about it.
pub fn report(&self) -> Value {
json!({
"format": self.format,
"reads_up_to": self.reads_up_to,
"unknown_changes": self.unknown_changes,
"not_understood": self.not_understood,
"notice": self.notice(),
})
}
/// Whether the next save needs a container to keep what the file holds: plain JSON has no room
/// for parts beside the file's own, nor for the container's own `container.json`.
pub fn needs_container(&self) -> bool {
!self.parts.is_empty() || self.container.is_some()
}
/// Whether anything kept back names this text (a blob's name in a container).
pub fn mentions(&self, text: &str) -> bool {
fn kept(k: &Kept, text: &str) -> bool {
match k {
Kept::Field(v) => value_mentions(v, text),
Kept::Fields(m) => m.values().any(|x| kept(x, text)),
Kept::Items(l) => l.iter().any(|(_, x)| kept(x, text)),
}
}
self.kept.as_ref().is_some_and(|k| kept(k, text))
|| self.sections.iter().any(|s| value_mentions(&s.value, text))
|| self.elements.iter().any(|(_, v)| value_mentions(v, text))
}
/// Say that a section the reader couldn't finish reading is left out. `empty` is what the
/// app writes for the section while it holds nothing, if anything.
pub fn keep_section(&mut self, key: &str, value: Value, empty: Option<Value>, why: &str) {
self.not_understood.push(format!("section `{key}` ({why})"));
self.sections.push(Section { key: key.into(), value, empty });
}
/// Say that an element of the top-level list `key` is left out, as `note` names it; it goes
/// back at the end of the list when the file is saved.
pub fn keep_element(&mut self, key: &str, element: Value, note: String) {
self.not_understood.push(note);
self.elements.push((key.into(), element));
}
/// Compare the file as it reads (`file`, after the same upgrade as what was read) with what was
/// read of it (`known`, its typed record written out): what `known` lacks is kept and named.
/// An object whose `type` is `placeholder_tag` (an element the app kept whole) is not
/// compared; nor are these top-level fields: those in `container_only` (what a container adds
/// beside the file's own), `format` and `format_pending`, and the sections kept whole (they go
/// back whole).
pub fn compare(&mut self, file: &Value, known: &Value, placeholder_tag: &str, container_only: &[&str]) {
let mut paths = Vec::new();
let mut top_only: Vec<&str> = container_only.to_vec();
top_only.extend(["format", "format_pending"]);
top_only.extend(self.sections.iter().map(|s| s.key.as_str()));
// a list typed writing leaves out holds only elements kept on their own: not kept twice
top_only.extend(self.elements.iter().map(|(key, _)| key.as_str()).filter(|key| known.get(*key).is_none()));
let kept = vanished(file, known, "", &mut paths, placeholder_tag, &top_only);
self.kept = kept;
self.not_understood.extend(paths);
for section in &mut self.sections {
section.empty = known.get(&section.key).cloned();
}
}
/// Write back what was kept of the file into `value`, the file as this build writes it.
/// Returns what couldn't go back, because the file was changed around it.
pub fn restore_into(&self, value: &mut Value) -> Vec<String> {
let mut lost = Vec::new();
if let Some(kept) = &self.kept {
apply(kept, value, "", &mut lost);
}
if let Some(top) = value.as_object_mut() {
for section in &self.sections {
if top.get(&section.key) == section.empty.as_ref() {
top.insert(section.key.clone(), section.value.clone());
} else {
lost.push(format!("section `{}`, replaced by what was edited here", section.key));
}
}
// each list's records indexed once, by the identity records are matched by
let mut indexes: HashMap<&str, Positions> = HashMap::new();
for (key, element) in &self.elements {
let name = element.get("name");
let anchor = anchor_of(element);
match top.entry(key.clone()).or_insert_with(|| json!([])) {
Value::Array(list) => {
let index = indexes.entry(key.as_str()).or_insert_with(|| Positions::of(list));
if anchor.as_ref().is_some_and(|a| index.find(list.len(), a).is_some()) {
lost.push(format!(
"`{key}` element {}, replaced by what was made here",
name.and_then(Value::as_str).unwrap_or("unnamed")
));
} else {
index.add(element, list.len());
list.push(element.clone());
}
}
_ => lost.push(format!("`{key}` element {}, replaced by what was made here", name.and_then(Value::as_str).unwrap_or("unnamed"))),
}
}
}
lost
}
}
/// Whether a string anywhere in `v` is `text`; iterative, so no nesting depth overflows the stack.
fn value_mentions(v: &Value, text: &str) -> bool {
let mut todo = vec![v];
while let Some(v) = todo.pop() {
match v {
Value::String(s) if s == text => return true,
Value::Array(a) => todo.extend(a),
Value::Object(o) => todo.extend(o.values()),
_ => {}
}
}
false
}
/// The first line of a reader's error, without the list of every kind it knows, and without the
/// `"<noun> file: "` its reader starts it with.
pub fn short<'a>(why: &'a str, noun: &str) -> &'a str {
let why = why.split(", expected").next().unwrap_or(why);
why.strip_prefix(&format!("{noun} file: ")).unwrap_or(why)
}
/// Leave a section out and keep it. A section the file needs is left empty.
fn move_out(top: &mut Map<String, Value>, key: &str, why: &str, noun: &str, file: &mut NewerFile) {
let Some(value) = top.remove(key) else { return };
file.not_understood.push(format!("section `{key}` ({})", short(why, noun)));
file.sections.push(Section { key: key.into(), value, empty: None });
}
/// Every top-level section of `top` that doesn't read by itself (`piece`, the app's reading of a
/// file holding just that section) is left out and kept in `file`. `noun` is what a file of the
/// app's is called, which its reader's errors start with.
pub fn leave_out_unreadable<E: std::fmt::Display>(
top: &mut Map<String, Value>,
file: &mut NewerFile,
noun: &str,
piece: impl Fn(&str, &Value) -> Result<(), E>,
) {
let keys: Vec<String> = top.keys().cloned().collect();
for key in &keys {
let value = top.get(key).cloned().unwrap_or_default();
if let Err(why) = piece(key, &value) {
move_out(top, key, &why.to_string(), noun, file);
}
}
}
/// Read `top` with `attempt`, leaving out the sections it can't be read with: if it doesn't read,
/// each section but the `core` ones (which the file can't be read without) is tried left out alone,
/// and as a last resort every one of them is. What is left out is kept in `file`. The error is the
/// first attempt's, when even the core alone doesn't read.
pub fn read_around<D, E: std::fmt::Display>(
top: &mut Map<String, Value>,
file: &mut NewerFile,
core: &[&str],
noun: &str,
attempt: impl Fn(&Map<String, Value>) -> Result<D, E>,
) -> Result<D, String> {
let first = match attempt(top) {
Ok(doc) => return Ok(doc),
Err(why) => why.to_string(),
};
// Sections that are fine by themselves but not together with the rest.
let keys: Vec<String> = top.keys().filter(|k| !core.contains(&k.as_str())).cloned().collect();
// Each try reads the whole file: a file of many sections goes straight to the last resort.
for key in keys.iter().filter(|_| keys.len() <= MAX_SECTION_TRIES) {
let mut without = top.clone();
without.remove(key);
if let Ok(doc) = attempt(&without) {
move_out(top, key, &first, noun, file);
return Ok(doc);
}
}
let mut bare = top.clone();
bare.retain(|key, _| core.contains(&key.as_str()));
match attempt(&bare) {
Ok(doc) => {
for key in &keys {
move_out(top, key, &first, noun, file);
}
Ok(doc)
}
Err(_) => Err(first),
}
}
/// A value a writer leaves out without losing anything: not worth telling anyone about.
pub fn looks_default(v: &Value) -> bool {
match v {
Value::Null => true,
Value::Bool(b) => !b,
Value::Number(n) => n.as_f64() == Some(0.0),
Value::String(s) => s.is_empty(),
Value::Array(a) => a.is_empty(),
Value::Object(o) => o.is_empty(),
}
}
/// What `raw` has that `known`, what was read of it, doesn't. `paths` gets those fields' names.
/// `top_only` are top-level keys left out of the comparison.
fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec<String>, placeholder_tag: &str, top_only: &[&str]) -> Option<Kept> {
match (raw, known) {
(Value::Object(raw), Value::Object(known)) => {
if known.get("type").and_then(Value::as_str) == Some(placeholder_tag) {
return None;
}
let mut kept = BTreeMap::new();
for (key, value) in raw {
if path.is_empty() && top_only.contains(&key.as_str()) {
continue;
}
let inner = if path.is_empty() { key.clone() } else { format!("{path}.{key}") };
match known.get(key) {
None => {
if !looks_default(value) {
paths.push(format!("field `{inner}`"));
}
kept.insert(key.clone(), Kept::Field(value.clone()));
}
Some(known) => {
if let Some(k) = vanished(value, known, &inner, paths, placeholder_tag, top_only) {
kept.insert(key.clone(), k);
}
}
}
}
(!kept.is_empty()).then_some(Kept::Fields(kept))
}
(Value::Array(raw), Value::Array(known)) => {
let mut items = Vec::new();
let positions = Positions::of(known);
for (at, element) in raw.iter().enumerate() {
// an identity typed reading kept: an id it dropped falls back to the name
let anchor = surviving_anchor(element, &positions).unwrap_or(Anchor::Index { at, of: raw.len() });
let Some(partner) = positions.find(known.len(), &anchor).and_then(|i| known.get(i)) else { continue };
let label = match (element.get("name").and_then(Value::as_str), &anchor) {
(Some(name), _) => format!("{path}[{name}]"),
(None, Anchor::Id(id)) => format!("{path}[{id}]"),
_ => format!("{path}[{at}]"),
};
if let Some(k) = vanished(element, partner, &label, paths, placeholder_tag, top_only) {
items.push((anchor, k));
}
}
(!items.is_empty()).then_some(Kept::Items(items))
}
_ => None,
}
}
fn anchor_of(element: &Value) -> Option<Anchor> {
if let Some(id) = element.get("id").and_then(Value::as_u64) {
return Some(Anchor::Id(id));
}
element.get("name").and_then(Value::as_str).map(|name| Anchor::Name(name.to_string()))
}
/// The identity of `element` that `known`'s records still carry: its id, or else its name. When
/// typed reading kept neither, there is none, and the record is matched by its place.
fn surviving_anchor(element: &Value, known: &Positions) -> Option<Anchor> {
let id = element.get("id").and_then(Value::as_u64).map(Anchor::Id);
let name = element.get("name").and_then(Value::as_str).map(|n| Anchor::Name(n.to_string()));
[id, name].into_iter().flatten().find(|anchor| known.find(usize::MAX, anchor).is_some())
}
/// Where each id and name first appears in a list, so matching records is not a scan per record.
struct Positions {
ids: HashMap<u64, usize>,
names: HashMap<String, usize>,
}
impl Positions {
fn of(list: &[Value]) -> Positions {
let (mut ids, mut names) = (HashMap::new(), HashMap::new());
for (i, x) in list.iter().enumerate() {
if let Some(id) = x.get("id").and_then(Value::as_u64) {
ids.entry(id).or_insert(i);
}
if let Some(name) = x.get("name").and_then(Value::as_str) {
names.entry(name.to_string()).or_insert(i);
}
}
Positions { ids, names }
}
/// Count `x`, now at `i`, in.
fn add(&mut self, x: &Value, i: usize) {
if let Some(id) = x.get("id").and_then(Value::as_u64) {
self.ids.entry(id).or_insert(i);
}
if let Some(name) = x.get("name").and_then(Value::as_str) {
self.names.entry(name.to_string()).or_insert(i);
}
}
/// The position of the record `anchor` names in a list of `len` records (an index anchor
/// only while the list keeps its length).
fn find(&self, len: usize, anchor: &Anchor) -> Option<usize> {
match anchor {
Anchor::Id(id) => self.ids.get(id).copied(),
Anchor::Name(name) => self.names.get(name).copied(),
Anchor::Index { at, of } => (len == *of && at < of).then_some(*at),
}
}
}
/// Put the kept fields back. An element that is gone was deleted here, its fields with it; one
/// whose place can't be told any more, and a field whose record is gone, go into `lost`.
fn apply(kept: &Kept, value: &mut Value, path: &str, lost: &mut Vec<String>) {
match kept {
Kept::Field(_) => {}
Kept::Fields(fields) => {
let Some(object) = value.as_object_mut() else {
lost.push(format!("field `{path}`"));
return;
};
for (key, sub) in fields {
let inner = if path.is_empty() { key.clone() } else { format!("{path}.{key}") };
match (sub, object.get_mut(key)) {
(Kept::Field(v), None) => {
object.insert(key.clone(), v.clone());
}
// the edited value stays; a retained one it replaced is reported
(Kept::Field(v), Some(existing)) => {
if existing != v {
lost.push(format!("field `{inner}`"));
}
}
(sub, Some(inner_value)) => apply(sub, inner_value, &inner, lost),
// The list it was in is empty and not written: its elements were deleted.
(Kept::Items(_), None) => {}
(_, None) => lost.push(format!("field `{inner}`")),
}
}
}
Kept::Items(items) => {
let Some(list) = value.as_array_mut() else {
lost.push(format!("field `{path}`"));
return;
};
let positions = Positions::of(list);
let len = list.len();
for (anchor, sub) in items {
match (positions.find(len, anchor).and_then(|i| list.get_mut(i)), anchor) {
(Some(element), _) => apply(sub, element, path, lost),
(None, Anchor::Index { at, .. }) => lost.push(format!("element {at} of `{path}`")),
(None, _) => {}
}
}
}
}
}
#[cfg(test)]
#[path = "newer_tests.rs"]
mod tests;
+218
View File
@@ -0,0 +1,218 @@
use super::*;
fn newer() -> Newer {
Newer { format: 9, reads_up_to: 8, unknown_changes: Vec::new() }
}
/// What typed reading dropped is named and goes back on saving, where it was.
#[test]
fn dropped_fields_are_named_and_put_back() {
let file = json!({"name": "A", "format": "app/9", "items": [{"id": 1, "size": 2, "colour": "red"}], "extra": {"x": 1}});
let known = json!({"name": "A", "items": [{"id": 1, "size": 2}]});
let mut said = NewerFile::new("App", &newer());
said.compare(&file, &known, "placeholder", &[]);
assert_eq!(said.not_understood, ["field `extra`", "field `items[1].colour`"]);
let mut saved = json!({"name": "A", "items": [{"id": 1, "size": 3}]});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"name": "A", "items": [{"id": 1, "size": 3, "colour": "red"}], "extra": {"x": 1}}));
assert_eq!(said.notice(), "Saved by a newer App (format 9, this version reads up to 8). If something's missing, update.");
}
/// A section that can't be read is left out, kept, and goes back while it is unchanged here.
#[test]
fn unreadable_sections_are_left_out_and_kept() {
let mut top = match json!({"name": "A", "good": 1, "bad": 2}) {
Value::Object(m) => m,
_ => unreachable!(),
};
let mut said = NewerFile::new("App", &newer());
leave_out_unreadable(&mut top, &mut said, "design", |key: &str, _: &Value| if key == "bad" { Err("design file: no") } else { Ok(()) });
assert!(!top.contains_key("bad"));
assert_eq!(said.not_understood, ["section `bad` (no)"]);
let read =
read_around(
&mut top,
&mut said,
&["name"],
"design",
|top: &Map<String, Value>| {
if top.contains_key("good") { Err("design file: clash") } else { Ok(top.len()) }
},
);
assert_eq!(read, Ok(1));
assert_eq!(said.not_understood, ["section `bad` (no)", "section `good` (clash)"]);
let mut saved = json!({"name": "A"});
said.compare(&saved.clone(), &saved.clone(), "placeholder", &[]);
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"name": "A", "good": 1, "bad": 2}));
}
/// Notices, warnings and reports name what was left out, and kept values can be searched.
#[test]
fn warnings_report_and_mentions_cover_everything_kept() {
let mut said = NewerFile::new("App", &Newer { format: 8, reads_up_to: 8, unknown_changes: vec!["curves".into()] });
assert_eq!(said.notice(), "Saved by a newer App (format 8 with `curves`, which this version doesn't know). If something's missing, update.");
assert!(!said.needs_container());
said.keep_section("plugins", json!({"tool": "ruler"}), Some(json!({})), "unknown");
said.keep_element("shapes", json!({"name": "star", "points": ["tip"]}), "shape `star`".into());
let file = json!({"shapes": [{"name": "a", "tags": ["deep"], "more": {"x": "nested"}}], "n": 1});
said.compare(&file, &json!({"shapes": [{"name": "a"}], "n": 1}), "placeholder", &[]);
assert_eq!(said.warnings()[0], said.notice());
assert!(said.warnings().contains(&"Not read: section `plugins` (unknown). Saving keeps it.".to_string()));
let report = said.report();
assert_eq!(report["format"], 8);
assert_eq!(report["unknown_changes"], json!(["curves"]));
assert_eq!(report["notice"], json!(said.notice()));
for text in ["ruler", "tip", "deep", "nested"] {
assert!(said.mentions(text), "{text}");
}
assert!(!said.mentions("absent"));
said.parts.push(("blob".into(), Arc::from(&b"x"[..])));
assert!(said.needs_container());
}
/// A kept section or element that was replaced here is reported lost, not written over.
#[test]
fn replaced_sections_and_elements_are_reported_lost() {
let mut said = NewerFile::new("App", &newer());
said.keep_section("plugins", json!({"tool": "ruler"}), None, "unknown");
said.keep_element("shapes", json!({"name": "star"}), "shape `star`".into());
said.keep_element("shapes", json!({"points": 3}), "an unnamed shape".into());
let mut saved = json!({"plugins": {"edited": true}, "shapes": [{"name": "star"}]});
let lost = said.restore_into(&mut saved);
assert_eq!(lost, ["section `plugins`, replaced by what was edited here", "`shapes` element star, replaced by what was made here"]);
assert_eq!(saved["shapes"], json!([{"name": "star"}, {"points": 3}]));
let mut fresh = json!({});
said.sections.clear();
assert!(said.restore_into(&mut fresh).is_empty());
assert_eq!(fresh["shapes"], json!([{"name": "star"}, {"points": 3}]));
}
/// Retained container metadata alone needs a container to survive the save.
#[test]
fn container_metadata_alone_needs_a_container() {
let mut said = NewerFile::new("App", &newer());
assert!(!said.needs_container());
said.container = Some(json!({"layout": 2}));
assert!(said.needs_container());
}
/// An edit that replaces a retained field keeps the edit and reports the replaced value.
#[test]
fn a_replaced_retained_field_is_reported() {
let mut said = NewerFile::new("App", &newer());
said.compare(&json!({"extra": 1}), &json!({}), "placeholder", &[]);
let mut edited = json!({"extra": 2});
assert_eq!(said.restore_into(&mut edited), ["field `extra`"]);
assert_eq!(edited, json!({"extra": 2}));
let mut same = json!({"extra": 1});
assert!(said.restore_into(&mut same).is_empty());
}
/// Records are matched by id and by name across long lists, and an index anchor only while the
/// list keeps its length.
#[test]
fn records_match_by_id_name_or_unchanged_index() {
let records: Vec<Value> = (0..2000).map(|i| json!({"id": i, "extra": i})).collect();
let known: Vec<Value> = (0..2000).map(|i| json!({"id": i})).collect();
let mut said = NewerFile::new("App", &newer());
said.compare(&json!({"items": records}), &json!({"items": known}), "placeholder", &[]);
let mut saved = json!({"items": (0..2000).rev().map(|i| json!({"id": i})).collect::<Vec<_>>()});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved["items"][0], json!({"id": 1999, "extra": 1999}));
let mut by_name = NewerFile::new("App", &newer());
by_name.compare(&json!({"l": [{"name": "a", "x": 1}, [1, 2]]}), &json!({"l": [{"name": "a"}, [1]]}), "placeholder", &[]);
let mut renamed = json!({"l": [{"name": "a"}]});
by_name.restore_into(&mut renamed);
assert_eq!(renamed["l"][0], json!({"name": "a", "x": 1}));
}
/// A section kept whole goes back whole, even when this build's record of it has fields.
#[test]
fn a_section_kept_whole_is_not_compared_field_by_field() {
let mut said = NewerFile::new("App", &newer());
said.keep_section("plugins", json!({"mode": "future", "payload": 1}), None, "unreadable");
let file = json!({"name": "A", "plugins": {"mode": "future", "payload": 1}});
said.compare(&file, &json!({"name": "A", "plugins": {"mode": "none"}}), "placeholder", &[]);
let mut saved = json!({"name": "A", "plugins": {"mode": "none"}});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, file);
}
/// A kept element goes back only while no record has its id or name.
#[test]
fn a_kept_element_keeps_record_identity() {
let mut said = NewerFile::new("App", &newer());
said.keep_element("items", json!({"id": 7, "kind": "future"}), "element 7".into());
said.keep_element("items", json!({"kind": "anonymous"}), "unnamed".into());
let mut clash = json!({"items": [{"id": 7, "kind": "made here"}]});
assert_eq!(said.restore_into(&mut clash).len(), 1);
assert_eq!(clash, json!({"items": [{"id": 7, "kind": "made here"}, {"kind": "anonymous"}]}));
let mut free = json!({"items": []});
assert!(said.restore_into(&mut free).is_empty());
assert_eq!(free, json!({"items": [{"id": 7, "kind": "future"}, {"kind": "anonymous"}]}));
}
/// Container-only fields are left out at the top level only.
#[test]
fn container_only_fields_are_top_level() {
let mut said = NewerFile::new("App", &newer());
let file = json!({"thumbnail": "t", "layer": {"thumbnail": "nested"}});
said.compare(&file, &json!({"layer": {}}), "placeholder", &["thumbnail"]);
assert_eq!(said.not_understood, ["field `layer.thumbnail`"]);
let mut saved = json!({"layer": {}});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"layer": {"thumbnail": "nested"}}));
}
/// An element kept on its own is not kept again with its list when typed writing leaves the list out.
#[test]
fn a_kept_element_is_not_restored_twice() {
let mut said = NewerFile::new("App", &newer());
said.keep_element("items", json!({"kind": "future"}), "unnamed".into());
said.compare(&json!({"items": [{"kind": "future"}]}), &json!({}), "placeholder", &[]);
let mut saved = json!({});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"items": [{"kind": "future"}]}));
}
/// A record whose id typed reading drops is matched by its name, and its fields go back.
#[test]
fn a_dropped_id_falls_back_to_the_name() {
let mut said = NewerFile::new("App", &newer());
said.compare(&json!({"items": [{"id": 7, "name": "a", "extra": 1}]}), &json!({"items": [{"name": "a"}]}), "placeholder", &[]);
let mut saved = json!({"items": [{"name": "a"}]});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"items": [{"name": "a", "id": 7, "extra": 1}]}));
}
/// Kept elements sharing a list are checked against one index, the ones put back included.
#[test]
fn kept_elements_check_each_other() {
let mut said = NewerFile::new("App", &newer());
said.keep_element("items", json!({"id": 1, "v": "first"}), "1".into());
said.keep_element("items", json!({"id": 1, "v": "again"}), "1 again".into());
let mut saved = json!({"items": []});
assert_eq!(said.restore_into(&mut saved).len(), 1);
assert_eq!(saved, json!({"items": [{"id": 1, "v": "first"}]}));
// deep nesting is searched without recursion
let mut deep = json!("needle");
for _ in 0..20_000 {
deep = Value::Array(vec![deep]);
}
said.keep_section("deep", deep, None, "test");
assert!(said.mentions("needle"));
// serde_json drops deep values recursively; leak it rather than overflow the test thread
std::mem::forget(said);
}
/// A record whose whole identity typed reading drops is matched by its place while the list keeps
/// its length, so an unchanged save keeps its fields.
#[test]
fn a_record_without_a_surviving_identity_is_matched_by_place() {
let mut said = NewerFile::new("App", &newer());
said.compare(&json!({"items": [{"id": 7, "extra": 1}]}), &json!({"items": [{}]}), "placeholder", &[]);
let mut saved = json!({"items": [{}]});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"items": [{"id": 7, "extra": 1}]}));
}
+3
View File
@@ -19,6 +19,9 @@ heic-rs = { version = "=0.1.4", default-features = false, features = ["std"], op
heic-rs = { version = "=0.1.4", default-features = false, features = ["std", "parallel"], optional = true }
rayon = { workspace = true, optional = true }
[dev-dependencies]
proptest = { workspace = true }
[features]
default = ["decode"]
# The decoder (probe, decode, decode_thumbnail, read_metadata): heic-rs for the container and HEVC.
+17 -4
View File
@@ -36,11 +36,24 @@ Errors are `Error::Unsupported` (image sequences, overlays, 4:2:2/4:4:4, …), `
`Error::Malformed`. It never panics: every call into heic-rs runs under `catch_unwind`, and a panic
inside it (PhotoCraft's fuzzing found two in 0.1.1) becomes `Error::Malformed`.
Features: `decode` (default) is the decoder above. `metadata` adds `craft_heif::items::read`: the
first `Exif` item and XMP packet from the box structure alone (`iinf`, `iloc`, `idat`; at most 4096
boxes, 1024 items, 1024 extents per item, 32 MiB per item), without heic-rs. A build with
Features: `decode` (default) is the decoder above. `metadata` adds `craft_heif::items`, which reads
the `Exif` and XMP items from the box structure alone (`pitm`, `iinf`, `iref`, `iloc`, `idat`),
without heic-rs:
- `items::read` returns the photo's own EXIF and XMP: the items `cdsc` ties to the primary image or
to an image it is made from (`dimg`: grid tiles, `iden`, `iovl`, `tmap`), else those tied to no
image. A thumbnail's or depth map's metadata is not the photo's.
- `items::scan` returns every EXIF and XMP item, whichever image it describes, and says when it
can't be sure it saw them all: `Scan::complete` is false for an incomplete or damaged item
catalog, and an item whose bytes can't be read comes back as `Unreadable`. An app that must
prove a file has no location (before an export that removes it) refuses when unsure.
- `items::exif_tiff` finds the TIFF stream in an `Exif` item, with or without a correct offset
field or an `Exif\0\0` prefix.
The reader walks at most 65,536 boxes and walks `iloc` once. It gathers at most 16 MiB per item,
and a scan gathers at most 64 MiB in all. A build with
`default-features = false, features = ["metadata"]` has no heic-rs dependency, and its
`read_metadata` is that reader. With both features, a test checks that the two readers agree.
`read_metadata` is `items::read`. With both features, a test checks that the two readers agree.
Feature `testdata` exposes `craft_heif::testdata`: lossless synthetic HEIF files (PCM-coded
32 × 32 HEVC pictures, grids, every property above) for the tests of the crates above.
+666 -414
View File
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+3
View File
@@ -13,7 +13,10 @@ workspace = true
[features]
corpus = []
default = []
# Opt in to threaded AVIF encoding without enabling AVIF (or assembly) by itself.
threads = ["ravif?/threading", "rav1e?/threading"]
sgi = []
sunraster = []
avif = ["image/avif", "dep:ravif", "dep:rav1e", "dep:avif-serialize"]
heif = ["dep:craft-heif"]
# AVIF decode: craft-video's clean-room pure-Rust AV1 decoder. Native targets only (on wasm32 a
+9 -2
View File
@@ -43,7 +43,7 @@ craft-image-codecs = { git = "https://github.com/storytold/craft-libs", rev = "<
| TIFF | yes | yes | U8, U16, F32 | all six (CMYK, CMYK+A included) | yes | yes (tag 34675) | no | yes (tag 700) | yes | yes (Description, Make, Model, Software, DateTime, Artist, Copyright) | no | `tiff` |
| WebP | yes | yes | U8 | RGB, RGBA | yes | yes | yes | yes | no | no | yes (VP8, when `webp_lossless` is off) | `image-webp` / built-in VP8 |
| GIF | yes | yes | U8 | RGBA | 1-bit | no | no | no | no | no | yes (256-colour palette) | `image` |
| BMP | yes | yes | U8 | RGB, RGBA | yes | no | no | no | no | no | no | `image` |
| BMP | yes | yes | U8 | RGB, RGBA | yes | no | no | no | yes (header pixels per metre) | no | no | `image` |
| TGA | yes | yes | U8 | Gray, GrayA, RGB, RGBA | yes | no | no | no | no | no | no | `image` (read) / built in (RLE write, one scanline per packet) |
| ICO | yes | yes | U8 | RGBA (at most 256×256) | yes | no | no | no | no | no | no | `image` |
| DDS | yes | yes | U8 | Gray, GrayA, RGB, RGBA | yes | no | no | no | no | no | yes (BC1–BC3, when `EncodeOptions::dds` asks for one) | built in |
@@ -57,6 +57,7 @@ craft-image-codecs = { git = "https://github.com/storytold/craft-libs", rev = "<
| Cineon | yes | yes | U16 (10-bit code values read and written exactly through U16) | Gray, RGB | no | no | no | no | no | no | no | built in |
| DICOM | yes | **no** | U8, U16 | Gray, RGB | no | no | no | no | no | no | n/a | built in |
| Amiga IFF (ILBM/PBM) | yes | yes (24-bit ILBM) | U8 | RGB, RGBA | yes (1-bit mask plane on write) | no | no | no | no | no | no | built in |
| Sun Raster (`.ras`) | with feature `sunraster` (incl. RLE and colour maps) | with feature `sunraster` (uncompressed) | U8 | Gray, RGB, RGBA | yes | no | no | no | no | no | no | `sunraster` (off by default) |
| SGI (`.sgi`, `.rgb`, `.bw`) | with feature `sgi` | with feature `sgi` (RLE) | U8, U16 | Gray, Gray+A, RGB, RGBA | yes | no | no | no | no | no | no | `sgi` (off by default) |
| JPEG-LS (`.jls`) | yes (any NEAR; one scan per component) | yes (lossless) | U8, U16 | Gray, RGB | no | no | no | no | no | no | no | built in |
| AVIF | with feature `avif-decode` (native only) | only with feature `avif` | U8, U16 (10/12-bit decodes to U16; PQ/HLG to linear F32) | RGB, RGBA | yes (auxiliary alpha) | read (`colr` prof/rICC) | read | read | no | no | yes | `craft-video` AV1 decoder; `image` + `ravif` to write |
@@ -78,7 +79,8 @@ the same.
Palette BMP and PNG pixels decode as RGB(A) in `Image`; `bmp_palette` and `png_palette` expose the
embedded table (and a PNG's transparent entry) so `photocraft-io` can open them in Indexed Color mode.
`bmp_palette` returns the colours the header declares, not the decoder's 256-entry padding.
`encode_bmp_indexed` writes the inverse: indices plus a table as an 8-bit palette BMP.
`encode_bmp_indexed` writes the inverse: indices plus a table as an 8-bit palette BMP;
`encode_bmp_indexed_with_dpi` also writes a resolution in the header.
## Documented asymmetries and limitations
@@ -235,6 +237,11 @@ embedded table (and a PNG's transparent entry) so `photocraft-io` can open them
the EXIF and XMP resolution are rewritten to the image's (`export_exif`, `export_xmp`): the
RATIONAL values in place, ResolutionUnit set to 2, an entry that can't be rewritten removed
from IFD0 without moving any other data. IFD1 (the thumbnail's) is left as it is.
* **Resolution without decoding** (module `resolution_probe`): `file_resolution(bytes)` reads
the pixels per inch a JPEG, PNG, WebP, BMP or classic TIFF records from its headers alone, for
placing a picture at its physical size before (or without) decoding it. JPEG, PNG, WebP and
TIFF answer as `decode` would; BMP reads the info header's pixels per metre. Values outside
1–100 000 ppi count as none.
* **JPEG**
* 8-bit only. Neither decoder backend supports 12-bit.
* A file cut off inside its image data decodes leniently (a baseline JPEG's missing rows come
+3
View File
@@ -1,6 +1,7 @@
pub(crate) mod avif;
#[cfg(all(feature = "avif-decode", not(target_arch = "wasm32")))]
pub(crate) mod avif_decode;
pub(crate) mod bmp;
pub(crate) mod dds;
pub(crate) mod dds_bc;
pub(crate) mod deep_exr;
@@ -22,6 +23,8 @@ pub(crate) mod png;
pub(crate) mod pnm;
#[cfg(feature = "sgi")]
pub(crate) mod sgi;
#[cfg(feature = "sunraster")]
pub(crate) mod sunraster;
pub(crate) mod tga;
pub(crate) mod tiff;
pub(crate) mod tiff_exif;
+106
View File
@@ -0,0 +1,106 @@
//! A BMP's resolution: `biXPelsPerMeter` and `biYPelsPerMeter`, the pixels per metre of the
//! BITMAPINFOHEADER and every later header (V4, V5, OS/2 2.x), at file offsets 38 and 42. The
//! 12-byte BITMAPCOREHEADER of OS/2 1.x has no such fields. The `image` crate reads and writes
//! the pixels only, and writes 0 here, which readers take as "no resolution".
use super::png::dpi_of_ppm;
/// Bytes before the fields: the 14-byte BITMAPFILEHEADER, then the DIB header's size, width,
/// height, planes, bit count, compression and image size.
const X_PPM: usize = 38;
const Y_PPM: usize = 42;
/// The shortest DIB header that carries the resolution (BITMAPINFOHEADER).
const INFO_HEADER: u32 = 40;
const METERS_PER_INCH: f64 = 0.0254;
/// The little-endian 4 bytes at `at`, or `None` past the end of `bytes`.
fn le4(bytes: &[u8], at: usize) -> Option<[u8; 4]> {
bytes.get(at..at.checked_add(4)?)?.try_into().ok()
}
/// Whether `bytes` start a BMP whose header has the resolution fields.
fn has_resolution_fields(bytes: &[u8]) -> bool {
bytes.starts_with(b"BM") && le4(bytes, Y_PPM).is_some() && le4(bytes, 14).map(u32::from_le_bytes).is_some_and(|dib| dib >= INFO_HEADER)
}
/// The resolution in pixels per inch a BMP records, or `None` when its header has no fields for it
/// or either is 0 or negative (most writers leave them 0). A whole number of ppi stored rounded to
/// whole pixels per metre (300 ppi = 11811) reads back whole, as a PNG's `pHYs` does.
pub(crate) fn resolution(bytes: &[u8]) -> Option<(f32, f32)> {
if !has_resolution_fields(bytes) {
return None;
}
let field = |at: usize| le4(bytes, at).map(i32::from_le_bytes).and_then(|v| u32::try_from(v).ok());
let (x, y) = (field(X_PPM)?, field(Y_PPM)?);
(x > 0 && y > 0).then(|| (dpi_of_ppm(x), dpi_of_ppm(y)))
}
/// Pixels per metre for `ppi`, rounded as Photoshop and the PNG `pHYs` writer do (300 ppi = 11811),
/// or `None` for a resolution that has no positive whole number of pixels per metre.
fn ppm_of_dpi(ppi: f32) -> Option<i32> {
let ppm = (f64::from(ppi) / METERS_PER_INCH).round();
(ppm.is_finite() && ppm >= 1.0).then(|| ppm.min(f64::from(i32::MAX)) as i32)
}
/// Writes `dpi` into the resolution fields of the BMP in `bytes`. A header without the fields, or
/// a resolution that cannot be stored, leaves the bytes as they are.
pub(crate) fn set_resolution(bytes: &mut [u8], dpi: Option<(f32, f32)>) {
let Some((x, y)) = dpi.and_then(|(x, y)| Some((ppm_of_dpi(x)?, ppm_of_dpi(y)?))) else { return };
if !has_resolution_fields(bytes) {
return;
}
for (at, ppm) in [(X_PPM, x), (Y_PPM, y)] {
if let Some(field) = bytes.get_mut(at..at + 4) {
field.copy_from_slice(&ppm.to_le_bytes());
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// A 46-byte BMP start: file header, a DIB header of `dib` bytes, 11811 px/m each way.
fn start(dib: u32) -> Vec<u8> {
let mut b = vec![0u8; Y_PPM + 4];
b[..2].copy_from_slice(b"BM");
b[14..18].copy_from_slice(&dib.to_le_bytes());
b[X_PPM..X_PPM + 4].copy_from_slice(&11811i32.to_le_bytes());
b[Y_PPM..Y_PPM + 4].copy_from_slice(&11811i32.to_le_bytes());
b
}
#[test]
fn only_headers_with_the_fields_are_read_or_written() {
assert_eq!(resolution(&start(40)), Some((300.0, 300.0)));
assert_eq!(resolution(&start(124)), Some((300.0, 300.0)), "BITMAPV5HEADER");
// OS/2 1.x's 12-byte core header has no resolution: those bytes are pixels or palette.
let mut core = start(12);
assert_eq!(resolution(&core), None);
set_resolution(&mut core, Some((72.0, 72.0)));
assert_eq!(core, start(12), "left as it is");
// Too short, or not a BMP: nothing read, nothing written, no panic.
let mut short = start(40)[..Y_PPM + 3].to_vec();
assert_eq!(resolution(&short), None);
set_resolution(&mut short, Some((72.0, 72.0)));
assert_eq!(short, start(40)[..Y_PPM + 3]);
let mut other = start(40);
other[0] = b'X';
assert_eq!(resolution(&other), None);
set_resolution(&mut other, Some((72.0, 72.0)));
assert_eq!(&other[X_PPM..X_PPM + 4], &11811i32.to_le_bytes(), "not a BMP: left as it is");
}
#[test]
fn resolutions_without_a_whole_pixel_per_metre_are_not_written() {
for bad in [0.0, -300.0, 0.01, f32::NAN, f32::INFINITY] {
let mut b = start(40);
set_resolution(&mut b, Some((bad, 300.0)));
assert_eq!(b, start(40), "{bad}");
}
// A huge one is capped at the field's maximum rather than wrapping negative.
let mut b = start(40);
set_resolution(&mut b, Some((1e12, 300.0)));
assert_eq!(&b[X_PPM..X_PPM + 4], &i32::MAX.to_le_bytes());
}
}
+22 -7
View File
@@ -304,19 +304,14 @@ pub(crate) fn decode_deep(meta: &MetaData, bytes: &[u8], limits: &Limits) -> Res
let bytes_per_sample: usize = exr_types.iter().map(|t| sample_bytes(*t)).sum();
let multipart = meta.requirements.has_multiple_layers;
// The offset tables follow the headers: `chunk_count` u64 per header, in header order.
// Our part's table starts after the tables of all earlier parts.
let mut table_pos = header_end(bytes)?;
for h_ in meta.headers.iter().take(header_index) {
table_pos = table_pos.saturating_add((h_.chunk_count.min(1 << 24) as u64).saturating_mul(8));
}
// A chunk offset before the end of the last offset table points into table data.
let (table_pos, tables_end) = offset_tables(meta, bytes, header_index)?;
let mut cur = Cursor::new(bytes);
seek_to(&mut cur, table_pos)?;
let mut offsets = Vec::with_capacity(header.chunk_count.min(1 << 20));
for _ in 0..header.chunk_count {
offsets.push(u64(&mut cur)?);
}
let tables_end = cur.position();
// Block geometry.
let (tiles_x, tile_w, tile_h, slots) = match &header.blocks {
@@ -565,6 +560,26 @@ fn header_end(bytes: &[u8]) -> Result<u64, CodecError> {
Ok(cur.position())
}
/// Where part `header_index`'s offset table starts and where the last part's table ends. The
/// tables follow the headers, `chunk_count` u64 per header in header order, and the chunks
/// follow the last table. Sizes that overflow, or tables that run past the end of the file,
/// are errors.
fn offset_tables(meta: &MetaData, bytes: &[u8], header_index: usize) -> Result<(u64, u64), CodecError> {
let mut table_pos = header_end(bytes)?;
let mut tables_end = table_pos;
for (i, h) in meta.headers.iter().enumerate() {
let len = u64::try_from(h.chunk_count).ok().and_then(|n| n.checked_mul(8));
tables_end = len.and_then(|len| tables_end.checked_add(len)).ok_or_else(|| err("the offset tables are too large"))?;
if i < header_index {
table_pos = tables_end;
}
}
if tables_end > bytes.len() as u64 {
return Err(err(format!("the offset tables end at byte {tables_end}, past the end of the file")));
}
Ok((table_pos, tables_end))
}
/// Composite the deep samples into a flat image: per pixel, samples sorted by Z (stable; the
/// file is assumed messy), front-to-back `over` with premultiplied colour, un-premultiplied
/// into straight alpha at the end. Without any alpha channel the front-most sample wins,
+95 -21
View File
@@ -198,6 +198,15 @@ fn data_end(b: &[u8]) -> (DataEnd, u64) {
data_end_mode(b, false)
}
/// Whether JPEG `b` holds too little scan data for a `w` × `h` frame to be worth decoding: no scan
/// data at all, or under one byte for every [`MAX_PIXELS_PER_BYTE`] pixels (counted on the scan
/// data the marker walk crosses, so APP padding doesn't count). `zune-jpeg` allocates the whole
/// output before it reads a scan, so a few bytes could otherwise ask for a frame of any size.
pub(crate) fn scan_too_short(b: &[u8], w: u32, h: u32) -> bool {
let (end, scan_bytes) = data_end(b);
end == DataEnd::Empty || u64::from(w) * u64::from(h) / MAX_PIXELS_PER_BYTE > scan_bytes
}
/// Whether the marker walk actually reaches the first EOI, with malformed segment lengths refused.
pub(crate) fn reaches_eoi(b: &[u8]) -> bool {
data_end_mode(b, true).0 == DataEnd::Complete
@@ -322,6 +331,40 @@ fn without_extended_xmp(b: &[u8]) -> Option<Vec<u8>> {
Some(out)
}
/// Whether the first frame header of `b` samples its first component below the largest
/// horizontal or vertical sampling factor of the frame. zune-jpeg 0.5 sizes its upsampler's
/// scratch row from the first component and asserts on some such frames (jpeg-encoder writes
/// CMYK that way: C, M and Y at 1×1, K larger). `false` when no frame header can be read.
fn first_component_subsampled(b: &[u8]) -> bool {
if b.get(..2) != Some(&[0xFF, 0xD8]) {
return false;
}
let mut i = 2;
while let (Some(&0xFF), Some(&m)) = (b.get(i), b.get(i + 1)) {
match m {
0xFF => i += 1,
0x01 | 0xD0..=0xD8 => i += 2,
0xD9 | 0xDA => return false,
_ => {
let Some(len) = b.get(i + 2..i + 4).map(|l| usize::from(u16::from_be_bytes([l[0], l[1]]))) else { return false };
if matches!(m, 0xC0..=0xCF) && !matches!(m, 0xC4 | 0xC8 | 0xCC) {
let Some(seg) = b.get(i + 4..i + 2 + len.max(2)) else { return false };
let n = usize::from(seg.get(5).copied().unwrap_or(0));
let factors: Vec<(u8, u8)> = (0..n).filter_map(|c| seg.get(6 + c * 3 + 1)).map(|f| (f >> 4, f & 0x0F)).collect();
let (Some(&(h0, v0)), Some(hmax), Some(vmax)) =
(factors.first(), factors.iter().map(|f| f.0).max(), factors.iter().map(|f| f.1).max())
else {
return false;
};
return h0 < hmax || v0 < vmax;
}
i += 2 + len;
}
}
}
false
}
/// Whether `craft-jpeg` (rather than `zune-jpeg`) decodes the file: arithmetic coding
/// (SOF9–SOF11), lossless (SOF3), or a DCT frame whose precision isn't 8 bits (#4934, #5003).
/// This includes 12-bit progressive Huffman (SOF2), which `craft-jpeg` supports.
@@ -389,6 +432,30 @@ fn decode_craft_jpeg(bytes: &[u8], limits: &Limits, adobe_transform: Option<u8>,
/// truncated or not, are far above this.
const MAX_PIXELS_PER_BYTE: u64 = 2048;
/// Decode with craft-jpeg: the files [`uses_craft_jpeg`] routes away from `zune-jpeg`, and the
/// ones `zune-jpeg` panics on.
fn decode_whole(bytes: &[u8], limits: &Limits, meta: JpegMeta, end: DataEnd, scan_bytes: u64) -> Result<Image, CodecError> {
// craft-jpeg sizes its buffers from the frame header before it reads a scan.
if scan_bytes == 0 {
return Err(err("the file has no scan data"));
}
if let Some((w, h, _)) = meta.frame
&& end == DataEnd::Truncated
// Only truncated scans: arithmetic coding (SOF9–SOF11) can spend well under a bit on
// a flat block, so a complete scan may be shorter than this bound.
&& u64::from(w) * u64::from(h) / MAX_PIXELS_PER_BYTE > scan_bytes
{
return Err(err("the truncated scan is too short for the frame it declares"));
}
let (mut img, damaged) = decode_craft_jpeg(bytes, limits, meta.adobe_transform, meta.adobe_marker)?;
img.icc = meta.icc;
img.meta = Metadata { exif: meta.exif, xmp: meta.xmp, dpi: meta.dpi, photoshop_resources: meta.photoshop, ..Default::default() };
if end == DataEnd::Truncated || damaged {
img.warnings.push(DecodeWarning::Truncated { format: F });
}
Ok(img)
}
/// The stored size and EXIF block [`decode`] would find, from the marker segments before the
/// first scan: the first frame header's size, checked against `limits`, and the APP1 EXIF block.
pub(crate) fn probe(bytes: &[u8], limits: &Limits) -> Result<(u32, u32, Option<Vec<u8>>), CodecError> {
@@ -407,26 +474,9 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits) -> Result<Image, CodecError>
if let Some((w, h, nc)) = meta.frame {
limits.check_bytes(w, h, u64::from(nc.max(1)))?;
}
if uses_craft_jpeg(bytes) {
// craft-jpeg sizes its buffers from the frame header before it reads a scan.
if scan_bytes == 0 {
return Err(err("the file has no scan data"));
}
if let Some((w, h, _)) = meta.frame
&& end == DataEnd::Truncated
// Only truncated scans: arithmetic coding (SOF9–SOF11) can spend well under a bit on
// a flat block, so a complete scan may be shorter than this bound.
&& u64::from(w) * u64::from(h) / MAX_PIXELS_PER_BYTE > scan_bytes
{
return Err(err("the truncated scan is too short for the frame it declares"));
}
let (mut img, damaged) = decode_craft_jpeg(bytes, limits, meta.adobe_transform, meta.adobe_marker)?;
img.icc = meta.icc;
img.meta = Metadata { exif: meta.exif, xmp: meta.xmp, dpi: meta.dpi, photoshop_resources: meta.photoshop, ..Default::default() };
if end == DataEnd::Truncated || damaged {
img.warnings.push(DecodeWarning::Truncated { format: F });
}
return Ok(img);
// Where a panic aborts (wasm32), the frames zune-jpeg may panic on can't be caught below.
if uses_craft_jpeg(bytes) || (cfg!(panic = "abort") && first_component_subsampled(bytes)) {
return decode_whole(bytes, limits, meta, end, scan_bytes);
}
let options = DecoderOptions::default().set_strict_mode(false).set_max_width(65535).set_max_height(65535);
let stripped = without_extended_xmp(bytes);
@@ -452,7 +502,12 @@ pub(crate) fn decode(bytes: &[u8], limits: &Limits) -> Result<Image, CodecError>
return Err(err(format!("{what} is too short for the frame it declares")));
}
dec.set_options(dec.options().jpeg_set_out_colorspace(out_cs));
let mut px = dec.decode().map_err(err)?;
// zune-jpeg 0.5 asserts on some subsampled four-component frames whose first component isn't
// the largest (its upsampler's scratch row is sized from that component): craft-jpeg reads them.
let Ok(decoded) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| dec.decode())) else {
return decode_whole(bytes, limits, meta, end, scan_bytes);
};
let mut px = decoded.map_err(err)?;
let expected = w as usize * h as usize * layout.channels();
if px.len() < expected {
return Err(err("short pixel buffer"));
@@ -1018,6 +1073,25 @@ mod tests {
(v * 65535.0 / 4095.0).round() as u16
}
#[test]
fn first_component_subsampling() {
let sof = |comps: &[(u8, u8)]| {
let mut s = vec![0xFF, 0xD8, 0xFF, 0xE0, 0, 4, 0, 0, 0xFF, 0xC2, 0, (8 + 3 * comps.len()) as u8, 8, 0, 8, 0, 8, comps.len() as u8];
for (i, &(h, v)) in comps.iter().enumerate() {
s.extend([i as u8 + 1, (h << 4) | v, 0]);
}
s
};
assert!(first_component_subsampled(&sof(&[(1, 1), (1, 1), (1, 1), (2, 2)])));
assert!(first_component_subsampled(&sof(&[(1, 2), (1, 2), (2, 1)])));
assert!(!first_component_subsampled(&sof(&[(2, 2), (1, 1), (1, 1)])));
assert!(!first_component_subsampled(&sof(&[(1, 1); 4])));
assert!(!first_component_subsampled(&sof(&[])));
assert!(!first_component_subsampled(&[0xFF, 0xD8, 0xFF, 0xDA, 0, 2]));
assert!(!first_component_subsampled(&[0xFF, 0xD8, 0xFF, 0xC0, 0, 20, 8]));
assert!(!first_component_subsampled(b"not a jpeg"));
}
#[test]
fn routing() {
assert!(!uses_craft_jpeg(&[0xFF, 0xD8, 0xFF, 0xC0, 0, 11, 8, 0, 8, 0, 8, 1, 1, 0x11, 0]));
+281
View File
@@ -0,0 +1,281 @@
//! Sun Raster files (`.ras`, `.sun`): 1-bit, 8-bit (gray or colour-mapped), 24-bit RGB and 32-bit
//! RGBA, uncompressed or byte-encoded (RLE) on read; uncompressed on write.
//!
//! Follows the published rasterfile format: a 32-byte big-endian header, an optional colour map,
//! then rows padded to an even number of bytes. In the "standard" type (1) 24- and 32-bit pixels
//! are stored blue-first; in the "RGB" type (3) red-first. Other types (TIFF, IFF, experimental)
//! are reported as unsupported.
use crate::error::CodecError;
use crate::fidelity::Plan;
use crate::format::Format;
use crate::image::{ChannelLayout, Image, SampleType};
use crate::options::Limits;
const F: Format = Format::SunRaster;
const HEADER: usize = 32;
const MAGIC: u32 = 0x59a6_6a95;
fn bad(reason: impl std::fmt::Display) -> CodecError {
CodecError::malformed(F, reason)
}
fn be32(b: &[u8], i: usize) -> u32 {
u32::from_be_bytes([b[i], b[i + 1], b[i + 2], b[i + 3]])
}
/// Whether `b` starts with a Sun rasterfile header.
pub(crate) fn looks_like_sunraster(b: &[u8]) -> bool {
b.len() >= HEADER && be32(b, 0) == MAGIC && matches!(be32(b, 12), 1 | 8 | 24 | 32)
}
/// Expand byte-encoded data (`0x80` escapes a run) until `want` bytes are produced.
fn unrle(src: &[u8], want: usize) -> Result<Vec<u8>, CodecError> {
let mut out = Vec::new();
out.try_reserve_exact(want).map_err(|_| bad("pixel buffer is too large"))?;
let mut i = 0;
while out.len() < want {
let b = *src.get(i).ok_or_else(|| bad("truncated RLE data"))?;
i += 1;
if b != 0x80 {
out.push(b);
continue;
}
let n = *src.get(i).ok_or_else(|| bad("truncated RLE data"))?;
i += 1;
if n == 0 {
out.push(0x80);
continue;
}
let v = *src.get(i).ok_or_else(|| bad("truncated RLE data"))?;
i += 1;
let count = usize::from(n) + 1;
if count > want - out.len() {
return Err(bad("RLE run passes the end of the image"));
}
out.resize(out.len() + count, v);
}
Ok(out)
}
pub(crate) fn decode(b: &[u8], limits: &Limits) -> Result<Image, CodecError> {
if !looks_like_sunraster(b) {
return Err(bad("invalid Sun raster header"));
}
let (w, h, depth, kind, map_type, map_len) = (be32(b, 4), be32(b, 8), be32(b, 12) as usize, be32(b, 20), be32(b, 24), be32(b, 28) as usize);
if !matches!(kind, 0..=3) {
return Err(CodecError::unsupported(F, format!("Sun raster type {kind} is not supported")));
}
if map_type > 2 {
return Err(CodecError::unsupported(F, format!("Sun raster map type {map_type} is not supported")));
}
let layout = match depth {
1 | 8 if map_type != 1 => ChannelLayout::Gray,
1 | 8 | 24 => ChannelLayout::Rgb,
_ => ChannelLayout::Rgba,
};
limits.check(w, h, layout, SampleType::U8)?;
let (wu, hu) = (w as usize, h as usize);
let stride = wu.checked_mul(depth).and_then(|n| n.checked_add(15)).ok_or_else(|| bad("image size overflow"))? / 16 * 2;
let want = stride.checked_mul(hu).ok_or_else(|| bad("image size overflow"))?;
let data_at = HEADER.checked_add(map_len).ok_or_else(|| bad("colour map size overflow"))?;
let map = b.get(HEADER..data_at).ok_or_else(|| bad("truncated colour map"))?;
let rest = &b[data_at..];
let raw;
let rows: &[u8] = if kind == 2 {
raw = unrle(rest, want)?;
&raw
} else {
rest.get(..want).ok_or_else(|| bad("the file is too short for the declared image"))?
};
let channels = layout.channels();
let mut out = Vec::new();
out.try_reserve_exact(wu * hu * channels).map_err(|_| bad("pixel buffer is too large"))?;
let entries = map_len / 3;
for y in 0..hu {
let row = &rows[y * stride..(y + 1) * stride];
for x in 0..wu {
match depth {
1 => {
let bit = (row[x / 8] >> (7 - x % 8)) & 1;
if map_type == 1 {
// The bit is an index into the (two-entry or longer) colour map.
let i = usize::from(bit);
if i >= entries {
return Err(bad("pixel index is outside the colour map"));
}
out.extend_from_slice(&[map[i], map[entries + i], map[2 * entries + i]]);
} else {
// A set bit is black.
out.push(if bit == 1 { 0 } else { 255 });
}
}
8 if map_type == 1 => {
let i = usize::from(row[x]);
if i >= entries {
return Err(bad("pixel index is outside the colour map"));
}
out.extend_from_slice(&[map[i], map[entries + i], map[2 * entries + i]]);
}
8 => out.push(row[x]),
24 => {
let p = &row[3 * x..3 * x + 3];
out.extend_from_slice(&if kind == 3 { [p[0], p[1], p[2]] } else { [p[2], p[1], p[0]] });
}
_ => {
let p = &row[4 * x..4 * x + 4];
// In the standard types the first byte (X in XBGR) is unused, not alpha.
out.extend_from_slice(&if kind == 3 { [p[1], p[2], p[3], p[0]] } else { [p[3], p[2], p[1], 255] });
}
}
}
}
Image::from_u8(w, h, layout, out)
}
/// A buffer of `cap` bytes, or an error instead of an abort when the allocation cannot be made.
fn reserve(cap: usize) -> std::result::Result<Vec<u8>, std::collections::TryReserveError> {
let mut v = Vec::new();
v.try_reserve_exact(cap)?;
Ok(v)
}
/// Writes an uncompressed rasterfile: 8-bit gray, 24-bit RGB or 32-bit RGBA, in the RGB-ordered type.
pub(crate) fn encode(src: &Image, plan: Plan) -> Result<Vec<u8>, CodecError> {
let enc = |msg: String| CodecError::encode(F, msg);
let img = src.converted(plan.layout, plan.sample);
if img.sample_type() != SampleType::U8 {
return Err(enc(format!("Sun raster stores 8 bits per channel, not {:?}", img.sample_type())));
}
let depth: usize = match img.layout() {
ChannelLayout::Gray => 8,
ChannelLayout::Rgb => 24,
ChannelLayout::Rgba => 32,
l => return Err(enc(format!("no Sun raster layout for {l:?}"))),
};
let (w, h) = img.dimensions();
let (wu, hu) = (w as usize, h as usize);
let bytes = depth / 8;
let stride = (wu * bytes).next_multiple_of(2);
let len = stride.checked_mul(hu).and_then(|n| u32::try_from(n).ok()).ok_or_else(|| enc("image too large".into()))?;
let data = img.data();
if data.len() != wu * hu * bytes {
return Err(enc("the image has the wrong number of samples".into()));
}
let mut out = reserve(HEADER.checked_add(len as usize).ok_or_else(|| enc("image too large".into()))?)
.map_err(|_| enc("the image is too large to encode".into()))?;
for v in [MAGIC, w, h, depth as u32, len, 3, 0, 0] {
out.extend_from_slice(&v.to_be_bytes());
}
for y in 0..hu {
let row = &data[y * wu * bytes..(y + 1) * wu * bytes];
if depth == 32 {
// Stored alpha first.
for p in row.as_chunks::<4>().0 {
out.extend_from_slice(&[p[3], p[0], p[1], p[2]]);
}
} else {
out.extend_from_slice(row);
}
out.resize(HEADER + (y + 1) * stride, 0);
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
fn pattern(w: u32, h: u32, layout: ChannelLayout) -> Image {
let n = w as usize * h as usize * layout.channels();
Image::from_u8(w, h, layout, (0..n).map(|i| ((i * 13 + i / 3) % 253) as u8).collect()).unwrap()
}
#[test]
fn round_trips_gray_rgb_and_rgba_with_odd_widths() {
for layout in [ChannelLayout::Gray, ChannelLayout::Rgb, ChannelLayout::Rgba] {
let img = pattern(13, 7, layout);
let bytes = encode(&img, Plan { layout, sample: SampleType::U8 }).unwrap();
assert!(looks_like_sunraster(&bytes));
let back = decode(&bytes, &Limits::default()).unwrap();
assert_eq!((back.layout(), back.data()), (layout, img.data()));
}
}
fn file(depth: u32, kind: u32, map: &[u8], map_type: u32, w: u32, h: u32, data: &[u8]) -> Vec<u8> {
let mut f = Vec::new();
for v in [MAGIC, w, h, depth, data.len() as u32, kind, map_type, map.len() as u32] {
f.extend_from_slice(&v.to_be_bytes());
}
f.extend_from_slice(map);
f.extend_from_slice(data);
f
}
#[test]
fn standard_type_is_blue_first_and_rle_expands() {
let f = file(24, 1, &[], 0, 1, 1, &[1, 2, 3, 0]);
assert_eq!(decode(&f, &Limits::default()).unwrap().data(), &[3, 2, 1]);
// 0x80 0x00 is a literal 0x80; 0x80 n v is n + 1 copies of v. The 2x2 image has stride 2.
let f = file(8, 2, &[], 0, 2, 2, &[0x80, 0x00, 0x80, 0x00, 0x80, 0x01, 7]);
assert_eq!(decode(&f, &Limits::default()).unwrap().data(), &[0x80, 0x80, 7, 7]);
// A run that passes the end of the image is an error.
assert!(decode(&file(8, 2, &[], 0, 2, 2, &[0x80, 0x09, 7]), &Limits::default()).is_err());
}
#[test]
fn colour_mapped_and_one_bit_files() {
// Two entries: red plane, green plane, blue plane.
let map = [10, 20, 30, 40, 50, 60];
let f = file(8, 1, &map, 1, 2, 1, &[0, 1]);
let img = decode(&f, &Limits::default()).unwrap();
assert_eq!((img.layout(), img.data()), (ChannelLayout::Rgb, &[10, 30, 50, 20, 40, 60][..]));
let f = file(8, 1, &map, 1, 1, 1, &[5, 0]);
assert!(decode(&f, &Limits::default()).is_err());
let f = file(1, 1, &[], 0, 3, 1, &[0b1010_0000, 0]);
assert_eq!(decode(&f, &Limits::default()).unwrap().data(), &[0, 255, 0]);
}
#[test]
fn one_bit_files_with_a_colour_map_read_as_colour() {
// Two entries: red plane, green plane, blue plane; bit 0 is entry 0.
let map = [10, 20, 30, 40, 50, 60];
let f = file(1, 1, &map, 1, 3, 1, &[0b0100_0000, 0]);
let img = decode(&f, &Limits::default()).unwrap();
assert_eq!((img.layout(), img.data()), (ChannelLayout::Rgb, &[10, 30, 50, 20, 40, 60, 10, 30, 50][..]));
// A one-entry map has no entry for a set bit.
assert!(decode(&file(1, 1, &[1, 2, 3], 1, 1, 1, &[0b1000_0000, 0]), &Limits::default()).is_err());
}
#[test]
fn the_unused_first_byte_of_standard_32_bit_pixels_is_not_alpha() {
let img = decode(&file(32, 1, &[], 0, 1, 1, &[0, 30, 20, 10]), &Limits::default()).unwrap();
assert_eq!((img.layout(), img.data()), (ChannelLayout::Rgba, &[10, 20, 30, 255][..]));
// In the RGB-ordered type the first byte is the alpha the writer stores there.
let img = decode(&file(32, 3, &[], 0, 1, 1, &[7, 10, 20, 30]), &Limits::default()).unwrap();
assert_eq!(img.data(), &[10, 20, 30, 7]);
}
#[test]
fn an_output_buffer_too_large_to_allocate_is_an_error() {
assert!(reserve(usize::MAX).is_err());
assert!(reserve(16).is_ok());
}
#[test]
fn hostile_input_is_an_error_not_a_panic() {
let img = pattern(9, 5, ChannelLayout::Rgb);
let good = encode(&img, Plan { layout: ChannelLayout::Rgb, sample: SampleType::U8 }).unwrap();
for len in 0..good.len() {
let _ = decode(&good[..len], &Limits::default());
}
let mut huge = good.clone();
huge[4..8].copy_from_slice(&u32::MAX.to_be_bytes());
huge[8..12].copy_from_slice(&u32::MAX.to_be_bytes());
assert!(decode(&huge, &Limits::default()).is_err());
let mut bad_map = good;
bad_map[28..32].copy_from_slice(&u32::MAX.to_be_bytes());
assert!(decode(&bad_map, &Limits::default()).is_err());
assert!(matches!(decode(&file(24, 4, &[], 0, 1, 1, &[0; 4]), &Limits::default()), Err(CodecError::Unsupported { .. })));
}
}
+92 -1
View File
@@ -71,6 +71,18 @@ const TAG_DNG_VERSION: u16 = 50706;
/// first IFD is often a reduced preview, so this runs before decoding.
/// Reads only bounds-checked header bytes; anything malformed is "not raw".
fn camera_raw(b: &[u8]) -> bool {
camera_raw_mode(b, false)
}
/// The older LightCraft RAW/TIFF classification policy, sharing this bounded directory walk.
pub(crate) fn legacy_camera_raw(b: &[u8]) -> bool {
camera_raw_mode(b, true)
}
fn camera_raw_mode(b: &[u8], legacy: bool) -> bool {
if legacy && b.get(8..10) == Some(b"CR") {
return true;
}
let le = match b.get(0..4) {
Some(b"II*\0") => true,
Some(b"MM\0*") => false,
@@ -92,8 +104,48 @@ fn camera_raw(b: &[u8]) -> bool {
let value = |e: usize| match u16_at(e.saturating_add(2))? {
3 => u16_at(e.saturating_add(8)).map(u32::from),
4 | 13 => u32_at(e.saturating_add(8)),
1 | 7 if legacy => b.get(e.saturating_add(8)).copied().map(u32::from),
_ => None,
};
// The compatibility policy recognizes Sony lossless YCbCr only when IFD0 names Sony.
let is_sony = if legacy {
u32_at(4)
.and_then(|v| usize::try_from(v).ok())
.and_then(|ifd| {
let n = usize::from(u16_at(ifd)?);
if n > 4096 {
return None;
}
for i in 0..n {
let e = ifd.checked_add(2)?.checked_add(i.checked_mul(12)?)?;
let tag = u16_at(e)?;
let ty = u16_at(e + 2)?;
let count = u32_at(e + 4)? as usize;
if tag == 0x010f {
let matches = (|| {
let size = match ty {
1 | 2 | 6 | 7 => 1usize,
3 | 8 => 2,
4 | 9 | 11 | 13 => 4,
5 | 10 | 12 => 8,
_ => return None,
};
let len = size.checked_mul(count)?;
let start = if len <= 4 { e + 8 } else { u32_at(e + 8)? as usize };
Some(b.get(start..start.checked_add(len)?)?.get(..4)?.eq_ignore_ascii_case(b"SONY"))
})()
.unwrap_or(false);
if matches {
return Some(true);
}
}
}
Some(false)
})
.unwrap_or(false)
} else {
false
};
let mut queue: Vec<u32> = u32_at(4).into_iter().collect();
let mut visited = 0;
while let Some(ifd) = queue.pop() {
@@ -104,6 +156,44 @@ fn camera_raw(b: &[u8]) -> bool {
}
let ifd = ifd as usize;
let Some(n) = u16_at(ifd) else { continue };
if legacy
&& (n > 4096
|| (n > 0
&& ifd
.checked_add(2)
.and_then(|p| usize::from(n - 1).checked_mul(12).and_then(|v| p.checked_add(v)))
.and_then(|p| p.checked_add(8))
.is_none_or(|end| end > b.len())))
{
continue;
}
if is_sony {
let mut cfa = false;
let (mut compression, mut photo, mut samples) = (None, None, None);
let (mut saw_compression, mut saw_photo, mut saw_samples) = (false, false, false);
for i in 0..usize::from(n) {
let e = ifd.saturating_add(2 + 12 * i);
match u16_at(e) {
Some(0x828e) => cfa = true,
Some(0x0103) if !saw_compression => {
saw_compression = true;
compression = value(e);
}
Some(0x0106) if !saw_photo => {
saw_photo = true;
photo = value(e);
}
Some(0x0115) if !saw_samples => {
saw_samples = true;
samples = value(e);
}
_ => {}
}
}
if cfa && compression == Some(7) && photo == Some(6) && samples == Some(3) {
return true;
}
}
for i in 0..usize::from(n) {
let e = ifd.saturating_add(2 + 12 * i);
let (Some(tag), Some(count)) = (u16_at(e), u32_at(e.saturating_add(4))) else {
@@ -111,6 +201,7 @@ fn camera_raw(b: &[u8]) -> bool {
};
match tag {
TAG_DNG_VERSION => return true,
0x0103 if legacy && matches!(value(e), Some(32767 | 32769 | 32770 | 34316 | 34713 | 65000 | 65535)) => return true,
TAG_PHOTOMETRIC if matches!(value(e), Some(PHOTOMETRIC_CFA | PHOTOMETRIC_LINEAR_RAW)) => {
return true;
}
@@ -125,7 +216,7 @@ fn camera_raw(b: &[u8]) -> bool {
_ => {}
}
}
if let Some(next) = u32_at(ifd.saturating_add(2 + 12 * usize::from(n))).filter(|&o| o != 0) {
if !legacy && let Some(next) = u32_at(ifd.saturating_add(2 + 12 * usize::from(n))).filter(|&o| o != 0) {
queue.push(next);
}
}
+20 -2
View File
@@ -45,6 +45,8 @@ pub enum Format {
Iff,
/// JPEG 2000 (JP2 container or raw J2K codestream): read-only; see [`ASYMMETRIC_EXCEPTIONS`].
Jp2,
/// Sun Raster (`.ras`): 1-, 8-, 24- and 32-bit, uncompressed or RLE.
SunRaster,
/// SGI image files (`.sgi`, `.rgb`, `.bw`): 8- and 16-bit gray, gray + alpha, RGB and RGBA.
Sgi,
/// JPEG-LS (`.jls`): lossless and near-lossless gray and RGB.
@@ -120,7 +122,7 @@ const TIFF_LAYOUTS: &[ChannelLayout] = &[L::Gray, L::GrayA, L::Rgb, L::Rgba, L::
impl Format {
/// Every format known to the crate (enabled or not).
pub const ALL: [Format; 26] = [
pub const ALL: [Format; 27] = [
Format::Png,
Format::Jpeg,
Format::Jp2,
@@ -145,6 +147,7 @@ impl Format {
Format::Cineon,
Format::Dicom,
Format::Iff,
Format::SunRaster,
Format::Sgi,
Format::JpegLs,
];
@@ -185,6 +188,7 @@ impl Format {
Format::Dicom => "DICOM",
Format::Iff => "Amiga IFF",
Format::Jp2 => "JPEG 2000",
Format::SunRaster => "Sun Raster",
Format::Sgi => "SGI",
Format::JpegLs => "JPEG-LS",
}
@@ -218,6 +222,7 @@ impl Format {
Format::Dicom => &["dcm", "dicom", "dc3"],
Format::Iff => &["iff", "lbm", "ilbm"],
Format::Jp2 => &["jp2", "j2k", "j2c", "jpf", "jpx"],
Format::SunRaster => &["ras", "sun"],
Format::Sgi => &["sgi", "rgb", "rgba", "bw"],
Format::JpegLs => &["jls"],
}
@@ -249,6 +254,7 @@ impl Format {
Format::Dicom => "application/dicom",
Format::Iff => "image/x-iff",
Format::Jp2 => "image/jp2",
Format::SunRaster => "image/x-sun-raster",
Format::Sgi => "image/x-sgi",
Format::JpegLs => "image/jls",
}
@@ -302,7 +308,8 @@ pub fn caps(format: Format) -> FormatCaps {
// Resolution travels in EXIF (WebP has no field of its own).
Format::WebP => FormatCaps { icc: true, exif: true, xmp: true, dpi: true, animation: true, ..base },
Format::Gif => FormatCaps { layouts: &[L::Rgba], animation: true, lossy: true, ..base },
Format::Bmp => base,
// Resolution in the header's pixels per metre (BITMAPINFOHEADER and later).
Format::Bmp => FormatCaps { dpi: true, ..base },
Format::Tga => FormatCaps { layouts: RGB_GRAY, ..base },
Format::Ico => FormatCaps { layouts: &[L::Rgba], ..base },
Format::Pnm => FormatCaps { depths: &[S::U8, S::U16, S::F32], layouts: ALL_LAYOUTS, ..base },
@@ -327,6 +334,13 @@ pub fn caps(format: Format) -> FormatCaps {
Format::Cineon => FormatCaps { depths: &[S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base },
Format::Dicom => FormatCaps { read: true, write: false, depths: &[S::U8, S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base },
Format::Iff => FormatCaps { depths: &[S::U8], layouts: &[L::Rgb, L::Rgba], alpha: true, ..base },
Format::SunRaster => FormatCaps {
read: cfg!(feature = "sunraster"),
write: cfg!(feature = "sunraster"),
layouts: &[L::Gray, L::Rgb, L::Rgba],
alpha: true,
..base
},
Format::Sgi => {
FormatCaps { read: cfg!(feature = "sgi"), write: cfg!(feature = "sgi"), depths: &[S::U8, S::U16], layouts: RGB_GRAY, alpha: true, ..base }
}
@@ -412,6 +426,10 @@ pub fn detect(bytes: &[u8]) -> Option<Format> {
if b.starts_with(b"/* XPM */") || b.starts_with(b"! XPM2\n") || b.starts_with(b"! XPM2\r\n") {
return Some(Format::Xpm);
}
#[cfg(feature = "sunraster")]
if crate::codecs::sunraster::looks_like_sunraster(b) {
return Some(Format::SunRaster);
}
#[cfg(feature = "sgi")]
if crate::codecs::sgi::looks_like_sgi(b) {
return Some(Format::Sgi);
+98
View File
@@ -0,0 +1,98 @@
//! A JPEG's headers and four-component samples read the way `zune-jpeg` and libjpeg-style readers
//! give them, for apps that keep a CMYK JPEG's inks: the component count and ICC profile from the
//! headers alone, and the ink amounts of a CMYK or YCCK JPEG.
//!
//! [`crate::decode`] reads the same files differently, on purpose: it keeps the samples of a
//! four-component JPEG without Adobe's marker as they are, rounds its YCCK conversion, and hands
//! arithmetic-coded, lossless and 12-bit files to its whole-coefficient decoder. The readers here
//! follow the older convention instead, so a document that kept a CMYK JPEG's inks that way reads
//! the same: every four-component file is taken to hold Adobe's inverted samples, YCCK is turned
//! into CMYK as libjpeg does (truncating), and only the 8-bit Huffman-coded frames `zune-jpeg`
//! decodes are read.
use zune_core::bytestream::ZCursor;
use zune_core::colorspace::ColorSpace;
use zune_core::options::DecoderOptions;
use zune_jpeg::JpegDecoder;
use crate::Format;
use crate::error::CodecError;
use crate::image::{ChannelLayout, Image, SampleType};
use crate::options::Limits;
/// A decoder of `bytes` whose headers have been read, with the default options (16384 pixels a
/// side at most); `None` when they aren't a JPEG or its headers can't be read.
fn headers(bytes: &[u8]) -> Option<JpegDecoder<ZCursor<&[u8]>>> {
if !bytes.starts_with(&[0xFF, 0xD8]) {
return None;
}
let mut d = JpegDecoder::new_with_options(ZCursor::new(bytes), DecoderOptions::default());
d.decode_headers().ok()?;
Some(d)
}
/// The number of colour components the frame header of JPEG `bytes` declares, whatever it is;
/// `None` when they aren't a JPEG whose headers can be read.
pub fn jpeg_components(bytes: &[u8]) -> Option<usize> {
Some(usize::from(headers(bytes)?.info()?.components))
}
/// The width and height the frame header of JPEG `bytes` declares, for any colour type; `None`
/// when they aren't a JPEG whose headers can be read (which includes sides past 16384 pixels).
pub fn jpeg_dimensions(bytes: &[u8]) -> Option<(u32, u32)> {
let (w, h) = headers(bytes)?.dimensions()?;
Some((u32::try_from(w).ok()?, u32::try_from(h).ok()?))
}
/// The ICC profile of JPEG `bytes`, put together from all its numbered `ICC_PROFILE` APP2
/// segments; `None` without one, when it is empty, and when the headers can't be read.
pub fn jpeg_icc_profile(bytes: &[u8]) -> Option<Vec<u8>> {
headers(bytes)?.icc_profile().filter(|icc| !icc.is_empty())
}
/// The ink amounts of a four-component (CMYK or YCCK) JPEG: C, M, Y and K bytes a pixel, 0 for
/// no ink, as a [`ChannelLayout::Cmyk`] image. The samples are taken to be Adobe's inverted ones
/// whether or not the file has Adobe's marker; YCCK is first turned into CMYK as libjpeg does
/// (the complement of the YCbCr colour's RGB, each value truncated).
///
/// Errors: bytes that aren't a JPEG with readable headers ([`CodecError::InvalidImage`]); one of
/// other than four components, or a frame `zune-jpeg` doesn't decode, such as an arithmetic-coded
/// or 12-bit one ([`CodecError::Unsupported`]); a size past `limits`; scan data too short for the
/// frame (under a byte for every 2048 pixels, refused before anything is allocated); and data that
/// doesn't decode,
/// including frames `zune-jpeg` panics on, which are caught.
pub fn decode_jpeg_inks(bytes: &[u8], limits: &Limits) -> Result<Image, CodecError> {
let bad = |m: &str| CodecError::InvalidImage(format!("JPEG inks: {m}"));
let mut d = headers(bytes).ok_or_else(|| bad("not a JPEG with readable headers"))?;
let space = d.input_colorspace().ok_or_else(|| bad("no colour space"))?;
let components = d.info().map_or(0, |i| i.components);
if components != 4 || !matches!(space, ColorSpace::CMYK | ColorSpace::YCCK) {
return Err(CodecError::unsupported(Format::Jpeg, format!("inks of a {components}-component {space:?} image")));
}
let (w, h) = d.dimensions().ok_or_else(|| bad("no size"))?;
let (w, h) = (u32::try_from(w).map_err(|_| bad("width"))?, u32::try_from(h).map_err(|_| bad("height"))?);
limits.check(w, h, ChannelLayout::Cmyk, SampleType::U8)?;
if crate::codecs::jpeg::scan_too_short(bytes, w, h) {
return Err(bad("the scan data is too short for the frame it declares"));
}
d.set_options(DecoderOptions::default().jpeg_set_out_colorspace(space));
// zune-jpeg 0.5 asserts on some subsampled four-component frames whose first component isn't
// the largest (a scratch row too short for its upsampler): that is an error here, not a panic.
let decoded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| d.decode()));
let mut data = decoded
.map_err(|_| CodecError::unsupported(Format::Jpeg, "inks: zune-jpeg failed on this file"))?
.map_err(|e| CodecError::unsupported(Format::Jpeg, format!("inks: {e}")))?;
for p in data.as_chunks_mut::<4>().0 {
if space == ColorSpace::YCCK {
let [y, cb, cr] = [p[0], p[1], p[2]].map(f32::from);
// libjpeg's YCCK → CMYK: the complement of the YCbCr colour's RGB.
p[0] = (434.456 - y - 1.402 * cr).clamp(0.0, 255.0) as u8;
p[1] = (119.541 - y + 0.344 * cb + 0.714 * cr).clamp(0.0, 255.0) as u8;
p[2] = (481.816 - y - 1.772 * cb).clamp(0.0, 255.0) as u8;
}
for v in p.iter_mut() {
*v = 255 - *v;
}
}
Image::from_u8(w, h, ChannelLayout::Cmyk, data)
}
+62 -5
View File
@@ -26,15 +26,19 @@ mod fidelity;
mod format;
mod image;
mod inspection;
pub mod jpeg_inks;
pub mod jpeg_sef;
mod lab;
mod options;
pub mod orientation;
pub mod png_probe;
pub mod raw_metadata;
pub mod resolution;
pub mod resolution_probe;
pub mod source_settings;
pub mod tiff_probe;
pub mod web;
pub mod webp_probe;
pub use crate::allocation::{probe_allocation_size, probe_stored_size};
pub use crate::cicp::Cicp;
pub use crate::cicp::ContentLightLevel;
@@ -64,12 +68,15 @@ pub use crate::options::{
};
pub use crate::orientation::{OrientSteps, exif_orientation, upright_exif, upright_xmp};
pub use crate::resolution::{exif_resolution, export_exif, export_xmp, photoshop_resolution, xmp_resolution};
pub use crate::resolution_probe::file_resolution;
pub use crate::source_settings::{WebpEncoding, jpeg_quality, webp_encoding};
pub use half::f16;
#[cfg(feature = "sgi")]
use crate::codecs::sgi;
use crate::codecs::{dds, dicom, dpx, exr, fits, heif, iff, jp2, jpeg, jpegls, jxl, pcx, png, pnm, tiff, via_image, webp, xpm};
#[cfg(feature = "sunraster")]
use crate::codecs::sunraster;
use crate::codecs::{bmp, dds, dicom, dpx, exr, fits, heif, iff, jp2, jpeg, jpegls, jxl, pcx, png, pnm, tiff, via_image, webp, xpm};
/// Detect the format and decode with default [`Limits`].
pub fn decode(bytes: &[u8]) -> Result<Image, CodecError> {
@@ -108,6 +115,16 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res
Format::Cineon => dpx::decode_cineon(bytes, l),
Format::Dicom => dicom::decode(bytes, l),
Format::Iff => iff::decode(bytes, l),
Format::SunRaster => {
#[cfg(feature = "sunraster")]
{
sunraster::decode(bytes, l)
}
#[cfg(not(feature = "sunraster"))]
{
Err(CodecError::unsupported(format, "Sun Raster support requires the sunraster feature"))
}
}
Format::Sgi => {
#[cfg(feature = "sgi")]
{
@@ -127,7 +144,11 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res
Format::Avif => codecs::avif_decode::decode(bytes, l, opts.keep_orientation),
#[cfg(not(all(feature = "avif-decode", not(target_arch = "wasm32"))))]
Format::Avif => via_image::decode(format, bytes, l),
Format::Gif | Format::Bmp | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr => via_image::decode(format, bytes, l),
Format::Bmp => via_image::decode(format, bytes, l).map(|mut img| {
img.meta.dpi = bmp::resolution(bytes);
img
}),
Format::Gif | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr => via_image::decode(format, bytes, l),
}?;
finish_format(format, bytes, img, opts)
}
@@ -257,6 +278,12 @@ pub fn probe_size(bytes: &[u8]) -> Result<(u32, u32), CodecError> {
Ok(if (5..=8).contains(&o) { (h, w) } else { (w, h) })
}
/// Classify classic TIFF camera RAWs using LightCraft's historical bounded heuristic.
/// This compatibility policy differs from the decoder's own RAW rejection policy.
pub fn legacy_tiff_raw_hint(bytes: &[u8]) -> bool {
tiff::legacy_camera_raw(bytes)
}
/// A JPEG's frame as its header declares it, read without decoding pixels: the stored size, the
/// layout [`decode`] returns, whether [`decode`] hands the file to its whole-coefficient
/// decoder (arithmetic-coded, lossless or 12-bit frames), whose working set is several times the
@@ -304,10 +331,26 @@ pub fn bmp_palette(bytes: &[u8]) -> Result<Option<Vec<[u8; 3]>>, CodecError> {
/// Encodes `indices` (row-major, one byte per pixel) as an 8-bit palette BMP with the colour
/// table `palette` (1 to 256 entries). Used for Indexed Color documents; BMP has no transparency.
/// The header's resolution is left unset; [`encode_bmp_indexed_with_dpi`] writes one.
pub fn encode_bmp_indexed(width: u32, height: u32, indices: &[u8], palette: &[[u8; 3]]) -> Result<Vec<u8>, CodecError> {
via_image::encode_bmp_indexed(width, height, indices, palette)
}
/// [`encode_bmp_indexed`] with the resolution `dpi` (pixels per inch, horizontal then vertical)
/// in the header's pixels per metre, as [`encode`] writes [`Metadata::dpi`] for a BMP. `None`, or
/// a resolution with no positive whole number of pixels per metre, leaves the fields unset.
pub fn encode_bmp_indexed_with_dpi(
width: u32,
height: u32,
indices: &[u8],
palette: &[[u8; 3]],
dpi: Option<(f32, f32)>,
) -> Result<Vec<u8>, CodecError> {
let mut bytes = via_image::encode_bmp_indexed(width, height, indices, palette)?;
bmp::set_resolution(&mut bytes, dpi);
Ok(bytes)
}
/// Lists the pages (image directories) of a TIFF or BigTIFF file without decoding pixels: the
/// main IFD chain and its SubIFDs, each with its size, kind (page, reduced-resolution copy or
/// mask) and storage. [`TiffInfo::default_page`] is the page [`decode`] opens.
@@ -500,6 +543,16 @@ pub fn encode(image: &Image, format: Format, opts: &EncodeOptions) -> Result<Vec
Format::Cineon => dpx::encode_cineon(image, plan),
Format::Dicom => Err(CodecError::unsupported(format, "DICOM is import-only; save as TIFF or PNG instead")),
Format::Iff => iff::encode(image, plan),
Format::SunRaster => {
#[cfg(feature = "sunraster")]
{
sunraster::encode(image, plan)
}
#[cfg(not(feature = "sunraster"))]
{
Err(CodecError::unsupported(format, "Sun Raster support requires the sunraster feature"))
}
}
Format::Sgi => {
#[cfg(feature = "sgi")]
{
@@ -515,8 +568,12 @@ pub fn encode(image: &Image, format: Format, opts: &EncodeOptions) -> Result<Vec
Format::Heif => Err(CodecError::unsupported(format, "encoding is not available in this build")),
Format::Jp2 => Err(CodecError::unsupported(format, "JPEG 2000 is import-only; save as PNG or TIFF instead")),
Format::Jxl => jxl::encode(image, plan),
Format::Gif | Format::Bmp | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr | Format::Avif => {
via_image::encode(format, image, plan, opts)
}
// The `image` crate leaves a BMP's resolution fields 0; Photoshop fills them in. They are
// part of the header, not metadata, so they are written whatever `embed_metadata` says.
Format::Bmp => via_image::encode(format, image, plan, opts).map(|mut bytes| {
bmp::set_resolution(&mut bytes, image.meta.dpi);
bytes
}),
Format::Gif | Format::Tga | Format::Ico | Format::Qoi | Format::Hdr | Format::Avif => via_image::encode(format, image, plan, opts),
}
}
+1 -1
View File
@@ -88,7 +88,7 @@ pub fn header_with_max_pixels(b: &[u8], max_pixels: u64) -> Result<(u32, u32, u1
if !after_idat {
return Err(err("truncated image data"));
}
Ok((w, h, exif.map_or(1, crate::exif_orientation)))
Ok((w, h, exif.map_or(1, craft_metadata::compat_orientation::orientation)))
}
/// Container color declarations preceding image data. Raw values leave color policy to callers.
+320
View File
@@ -0,0 +1,320 @@
//! Explicit verbatim EXIF and JPEG APP metadata helpers for callers that preserve stored orientation.
//! Normal encoder APIs still normalize metadata for upright pixels; callers opt in to these helpers.
use crate::{CodecError, Format};
/// Borrowed metadata used to construct JPEG APP segments.
#[derive(Clone, Copy, Default)]
pub struct JpegMetadata<'a> {
/// TIFF-structured EXIF bytes.
pub exif: Option<&'a [u8]>,
/// XMP packet.
pub xmp: Option<&'a str>,
/// ICC profile bytes.
pub icc: Option<&'a [u8]>,
/// Pixels per inch, if nonzero.
pub ppi: Option<u16>,
}
fn error(format: Format, message: &str) -> CodecError {
CodecError::encode(format, message)
}
/// Build JFIF, EXIF, XMP and ICC APP payloads in that order, preserving bytes and segment limits.
pub fn jpeg_segments(meta: JpegMetadata<'_>) -> Result<Vec<(u8, Vec<u8>)>, CodecError> {
let mut segments = Vec::new();
let density = meta.ppi.filter(|p| *p > 0).map_or([0, 0, 1, 0, 1], |p| {
let [hi, lo] = p.to_be_bytes();
[1, hi, lo, hi, lo]
});
segments.push((0xe0, [&b"JFIF\0\x01\x02"[..], &density, &[0, 0]].concat()));
if let Some(exif) = meta.exif {
if exif.len() > 65_527 {
return Err(error(Format::Jpeg, "EXIF larger than one APP1 segment (64 KiB)"));
}
segments.push((0xe1, [&b"Exif\0\0"[..], exif].concat()));
}
if let Some(xmp) = meta.xmp {
segments.push((0xe1, jpeg_xmp(xmp)?));
}
if let Some(icc) = meta.icc {
let chunks: Vec<_> = icc.chunks(65_519).collect();
if chunks.len() > 255 {
return Err(error(Format::Jpeg, "ICC profile too large for JPEG"));
}
for (i, chunk) in chunks.iter().enumerate() {
segments.push((0xe2, [&b"ICC_PROFILE\0"[..], &[i as u8 + 1, chunks.len() as u8], chunk].concat()));
}
}
Ok(segments)
}
/// Build a single standard XMP APP1 payload, rejecting extended-XMP-sized packets.
pub fn jpeg_xmp(xmp: &str) -> Result<Vec<u8>, CodecError> {
let payload = [&b"http://ns.adobe.com/xap/1.0/\0"[..], xmp.as_bytes()].concat();
if payload.len() > 65_533 {
return Err(error(Format::Jpeg, "XMP larger than one APP1 segment (extended XMP not supported)"));
}
Ok(payload)
}
/// Restore bytes in the first existing JPEG EXIF payload when it has room. No segment is inserted.
///
/// The EXIF APP1 is found by walking the marker segments before the first scan, so an `Exif\0\0`
/// inside another segment (a comment, say) is never taken for it, and the copy must fit inside
/// the APP1's own declared length, not merely inside the file.
pub fn replace_jpeg_exif(jpeg: &mut [u8], exif: &[u8]) {
if let Some(payload) = jpeg_exif_payload(jpeg)
&& let Some(dst) = jpeg.get_mut(payload).and_then(|p| p.get_mut(..exif.len()))
{
dst.copy_from_slice(exif);
}
}
/// The byte range of the TIFF data in the first EXIF APP1 (after `Exif\0\0`), up to the end of
/// that segment; `None` when the header segments end, are damaged, or hold no EXIF APP1.
fn jpeg_exif_payload(jpeg: &[u8]) -> Option<std::ops::Range<usize>> {
if jpeg.get(..2) != Some(&[0xff, 0xd8][..]) {
return None;
}
let mut p = 2usize;
loop {
// fill bytes (0xff) may precede a marker
while jpeg.get(p) == Some(&0xff) && jpeg.get(p + 1) == Some(&0xff) {
p += 1;
}
let (&0xff, &marker) = (jpeg.get(p)?, jpeg.get(p + 1)?) else { return None };
match marker {
// TEM and RSTn carry no length
0x01 | 0xd0..=0xd7 => p += 2,
// start of scan, end of image: no header segment follows
0xda | 0xd9 | 0x00 => return None,
_ => {
let len = usize::from(u16::from_be_bytes([*jpeg.get(p + 2)?, *jpeg.get(p + 3)?]));
let end = p.checked_add(2)?.checked_add(len).filter(|&e| len >= 2 && e <= jpeg.len())?;
if marker == 0xe1 && jpeg.get(p + 4..end)?.starts_with(b"Exif\0\0") {
return Some(p + 10..end);
}
p = end;
}
}
}
}
/// Replace the existing PNG eXIf chunk, preserving the other chunks and recomputing its CRC.
/// No chunk is inserted when one is absent.
pub fn replace_png_exif(png: &mut Vec<u8>, exif: &[u8]) -> Result<(), CodecError> {
let len = u32::try_from(exif.len()).map_err(|_| error(Format::Png, "EXIF too large"))?;
let mut p = 8usize;
while let Some(bytes) = p.checked_add(4).and_then(|end| png.get(p..end)) {
let size = u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as usize;
let Some(end) = p.checked_add(12).and_then(|v| v.checked_add(size)).filter(|end| *end <= png.len()) else { break };
if png.get(p + 4..p + 8) == Some(b"eXIf") {
let mut chunk = Vec::new();
chunk.extend_from_slice(&len.to_be_bytes());
chunk.extend_from_slice(b"eXIf");
chunk.extend_from_slice(exif);
let crc = crc32fast::hash(&chunk[4..]);
chunk.extend_from_slice(&crc.to_be_bytes());
png.splice(p..end, chunk);
break;
}
p = end;
}
Ok(())
}
/// Replace the existing WebP EXIF chunk, including padding and the RIFF size. No chunk is inserted
/// when one is absent. The input must be an encoded WebP as emitted by this crate.
pub fn replace_webp_exif(webp: &mut Vec<u8>, exif: &[u8]) -> Result<(), CodecError> {
let len = u32::try_from(exif.len()).map_err(|_| error(Format::WebP, "EXIF too large"))?;
let mut p = 12usize;
while let Some(h) = webp.get(p..p.saturating_add(8)) {
let size = u32::from_le_bytes([h[4], h[5], h[6], h[7]]) as usize;
let end = p.saturating_add(8).saturating_add(size).saturating_add(size & 1);
if &h[..4] == b"EXIF" && end <= webp.len() {
let new_len = webp
.len()
.checked_sub(end - p)
.and_then(|v| v.checked_add(8))
.and_then(|v| v.checked_add(exif.len()))
.and_then(|v| v.checked_add(exif.len() & 1))
.ok_or_else(|| error(Format::WebP, "RIFF too large"))?;
let riff_len = u32::try_from(new_len.saturating_sub(8)).map_err(|_| error(Format::WebP, "RIFF too large"))?;
let mut chunk = Vec::new();
chunk.extend_from_slice(b"EXIF");
chunk.extend_from_slice(&len.to_le_bytes());
chunk.extend_from_slice(exif);
if exif.len() & 1 == 1 {
chunk.push(0);
}
webp.splice(p..end, chunk);
webp[4..8].copy_from_slice(&riff_len.to_le_bytes());
break;
}
p = end;
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
const EXIF: &[u8] = b"II\x2a\0\x08\0\0\0\x01\0\x12\x01\x03\0\x01\0\0\0\x06\0\0\0\0\0\0\0";
#[test]
fn replacements_match_previous_code_on_mutated_containers() {
fn old_png(png: &mut Vec<u8>, exif: &[u8]) {
let mut p = 8usize;
while let Some(b) = png.get(p..p + 4) {
let size = u32::from_be_bytes([b[0], b[1], b[2], b[3]]) as usize;
let end = p.saturating_add(12).saturating_add(size);
if end > png.len() {
break;
}
if png.get(p + 4..p + 8) == Some(b"eXIf") {
let mut chunk = (exif.len() as u32).to_be_bytes().to_vec();
chunk.extend_from_slice(b"eXIf");
chunk.extend_from_slice(exif);
let mut crc = 0xffff_ffffu32;
for &v in &chunk[4..] {
crc ^= u32::from(v);
for _ in 0..8 {
crc = if crc & 1 != 0 { (crc >> 1) ^ 0xedb8_8320 } else { crc >> 1 };
}
}
chunk.extend_from_slice(&(!crc).to_be_bytes());
png.splice(p..end, chunk);
return;
}
p = end;
}
}
fn old_webp(webp: &mut Vec<u8>, exif: &[u8]) {
let mut p = 12usize;
while let Some(h) = webp.get(p..p.saturating_add(8)) {
let size = u32::from_le_bytes([h[4], h[5], h[6], h[7]]) as usize;
let end = p.saturating_add(8).saturating_add(size).saturating_add(size & 1);
if &h[..4] == b"EXIF" && end <= webp.len() {
let mut chunk = b"EXIF".to_vec();
chunk.extend_from_slice(&(exif.len() as u32).to_le_bytes());
chunk.extend_from_slice(exif);
if exif.len() & 1 == 1 {
chunk.push(0);
}
webp.splice(p..end, chunk);
let size = (webp.len().saturating_sub(8)) as u32;
webp[4..8].copy_from_slice(&size.to_le_bytes());
return;
}
p = end;
}
}
let mut image = crate::Image::from_u8(2, 2, crate::ChannelLayout::Rgb, vec![80; 12]).unwrap();
image.meta.exif = Some(EXIF.to_vec());
for format in [Format::Jpeg, Format::Png, Format::WebP] {
let encoded = crate::encode(&image, format, &crate::EncodeOptions::default()).unwrap();
for at in 0..encoded.len() {
for value in [0, 255] {
let mut original = encoded.clone();
original[at] = value;
let mut shared = original.clone();
if format == Format::Jpeg {
// the old byte search; the segment walk writes where it did or, when the
// mutation broke the segment structure or the APP1 length, not at all
let untouched = original.clone();
if let Some(pos) = original.windows(6).position(|w| w == b"Exif\0\0")
&& let Some(dst) = original.get_mut(pos + 6..pos + 6 + EXIF.len())
{
dst.copy_from_slice(EXIF);
}
replace_jpeg_exif(&mut shared, EXIF);
assert!(shared == original || shared == untouched, "Jpeg: byte {at} value {value}");
if at == 0 && value == 255 {
// SOI kept: a real replacement
assert_eq!(shared, original);
}
continue;
} else if format == Format::Png {
old_png(&mut original, EXIF);
replace_png_exif(&mut shared, EXIF).unwrap();
} else {
old_webp(&mut original, EXIF);
replace_webp_exif(&mut shared, EXIF).unwrap();
}
assert_eq!(shared, original, "{format:?}: byte {at} value {value}");
}
}
}
}
#[test]
fn explicit_replacements_preserve_raw_orientation_in_real_encodings() {
let mut image = crate::Image::from_u8(2, 2, crate::ChannelLayout::Rgb, vec![80; 12]).unwrap();
image.meta.exif = Some(EXIF.to_vec());
for format in [Format::Jpeg, Format::Png, Format::WebP] {
let mut bytes = crate::encode(&image, format, &crate::EncodeOptions::default()).unwrap();
let normal = crate::decode_as_with(format, &bytes, &crate::DecodeOptions { keep_orientation: true, ..Default::default() }).unwrap();
assert_eq!(normal.meta.exif.as_deref().map(crate::exif_orientation), Some(1), "default {format:?}");
match format {
Format::Jpeg => replace_jpeg_exif(&mut bytes, EXIF),
Format::Png => replace_png_exif(&mut bytes, EXIF).unwrap(),
Format::WebP => replace_webp_exif(&mut bytes, EXIF).unwrap(),
_ => {}
}
let decoded = crate::decode_as_with(format, &bytes, &crate::DecodeOptions { keep_orientation: true, ..Default::default() }).unwrap();
assert_eq!(decoded.meta.exif.as_deref().map(crate::exif_orientation), Some(6), "{format:?}");
}
}
#[test]
fn jpeg_exif_is_found_by_segment_not_by_signature() {
fn segment(marker: u8, payload: &[u8]) -> Vec<u8> {
[&[0xff, marker][..], &((payload.len() + 2) as u16).to_be_bytes(), payload].concat()
}
let tail = [0xff, 0xda, 0, 2, 0xff, 0xd9];
let exif_app1 = segment(0xe1, &[&b"Exif\0\0"[..], &[0; 26]].concat());
// a comment carrying the signature comes first: only the APP1 is rewritten
let comment = segment(0xfe, &[&b"Exif\0\0"[..], &[7; 40]].concat());
let mut jpeg = [&[0xff, 0xd8, 0xff][..], &comment, &exif_app1, &tail].concat();
let before = jpeg.clone();
replace_jpeg_exif(&mut jpeg, &[9; 26]);
let app1_data = 3 + comment.len() + 10;
assert_eq!(&jpeg[..app1_data], &before[..app1_data]);
assert_eq!(&jpeg[app1_data..app1_data + 26], &[9; 26]);
assert_eq!(&jpeg[app1_data + 26..], &before[app1_data + 26..]);
// no EXIF APP1, only the comment: nothing changes
let mut only_comment = [&[0xff, 0xd8][..], &comment, &tail].concat();
let before = only_comment.clone();
replace_jpeg_exif(&mut only_comment, &[9; 26]);
assert_eq!(only_comment, before);
// a replacement larger than the APP1 is refused even though the file continues
let mut small = [&[0xff, 0xd8][..], &exif_app1, &segment(0xe2, &[1; 64]), &tail].concat();
let before = small.clone();
replace_jpeg_exif(&mut small, &[9; 28]);
assert_eq!(small, before);
// markers without a length, damaged lengths, a scan before any APP1, and a non-JPEG
let mut tem = [&[0xff, 0xd8, 0xff, 0x01][..], &exif_app1, &tail].concat();
replace_jpeg_exif(&mut tem, &[9; 26]);
assert_eq!(&tem[14..40], &[9; 26]);
for broken in [vec![0xff, 0xd8, 0xff, 0xe1, 0, 1], vec![0xff, 0xd8, 0xff, 0xe1, 0xff, 0xff, 0], tail.to_vec(), vec![0xff, 0xd8, 0xff, 0xda]] {
let mut b = broken.clone();
replace_jpeg_exif(&mut b, &[9; 2]);
assert_eq!(b, broken);
}
let mut no_marker = [&[0xff, 0xd8, 0x12][..], &exif_app1].concat();
let before = no_marker.clone();
replace_jpeg_exif(&mut no_marker, &[9; 26]);
assert_eq!(no_marker, before);
}
#[test]
fn jpeg_segments_keep_order_and_enforce_limits() {
let parts = jpeg_segments(JpegMetadata { exif: Some(EXIF), xmp: Some("<x/>"), icc: Some(&[1, 2]), ppi: Some(300) }).unwrap();
assert_eq!(parts.iter().map(|v| v.0).collect::<Vec<_>>(), [0xe0, 0xe1, 0xe1, 0xe2]);
assert!(jpeg_segments(JpegMetadata { exif: Some(&vec![0; 65_528]), ..Default::default() }).is_err());
assert!(jpeg_xmp(&"x".repeat(65_533)).is_err());
let profile = vec![42; 65_520];
let chunks = jpeg_segments(JpegMetadata { icc: Some(&profile), ..Default::default() }).unwrap();
assert_eq!(chunks.len(), 3);
assert_eq!(&chunks[1].1[12..14], &[1, 2]);
assert_eq!(&chunks[2].1[12..14], &[2, 2]);
}
}
+126
View File
@@ -0,0 +1,126 @@
//! The resolution a raster file records, read from its headers without decoding any pixels.
//!
//! A layout app placing a picture wants its physical size before (or without) decoding it: a
//! 600-pixel photo saved at 600 ppi places one inch wide. [`file_resolution`] answers from the
//! container alone:
//!
//! - JPEG: the resolution [`crate::decode`] reports, from the same marker scan (stopping at the
//! start of scan): Photoshop ResolutionInfo, then XMP, then EXIF, then the JFIF density.
//! - PNG: the `pHYs` chunk when its unit is the metre (read as the decoder reads it, so a stored
//! 11811 is 300 ppi), else the `eXIf` resolution; the chunk scan stops at `IDAT` or `IEND`.
//! - WebP: the IFD0 resolution of the `EXIF` chunk.
//! - BMP: the pixels-per-metre fields of a `BITMAPINFOHEADER` or later header.
//! - Classic TIFF: the IFD0 resolution tags.
//!
//! EXIF and TIFF resolution tags are read by [`crate::resolution::exif_resolution`]. Every scan
//! is bounded by the input length and a chunk count, and anything malformed, absent,
//! non-positive or past [`MAX_PPI`] gives `None`, so the caller keeps its own default.
use crate::Limits;
use crate::codecs::{jpeg, png, webp};
use crate::resolution::exif_resolution;
/// Resolutions above this many pixels per inch are treated as corrupt.
pub const MAX_PPI: f32 = 100_000.0;
/// PNG chunks looked at before giving up.
const MAX_CHUNKS: usize = 4096;
const PNG_SIGNATURE: &[u8] = b"\x89PNG\r\n\x1a\n";
/// Pixels per inch (horizontal, vertical) that `bytes` records in its headers, or `None` when
/// the format is not one listed in the [module documentation](self) or carries no usable value.
pub fn file_resolution(bytes: &[u8]) -> Option<(f32, f32)> {
let ppi = if bytes.starts_with(b"\xFF\xD8\xFF") {
jpeg::scan_metadata(bytes).dpi
} else if bytes.starts_with(PNG_SIGNATURE) {
png_phys(bytes).or_else(|| png_exif(bytes))
} else if bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WEBP".as_slice()) {
webp_exif(bytes)
} else if bytes.starts_with(b"BM") {
bmp(bytes)
} else if is_classic_tiff(bytes) {
exif_resolution(bytes)
} else {
None
};
ppi.filter(|&p| usable(p))
}
/// Whether both axes are finite and between 1 and [`MAX_PPI`].
pub fn usable((x, y): (f32, f32)) -> bool {
let ok = |v: f32| v.is_finite() && (1.0..=MAX_PPI).contains(&v);
ok(x) && ok(y)
}
fn is_classic_tiff(bytes: &[u8]) -> bool {
bytes.starts_with(b"II*\0") || bytes.starts_with(b"MM\0*")
}
fn be_u32(b: &[u8], at: usize) -> Option<u32> {
Some(u32::from_be_bytes(b.get(at..at.checked_add(4)?)?.try_into().ok()?))
}
fn le_u32(b: &[u8], at: usize) -> Option<u32> {
Some(u32::from_le_bytes(b.get(at..at.checked_add(4)?)?.try_into().ok()?))
}
fn le_i32(b: &[u8], at: usize) -> Option<i32> {
Some(i32::from_le_bytes(b.get(at..at.checked_add(4)?)?.try_into().ok()?))
}
fn png_phys(bytes: &[u8]) -> Option<(f32, f32)> {
let mut pos = PNG_SIGNATURE.len();
for _ in 0..MAX_CHUNKS {
let len = usize::try_from(be_u32(bytes, pos)?).ok()?;
let data = pos.checked_add(8)?;
match bytes.get(pos.checked_add(4)?..data)? {
// A `pHYs` chunk holds exactly nine bytes; the whole chunk, CRC included, must be there.
b"pHYs" if len == 9 => return bytes.get(data..data.checked_add(13)?).and_then(|chunk| phys(chunk.get(..9)?)),
b"pHYs" => return None,
b"IDAT" | b"IEND" => return None,
_ => pos = data.checked_add(len)?.checked_add(4)?,
}
}
None
}
/// `pHYs`: pixels per unit in x and y, then the unit (1 = metre; 0 = aspect ratio only).
fn phys(data: &[u8]) -> Option<(f32, f32)> {
let (x, y) = (be_u32(data, 0)?, be_u32(data, 4)?);
// As the decoder does: a zero count, or a unit other than the metre, records no resolution,
// so the `eXIf` resolution can still answer.
if *data.get(8)? != 1 || x == 0 || y == 0 {
return None;
}
Some((png::dpi_of_ppm(x), png::dpi_of_ppm(y)))
}
/// The resolution of a PNG's `eXIf` chunk, read by the decoder's own header probe.
fn png_exif(bytes: &[u8]) -> Option<(f32, f32)> {
let (_, _, exif) = png::probe(bytes, &Limits::default()).ok()?;
exif_resolution(&exif?)
}
/// The resolution of a WebP's `EXIF` chunk, read by the decoder's own container probe.
fn webp_exif(bytes: &[u8]) -> Option<(f32, f32)> {
let (_, _, exif) = webp::probe(bytes, &Limits::default()).ok()?;
exif_resolution(&exif?)
}
/// BMP: the info header size at 14 (40 or more for a header with resolution fields), then the
/// signed pixels-per-metre values at 38 and 42.
fn bmp(bytes: &[u8]) -> Option<(f32, f32)> {
if le_u32(bytes, 14)? < 40 {
return None;
}
let (x, y) = (le_i32(bytes, 38)?, le_i32(bytes, 42)?);
if x <= 0 || y <= 0 {
return None;
}
Some((png::dpi_of_ppm(x.unsigned_abs()), png::dpi_of_ppm(y.unsigned_abs())))
}
#[cfg(test)]
#[path = "resolution_probe_tests.rs"]
mod tests;
@@ -0,0 +1,266 @@
#![allow(clippy::unwrap_used)]
use super::{MAX_PPI, file_resolution, usable};
use crate::{ChannelLayout, EncodeOptions, Format, Image, SampleType};
/// A little- or big-endian classic TIFF whose IFD0 holds XResolution and YResolution
/// (`x`/1, `y`/1) and, when given, ResolutionUnit.
fn tiff(le: bool, x: u32, y: u32, unit: Option<u16>) -> Vec<u8> {
let u16b = |v: u16| if le { v.to_le_bytes() } else { v.to_be_bytes() };
let u32b = |v: u32| if le { v.to_le_bytes() } else { v.to_be_bytes() };
let entries: u16 = if unit.is_some() { 3 } else { 2 };
let rationals = 8 + 2 + 12 * u32::from(entries) + 4;
let mut t = Vec::new();
t.extend_from_slice(if le { b"II*\0" } else { b"MM\0*" });
t.extend_from_slice(&u32b(8));
t.extend_from_slice(&u16b(entries));
for (tag, offset) in [(0x011Au16, rationals), (0x011B, rationals + 8)] {
t.extend_from_slice(&u16b(tag));
t.extend_from_slice(&u16b(5));
t.extend_from_slice(&u32b(1));
t.extend_from_slice(&u32b(offset));
}
if let Some(unit) = unit {
t.extend_from_slice(&u16b(0x0128));
t.extend_from_slice(&u16b(3));
t.extend_from_slice(&u32b(1));
t.extend_from_slice(&u16b(unit));
t.extend_from_slice(&[0, 0]);
}
t.extend_from_slice(&u32b(0));
for v in [x, 1, y, 1] {
t.extend_from_slice(&u32b(v));
}
t
}
fn segment(marker: u8, payload: &[u8]) -> Vec<u8> {
let mut s = vec![0xFF, marker];
s.extend_from_slice(&u16::try_from(payload.len() + 2).unwrap().to_be_bytes());
s.extend_from_slice(payload);
s
}
fn jfif(units: u8, x: u16, y: u16) -> Vec<u8> {
let mut p = b"JFIF\0\x01\x02".to_vec();
p.push(units);
p.extend_from_slice(&x.to_be_bytes());
p.extend_from_slice(&y.to_be_bytes());
p.extend_from_slice(&[0, 0]);
segment(0xE0, &p)
}
fn exif_app1(tiff: &[u8]) -> Vec<u8> {
let mut p = b"Exif\0\0".to_vec();
p.extend_from_slice(tiff);
segment(0xE1, &p)
}
fn jpeg(segments: &[Vec<u8>]) -> Vec<u8> {
let mut j = vec![0xFF, 0xD8];
for s in segments {
j.extend_from_slice(s);
}
// Start of scan, then entropy data holding bytes that look like an APP0 marker.
j.extend_from_slice(&segment(0xDA, &[1, 1, 0, 0, 63, 0]));
j.extend_from_slice(&jfif(1, 9, 9));
j.extend_from_slice(&[0xFF, 0xD9]);
j
}
fn png_chunk(kind: &[u8; 4], data: &[u8]) -> Vec<u8> {
let mut c = u32::try_from(data.len()).unwrap().to_be_bytes().to_vec();
c.extend_from_slice(kind);
c.extend_from_slice(data);
c.extend_from_slice(&[0, 0, 0, 0]); // the CRC is not checked by a header probe
c
}
fn png(chunks: &[Vec<u8>]) -> Vec<u8> {
let mut p = b"\x89PNG\r\n\x1a\n".to_vec();
p.extend_from_slice(&png_chunk(b"IHDR", &[0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0, 0, 0]));
for c in chunks {
p.extend_from_slice(c);
}
p.extend_from_slice(&png_chunk(b"IEND", &[]));
p
}
fn phys(x: u32, y: u32, unit: u8) -> Vec<u8> {
let mut d = x.to_be_bytes().to_vec();
d.extend_from_slice(&y.to_be_bytes());
d.push(unit);
png_chunk(b"pHYs", &d)
}
fn bmp(header_size: u32, x: i32, y: i32) -> Vec<u8> {
let mut b = vec![0u8; 54];
b[..2].copy_from_slice(b"BM");
b[14..18].copy_from_slice(&header_size.to_le_bytes());
b[38..42].copy_from_slice(&x.to_le_bytes());
b[42..46].copy_from_slice(&y.to_le_bytes());
b
}
fn close(got: Option<(f32, f32)>, want: (f32, f32)) {
let (x, y) = got.unwrap();
assert!((x - want.0).abs() < 0.05 && (y - want.1).abs() < 0.05, "{x} {y} vs {want:?}");
}
#[test]
fn jpeg_jfif_density_in_inches_and_centimetres() {
assert_eq!(file_resolution(&jpeg(&[jfif(1, 300, 150)])), Some((300.0, 150.0)));
close(file_resolution(&jpeg(&[jfif(2, 100, 100)])), (254.0, 254.0));
// Units 0 is an aspect ratio, not a density.
assert_eq!(file_resolution(&jpeg(&[jfif(0, 1, 1)])), None);
// Nothing in the header: the APP0 look-alike after the start of scan is not read.
assert_eq!(file_resolution(&jpeg(&[])), None);
}
#[test]
fn jpeg_exif_wins_over_jfif_and_non_exif_app1_is_skipped() {
let xmp = segment(0xE1, b"http://ns.adobe.com/xap/1.0/\0<x:xmpmeta/>");
let exif = exif_app1(&tiff(false, 600, 600, Some(2)));
assert_eq!(file_resolution(&jpeg(&[jfif(1, 72, 72), xmp.clone(), exif])), Some((600.0, 600.0)));
// An unusable EXIF value falls back to JFIF.
let zero = exif_app1(&tiff(true, 0, 0, None));
assert_eq!(file_resolution(&jpeg(&[jfif(1, 96, 96), xmp, zero])), Some((96.0, 96.0)));
}
#[test]
fn jpeg_fill_bytes_and_standalone_markers_are_stepped_over() {
let mut j = vec![0xFF, 0xD8, 0xFF, 0xFF, 0xFF, 0x01, 0xFF, 0xD0];
j.extend_from_slice(&jfif(1, 200, 200));
assert_eq!(file_resolution(&j), Some((200.0, 200.0)));
}
#[test]
fn png_phys_in_metres_only_before_image_data() {
assert_eq!(file_resolution(&png(&[phys(11_811, 11_811, 1)])), Some((300.0, 300.0)));
assert_eq!(file_resolution(&png(&[phys(2, 1, 0)])), None);
assert_eq!(file_resolution(&png(&[png_chunk(b"IDAT", &[0; 4]), phys(11_811, 11_811, 1)])), None);
assert_eq!(file_resolution(&png(&[png_chunk(b"tEXt", b"a\0b"), phys(3_780, 3_780, 1)])).map(|p| p.0.round()), Some(96.0));
assert_eq!(file_resolution(&png(&[])), None);
}
/// What the probe reads is what a full decode reports, for each format the encoder writes with
/// a resolution.
#[test]
fn agrees_with_decode_for_encoded_files() {
let mut img = Image::from_raw(2, 1, ChannelLayout::Rgb, SampleType::U8, vec![10, 20, 30, 40, 50, 60]).unwrap();
img.meta.dpi = Some((300.0, 150.0));
for format in [Format::Jpeg, Format::Png, Format::WebP, Format::Tiff] {
let bytes = crate::encode(&img, format, &EncodeOptions::default()).unwrap();
let decoded = crate::decode_as(format, &bytes).unwrap().meta.dpi;
assert_eq!(file_resolution(&bytes), decoded, "{format:?}");
assert_eq!(decoded, Some((300.0, 150.0)), "{format:?}");
}
}
#[test]
fn webp_without_exif_has_none() {
let img = Image::from_raw(1, 1, ChannelLayout::Rgb, SampleType::U8, vec![1, 2, 3]).unwrap();
let bytes = crate::encode(&img, Format::WebP, &EncodeOptions::default()).unwrap();
assert_eq!(file_resolution(&bytes), None);
// A RIFF/WEBP container the decoder's probe refuses gives nothing either.
assert_eq!(file_resolution(b"RIFF\x04\0\0\0WEBP"), None);
}
#[test]
fn bmp_pixels_per_metre() {
close(file_resolution(&bmp(40, 3_780, 7_559)), (96.0, 192.0));
assert_eq!(file_resolution(&bmp(12, 3_780, 3_780)), None);
assert_eq!(file_resolution(&bmp(40, -3_780, 3_780)), None);
assert_eq!(file_resolution(&bmp(40, 0, 0)), None);
}
#[test]
fn tiff_tags_in_both_byte_orders_and_units() {
assert_eq!(file_resolution(&tiff(true, 300, 300, None)), Some((300.0, 300.0)));
close(file_resolution(&tiff(false, 100, 50, Some(3))), (254.0, 127.0));
assert_eq!(file_resolution(&tiff(true, 300, 300, Some(1))), None);
}
#[test]
fn other_formats_and_absurd_values_give_none() {
assert_eq!(file_resolution(b"GIF89a\x01\0\x01\0"), None);
assert_eq!(file_resolution(b""), None);
assert_eq!(file_resolution(&tiff(true, 1_000_000, 1_000_000, None)), None);
assert!(usable((1.0, MAX_PPI)));
assert!(!usable((0.5, 72.0)));
assert!(!usable((f32::NAN, 72.0)));
}
#[test]
fn truncated_and_corrupted_headers_never_panic() {
let samples = [
jpeg(&[jfif(1, 300, 300), exif_app1(&tiff(false, 600, 600, Some(2)))]),
png(&[phys(11_811, 11_811, 1)]),
bmp(40, 3_780, 3_780),
tiff(false, 300, 300, Some(2)),
];
for s in &samples {
for cut in 0..=s.len() {
let _ = file_resolution(&s[..cut]);
}
for at in 0..s.len() {
for v in [0u8, 0x7F, 0xFF] {
let mut c = s.clone();
c[at] = v;
let _ = file_resolution(&c);
}
}
}
}
#[test]
fn png_phys_must_be_a_whole_nine_byte_chunk() {
// A pHYs that declares no data doesn't borrow the bytes after it.
let mut empty = png_chunk(b"pHYs", &[]);
empty.extend_from_slice(&phys(11_811, 11_811, 1)[8..]);
assert_eq!(file_resolution(&png(&[empty])), None);
// Cut off before its CRC.
let whole = png(&[phys(11_811, 11_811, 1)]);
let crc_at = whole.len() - 12 - 4;
assert_eq!(file_resolution(&whole[..crc_at]), None);
assert_eq!(file_resolution(&whole[..crc_at + 4]), Some((300.0, 300.0)));
}
/// A PNG chunk with its real CRC, for the decoder's own probe, which checks it.
fn crc_chunk(kind: &[u8; 4], data: &[u8]) -> Vec<u8> {
let mut c = png_chunk(kind, data);
let body = c.get(4..c.len() - 4).unwrap().to_vec();
let mut crc = !0u32;
for b in body {
crc ^= u32::from(b);
for _ in 0..8 {
crc = (crc >> 1) ^ (0xEDB8_8320 & (crc & 1).wrapping_neg());
}
}
let n = c.len();
c[n - 4..].copy_from_slice(&(!crc).to_be_bytes());
c
}
#[test]
fn png_zero_phys_leaves_the_exif_resolution() {
let exif = crc_chunk(b"eXIf", &tiff(false, 600, 600, Some(2)));
let zero = |x: u32, y: u32| {
let mut d = x.to_be_bytes().to_vec();
d.extend_from_slice(&y.to_be_bytes());
d.push(1);
crc_chunk(b"pHYs", &d)
};
let png = |chunks: &[Vec<u8>]| {
let mut p = b"\x89PNG\r\n\x1a\n".to_vec();
p.extend_from_slice(&crc_chunk(b"IHDR", &[0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0, 0, 0]));
chunks.iter().for_each(|c| p.extend_from_slice(c));
// The decoder's probe reads up to the image data, so there has to be some.
p.extend_from_slice(&crc_chunk(b"IDAT", &[0x78, 0x9c, 0x63, 0, 0, 0, 1, 0, 1]));
p.extend_from_slice(&crc_chunk(b"IEND", &[]));
p
};
assert_eq!(file_resolution(&png(std::slice::from_ref(&exif))), Some((600.0, 600.0)));
assert_eq!(file_resolution(&png(&[zero(0, 0), exif.clone()])), Some((600.0, 600.0)));
assert_eq!(file_resolution(&png(&[zero(11_811, 0), exif])), Some((600.0, 600.0)));
}
+59
View File
@@ -0,0 +1,59 @@
//! Header-only WebP inspection with image-chunk completeness checks.
use crate::{CodecError, Format};
/// Stored width, height and EXIF orientation without decoding pixels. The image payload must
/// fit in the input, but its compressed contents are not validated. Uses the default decode limits.
pub fn header(bytes: &[u8]) -> Result<(u32, u32, u16), CodecError> {
let err = |s: &str| CodecError::malformed(Format::WebP, s);
let mut p = 12usize;
let mut has_image = false;
let mut exif = None;
while let Some(h) = bytes.get(p..p.saturating_add(8)) {
let size = u32::from_le_bytes([h[4], h[5], h[6], h[7]]) as usize;
let end = p.saturating_add(8).saturating_add(size);
if matches!(&h[..4], b"VP8 " | b"VP8L" | b"ANMF") {
if end > bytes.len() {
return Err(err("truncated image data"));
}
has_image = true;
} else if &h[..4] == b"EXIF" && end <= bytes.len() && exif.is_none() {
exif = bytes.get(p + 8..end);
}
p = end.saturating_add(size & 1);
}
if !has_image {
return Err(err("no image data"));
}
let (w, h) = crate::probe_size(bytes)?;
let orientation = exif.map_or(1, craft_metadata::compat_orientation::orientation);
let (w, h) = if (5..=8).contains(&orientation) { (h, w) } else { (w, h) };
Ok((w, h, orientation))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn preserves_stored_dimensions_with_exif_orientation() {
let mut image = crate::Image::from_u8(2, 3, crate::ChannelLayout::Rgb, vec![42; 18]).unwrap();
image.meta.exif = Some(b"II\x2a\x00\x08\x00\x00\x00\x01\x00\x12\x01\x03\x00\x01\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00".to_vec());
let mut bytes = crate::encode(&image, Format::WebP, &crate::EncodeOptions::default()).unwrap();
// The writer correctly makes exported pixels upright; simulate a camera file carrying orientation 6.
let exif = bytes.windows(4).position(|v| v == b"EXIF").unwrap();
let tiff = exif + 8;
bytes[tiff + 18] = 6;
assert_eq!(header(&bytes).unwrap(), (2, 3, 6));
}
#[test]
fn rejects_missing_and_truncated_image_chunks() {
let mut bytes = b"RIFF\x00\x00\x00\x00WEBP".to_vec();
assert!(header(&bytes).is_err());
bytes.extend_from_slice(b"VP8 ");
bytes.extend_from_slice(&10u32.to_le_bytes());
bytes.push(0);
assert!(header(&bytes).is_err());
}
}
@@ -0,0 +1,77 @@
//! A BMP keeps its resolution in the header's `biXPelsPerMeter` and `biYPelsPerMeter`, which
//! Photoshop writes and reads (storytold/photocraft#13630). BMP exports left both 0, so a 300 ppi
//! document saved as BMP opened at 72 ppi elsewhere, and an opened BMP's resolution was ignored.
use crate::common::test_image;
use craft_image_codecs::*;
/// The DIB header's size and its pixels per metre, horizontal then vertical (file offsets 14, 38
/// and 42, little-endian).
fn header(bytes: &[u8]) -> (u32, i32, i32) {
let at = |i: usize| [bytes[i], bytes[i + 1], bytes[i + 2], bytes[i + 3]];
(u32::from_le_bytes(at(14)), i32::from_le_bytes(at(38)), i32::from_le_bytes(at(42)))
}
/// A test image of `layout` at resolution `dpi`.
fn image(layout: ChannelLayout, dpi: Option<(f32, f32)>) -> Image {
let mut img = test_image(layout, SampleType::U8);
img.meta.dpi = dpi;
img
}
#[test]
fn bmp_export_writes_the_resolution_in_pixels_per_metre_and_reads_it_back() {
// RGB is written with a BITMAPINFOHEADER, RGBA with a BITMAPV4HEADER: same field offsets.
for (layout, dib) in [(ChannelLayout::Rgb, 40), (ChannelLayout::Rgba, 108)] {
// The fields are part of the header, not metadata: embed_metadata off keeps them.
for embed_metadata in [true, false] {
let opts = EncodeOptions { embed_metadata, ..Default::default() };
let img = image(layout, Some((300.0, 150.0)));
let bytes = encode(&img, Format::Bmp, &opts).unwrap();
// 300 / 0.0254 = 11811.02 and 150 / 0.0254 = 5905.51, rounded.
assert_eq!(header(&bytes), (dib, 11811, 5906), "{layout:?} metadata {embed_metadata}");
assert_eq!(decode(&bytes).unwrap().meta.dpi, Some((300.0, 150.0)), "{layout:?} metadata {embed_metadata}");
assert!(!fidelity_warnings_with(&img, Format::Bmp, &opts).contains(&FidelityWarning::DpiDropped), "{layout:?}");
}
}
}
#[test]
fn bmp_without_a_resolution_leaves_the_fields_unset() {
let bytes = encode(&image(ChannelLayout::Rgb, None), Format::Bmp, &EncodeOptions::default()).unwrap();
assert_eq!(header(&bytes), (40, 0, 0));
assert_eq!(decode(&bytes).unwrap().meta.dpi, None, "0 means no resolution: the caller's default applies");
}
#[test]
fn bmp_opens_at_the_resolution_its_header_records() {
let bytes = encode(&image(ChannelLayout::Rgb, None), Format::Bmp, &EncodeOptions::default()).unwrap();
let with = |x: i32, y: i32| {
let mut b = bytes.clone();
b[38..42].copy_from_slice(&x.to_le_bytes());
b[42..46].copy_from_slice(&y.to_le_bytes());
decode(&b).unwrap().meta.dpi
};
// What other writers store for 96 and 72 ppi reads back whole.
assert_eq!(with(3780, 3780), Some((96.0, 96.0)));
assert_eq!(with(2835, 2835), Some((72.0, 72.0)));
// 40000 px/m is exactly 1016 ppi; a value no whole ppi rounds to stays as it is (71.98).
let (x, y) = with(40000, 2834).unwrap();
assert_eq!(x, 1016.0);
assert!((y - 71.9836).abs() < 1e-3, "{y}");
// An axis that is 0 or negative leaves the resolution unknown.
assert_eq!(with(11811, 0), None);
assert_eq!(with(-11811, 11811), None);
}
#[test]
fn indexed_bmp_writes_the_resolution_it_is_given() {
let (idx, pal) = ([0u8, 1, 1, 0], [[0, 0, 0], [255, 0, 0]]);
let bytes = encode_bmp_indexed_with_dpi(2, 2, &idx, &pal, Some((300.0, 300.0))).unwrap();
assert_eq!(header(&bytes), (40, 11811, 11811));
assert_eq!(decode(&bytes).unwrap().meta.dpi, Some((300.0, 300.0)));
let unset = encode_bmp_indexed_with_dpi(2, 2, &idx, &pal, None).unwrap();
assert_eq!(header(&unset), (40, 0, 0));
// The existing function keeps writing no resolution, so its callers' files don't change.
assert_eq!(encode_bmp_indexed(2, 2, &idx, &pal).unwrap(), unset);
}
+100
View File
@@ -799,6 +799,106 @@ fn inconsistent_count_table_errors() {
assert!(matches!(decode_deep_exr(&bytes, &Limits::none()), Err(CodecError::Malformed { .. })));
}
/// A two-part file made from the single-part deep file `deep` and a 1×1 flat scan-line part,
/// the deep part first when `deep_first`: each header gains `name` and `chunkCount`, the
/// version field the multipart bit, and each chunk its part number. The offset tables follow
/// the headers in header order, then the chunks. Returns the file, where the first table
/// starts, where the second table starts and where the last table ends.
fn two_part(deep: &[u8], deep_first: bool) -> (Vec<u8>, usize, usize, usize) {
let end = header_end_of(deep);
let first = u64::from_le_bytes(deep[end..end + 8].try_into().expect("8")) as usize;
let offsets: Vec<usize> =
(0..(first - end) / 8).map(|i| u64::from_le_bytes(deep[end + 8 * i..end + 8 * i + 8].try_into().expect("8")) as usize).collect();
let deep_chunks: Vec<&[u8]> = offsets.iter().enumerate().map(|(i, &o)| &deep[o..offsets.get(i + 1).copied().unwrap_or(deep.len())]).collect();
let mut deep_header = deep[8..end - 1].to_vec();
attr(&mut deep_header, "name", "string", b"deep");
attr(&mut deep_header, "chunkCount", "int", &(deep_chunks.len() as i32).to_le_bytes());
let mut flat_header = Vec::new();
let mut chlist = b"R\0".to_vec();
chlist.extend_from_slice(&1i32.to_le_bytes()); // half
chlist.extend_from_slice(&[0u8; 4]); // pLinear, reserved
chlist.extend_from_slice(&1i32.to_le_bytes());
chlist.extend_from_slice(&1i32.to_le_bytes());
chlist.push(0);
attr(&mut flat_header, "channels", "chlist", &chlist);
attr(&mut flat_header, "compression", "compression", &[0]);
let dw: Vec<u8> = [0i32; 4].iter().flat_map(|v| v.to_le_bytes()).collect();
attr(&mut flat_header, "dataWindow", "box2i", &dw);
attr(&mut flat_header, "displayWindow", "box2i", &dw);
attr(&mut flat_header, "lineOrder", "lineOrder", &[0]);
attr(&mut flat_header, "pixelAspectRatio", "float", &1f32.to_le_bytes());
attr(&mut flat_header, "screenWindowCenter", "v2f", &[0f32.to_le_bytes(), 0f32.to_le_bytes()].concat());
attr(&mut flat_header, "screenWindowWidth", "float", &1f32.to_le_bytes());
attr(&mut flat_header, "name", "string", b"flat");
attr(&mut flat_header, "type", "string", b"scanlineimage");
attr(&mut flat_header, "chunkCount", "int", &1i32.to_le_bytes());
// The flat part's one chunk: y, the data size, one half sample.
let mut flat_chunk = 0i32.to_le_bytes().to_vec();
flat_chunk.extend_from_slice(&2i32.to_le_bytes());
flat_chunk.extend_from_slice(&f16::from_f32(0.5).to_le_bytes());
let flat_chunks: Vec<&[u8]> = vec![&flat_chunk];
let parts: [(&[u8], &[&[u8]]); 2] = if deep_first {
[(&deep_header, &deep_chunks), (&flat_header, &flat_chunks)]
} else {
[(&flat_header, &flat_chunks), (&deep_header, &deep_chunks)]
};
let mut out = deep[..4].to_vec();
out.extend_from_slice(&(2u32 | 0x800 | 0x1000).to_le_bytes());
for (header, _) in &parts {
out.extend_from_slice(header);
out.push(0); // end of this header
}
out.push(0); // end of the headers
let first_table = out.len();
let second_table = first_table + parts[0].1.len() * 8;
let tables_end = second_table + parts[1].1.len() * 8;
let mut pos = tables_end;
for (_, chunks) in &parts {
for c in chunks.iter() {
out.extend_from_slice(&(pos as u64).to_le_bytes());
pos += 4 + c.len();
}
}
for (part, (_, chunks)) in parts.iter().enumerate() {
for c in chunks.iter() {
out.extend_from_slice(&(part as i32).to_le_bytes());
out.extend_from_slice(c);
}
}
(out, first_table, second_table, tables_end)
}
/// The deep part of a two-part file decodes as the single-part file does, first or second.
/// Its chunks start after every part's offset table: a chunk offset into the other part's
/// table, or one byte before the end of the last table, is refused as table data, and a file
/// that ends inside the last table is refused even when the deep part's own table is whole.
#[test]
fn multipart_deep_chunks_start_after_every_offset_table() {
let single = gen_deep(4, 4, &RGBAZ, 0, None, COUNTS_ONE, &sample_rgba);
let want = decode_deep_exr(&single, &Limits::none()).expect("single part");
for deep_first in [true, false] {
let (bytes, ..) = two_part(&single, deep_first);
let deep = decode_deep_exr(&bytes, &Limits::none()).expect("two parts");
assert_eq!(deep.counts, want.counts);
assert_eq!(deep.channel("Z").expect("Z").samples, want.channel("Z").expect("Z").samples);
}
let refused = |bytes: &[u8], what: &str| match decode_deep_exr(bytes, &Limits::none()) {
Err(CodecError::Malformed { message, .. }) => assert!(message.contains(what), "{message}"),
Err(e) => panic!("{e}, want an error about {what}"),
Ok(_) => panic!("decoded, want an error about {what}"),
};
let (bytes, first_table, second_table, tables_end) = two_part(&single, true);
for off in [second_table, tables_end - 1] {
let mut into_table = bytes.clone();
into_table[first_table..first_table + 8].copy_from_slice(&(off as u64).to_le_bytes());
refused(&into_table, "outside the chunk area");
}
refused(&bytes[..tables_end - 4], "past the end of the file");
}
// ---------------------------------------------------------------------------------------------
// Real reference files: the deep test images of the OpenEXR repository at commit
// `69b2604fc76e370615438bdc8d2cd95b9349c12e`, read from `corpus/exr/` at the repo root with the
@@ -62,6 +62,8 @@ fn default_build_formats_all_symmetric() {
Format::Dpx,
Format::Cineon,
Format::Iff,
#[cfg(feature = "sunraster")]
Format::SunRaster,
#[cfg(feature = "sgi")]
Format::Sgi,
Format::JpegLs,
@@ -258,6 +260,20 @@ fn detect_magic_jpeg_ls() {
assert!(decode_as(Format::JpegLs, &[0xFF, 0xD8, 0xFF, 0xF7]).is_err());
}
#[cfg(feature = "sunraster")]
#[test]
fn detect_magic_sun_raster() {
use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode};
let img = Image::from_u8(5, 3, ChannelLayout::Rgb, (0..45).collect()).unwrap();
let bytes = encode(&img, Format::SunRaster, &EncodeOptions::default()).unwrap();
assert_eq!(detect(&bytes), Some(Format::SunRaster));
assert_eq!(from_extension("scan.RAS"), Some(Format::SunRaster));
assert_eq!(decode(&bytes).unwrap().data(), img.data());
assert!(!ASYMMETRIC_EXCEPTIONS.iter().any(|(f, _)| *f == Format::SunRaster));
assert_eq!(detect(&[0x59, 0xa6, 0x6a, 0x95]), None);
assert!(decode_as(Format::SunRaster, &[0x59, 0xa6, 0x6a, 0x95]).is_err());
}
#[test]
fn detect_magic_avif() {
assert_eq!(detect(b"\0\0\0\x1cftypavif\0\0\0\0avifmif1miaf"), Some(Format::Avif));
@@ -360,3 +376,15 @@ fn sgi_requires_explicit_feature() {
header[..12].copy_from_slice(&[1, 218, 0, 1, 0, 2, 0, 1, 0, 1, 0, 1]);
assert_eq!(detect(&header), None);
}
#[cfg(not(feature = "sunraster"))]
#[test]
fn sun_raster_disabled_by_default() {
use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode};
let bytes = [0x59, 0xa6, 0x6a, 0x95, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0];
assert_ne!(detect(&bytes), Some(Format::SunRaster));
assert!(!caps(Format::SunRaster).read && !caps(Format::SunRaster).write);
assert!(decode_as(Format::SunRaster, &bytes).is_err());
let image = Image::from_u8(1, 1, ChannelLayout::Gray, vec![42]).unwrap();
assert!(encode(&image, Format::SunRaster, &EncodeOptions::default()).is_err());
}
+215
View File
@@ -0,0 +1,215 @@
//! `jpeg_inks`: four-component JPEGs read as Adobe-inverted inks, as `zune-jpeg`-based readers
//! kept them, beside `decode`'s own reading.
use craft_image_codecs::jpeg_inks::{decode_jpeg_inks, jpeg_components, jpeg_dimensions, jpeg_icc_profile};
use craft_image_codecs::*;
/// A `w` × `h` four-component JPEG of `samples` at quality 100, written by `jpeg-encoder` as
/// `ty`, with ICC profile `icc` when given.
fn four(w: u16, h: u16, samples: &[u8], ty: jpeg_encoder::ColorType, icc: Option<&[u8]>) -> Vec<u8> {
let mut out = Vec::new();
let mut enc = jpeg_encoder::Encoder::new(&mut out, 100);
enc.set_sampling_factor(jpeg_encoder::SamplingFactor::R_4_4_4);
if let Some(icc) = icc {
enc.add_icc_profile(icc).unwrap();
}
enc.encode(samples, w, h, ty).unwrap();
out
}
/// Flat 8×8 blocks of ink, so the coding keeps each block's value.
fn blocks(w: usize, h: usize) -> Vec<u8> {
(0..w * h)
.flat_map(|i| {
let b = ((i % w) / 8 + (i / w) / 8 * 7) as u8;
[b.wrapping_mul(37), b.wrapping_mul(91), 255 - b.wrapping_mul(13), b.wrapping_mul(53)]
})
.collect()
}
/// The segments of `bytes` up to the scan without Adobe's APP14 marker.
fn without_adobe(bytes: &[u8]) -> Vec<u8> {
let mut out = bytes[..2].to_vec();
let mut at = 2;
loop {
let marker = bytes[at + 1];
if marker == 0xDA {
out.extend_from_slice(&bytes[at..]);
return out;
}
let len = usize::from(u16::from_be_bytes([bytes[at + 2], bytes[at + 3]]));
if !(marker == 0xEE && &bytes[at + 4..at + 9] == b"Adobe") {
out.extend_from_slice(&bytes[at..at + 2 + len]);
}
at += 2 + len;
}
}
/// Adobe CMYK reads to the inks it was written with, as `decode` reads it; without the marker
/// the samples are still taken as inverted (where `decode` keeps them as they are), and YCCK
/// turns into the same inks closely, truncating where `decode` rounds.
#[test]
fn four_component_jpegs_read_as_inverted_inks() {
let (w, h) = (32u16, 16u16);
let inks = blocks(usize::from(w), usize::from(h));
let cmyk = four(w, h, &inks, jpeg_encoder::ColorType::Cmyk, None);
let read = decode_jpeg_inks(&cmyk, &Limits::default()).unwrap();
assert_eq!((read.dimensions(), read.layout()), ((32, 16), ChannelLayout::Cmyk));
assert!(read.data().iter().zip(&inks).all(|(a, b)| a.abs_diff(*b) <= 2), "Adobe CMYK keeps its inks");
assert_eq!(read.data(), decode(&cmyk).unwrap().data(), "as decode reads Adobe CMYK");
// No marker: inverted all the same, where decode keeps the stored samples.
let bare = without_adobe(&cmyk);
assert_ne!(bare, cmyk);
let unmarked = decode_jpeg_inks(&bare, &Limits::default()).unwrap();
assert_eq!(unmarked.data(), read.data());
let kept = decode(&bare).unwrap();
assert!(kept.data().iter().zip(read.data()).all(|(k, r)| *k == 255 - *r), "decode keeps them as stored");
// YCCK: libjpeg's conversion, truncated.
let ycck = four(w, h, &inks, jpeg_encoder::ColorType::CmykAsYcck, None);
let read = decode_jpeg_inks(&ycck, &Limits::default()).unwrap();
assert!(read.data().iter().zip(&inks).all(|(a, b)| a.abs_diff(*b) <= 3), "YCCK keeps its inks closely");
let rounded = decode(&ycck).unwrap();
let differ = read.data().iter().zip(rounded.data()).filter(|(a, b)| a != b).count();
assert!(read.data().iter().zip(rounded.data()).all(|(a, b)| a.abs_diff(*b) <= 1), "within a step of decode's rounding");
assert!(differ > 0, "truncation differs from rounding somewhere");
}
/// The headers alone give the component count of any JPEG and its ICC profile, put together from
/// its numbered segments.
#[test]
fn headers_give_components_and_icc() {
let icc: Vec<u8> = (0..200_000u32).map(|i| (i * 7 % 251) as u8).collect();
let cmyk = four(8, 8, &[9; 256], jpeg_encoder::ColorType::Cmyk, Some(&icc));
assert_eq!((jpeg_components(&cmyk), jpeg_dimensions(&cmyk)), (Some(4), Some((8, 8))));
assert_eq!(jpeg_icc_profile(&cmyk).as_deref(), Some(icc.as_slice()), "three APP2 segments");
let gray = encode(&Image::from_u8(8, 8, ChannelLayout::Gray, vec![50; 64]).unwrap(), Format::Jpeg, &EncodeOptions::default()).unwrap();
assert_eq!((jpeg_components(&gray), jpeg_icc_profile(&gray)), (Some(1), None));
for not in [&b""[..], b"\xFF\xD8", b"\x89PNG\r\n\x1a\n"] {
assert_eq!((jpeg_components(not), jpeg_icc_profile(not), jpeg_dimensions(not)), (None, None, None));
}
}
/// A frame header past 16384 pixels a side, `zune-jpeg`'s default, isn't read.
#[test]
fn sides_past_zunes_default_are_not_read() {
let mut gray = encode(&Image::from_u8(8, 8, ChannelLayout::Gray, vec![50; 64]).unwrap(), Format::Jpeg, &EncodeOptions::default()).unwrap();
let sof = gray.windows(2).position(|w| w == [0xFF, 0xC0]).unwrap();
gray[sof + 5..sof + 9].copy_from_slice(&[0x40, 0x00, 0x40, 0x00]);
assert_eq!(jpeg_dimensions(&gray), Some((16384, 16384)));
gray[sof + 7..sof + 9].copy_from_slice(&[0x40, 0x01]);
assert_eq!((jpeg_dimensions(&gray), jpeg_components(&gray)), (None, None));
}
/// What isn't a four-component image `zune-jpeg` decodes is refused: other component counts, an
/// arithmetic-coded or 12-bit frame (`decode` hands those to craft-jpeg), a size past the limits,
/// and data that isn't a JPEG.
#[test]
fn other_jpegs_are_refused() {
let gray = encode(&Image::from_u8(8, 8, ChannelLayout::Gray, vec![50; 64]).unwrap(), Format::Jpeg, &EncodeOptions::default()).unwrap();
assert!(matches!(decode_jpeg_inks(&gray, &Limits::default()), Err(CodecError::Unsupported { .. })));
let cmyk = four(16, 8, &[40; 16 * 8 * 4], jpeg_encoder::ColorType::Cmyk, None);
let sof = cmyk.windows(2).position(|w| w == [0xFF, 0xC0]).unwrap();
for (marker, precision) in [(0xC9u8, 8u8), (0xC1, 12)] {
let mut other = cmyk.clone();
other[sof + 1] = marker;
other[sof + 4] = precision;
assert!(decode_jpeg_inks(&other, &Limits::default()).is_err(), "SOF {marker:#x}, {precision} bits");
}
// A tiny file declaring a large frame is refused before zune-jpeg allocates its output.
let mut huge = cmyk.clone();
huge[sof + 5..sof + 9].copy_from_slice(&[0x20, 0x00, 0x20, 0x00]);
assert!(matches!(decode_jpeg_inks(&huge, &Limits::default()), Err(CodecError::InvalidImage(m)) if m.contains("too short")));
let tight = Limits { max_pixels: 100, ..Limits::default() };
assert!(matches!(decode_jpeg_inks(&cmyk, &tight), Err(CodecError::LimitExceeded(_))));
assert!(matches!(decode_jpeg_inks(b"not a jpeg", &Limits::default()), Err(CodecError::InvalidImage(_))));
assert!(decode_jpeg_inks(&cmyk[..12], &Limits::default()).is_err(), "headers cut short");
}
/// The ink reader four-component JPEGs were read with before (vectorcraft's `jpeg_inks`, on
/// `zune-jpeg` as this crate uses it), kept as the reference the readers here match.
fn reference_inks(bytes: &[u8]) -> Option<(usize, usize, Vec<u8>)> {
use zune_core::{bytestream::ZCursor, colorspace::ColorSpace, options::DecoderOptions};
let mut d = zune_jpeg::JpegDecoder::new_with_options(ZCursor::new(bytes), DecoderOptions::default());
d.decode_headers().ok()?;
let space = d.input_colorspace()?;
if d.info()?.components != 4 || !matches!(space, ColorSpace::CMYK | ColorSpace::YCCK) {
return None;
}
let (w, h) = d.dimensions()?;
d.set_options(DecoderOptions::default().jpeg_set_out_colorspace(space));
let mut data = d.decode().ok()?;
for p in data.as_chunks_mut::<4>().0 {
if space == ColorSpace::YCCK {
let [y, cb, cr] = [p[0], p[1], p[2]].map(f32::from);
p[0] = (434.456 - y - 1.402 * cr).clamp(0.0, 255.0) as u8;
p[1] = (119.541 - y + 0.344 * cb + 0.714 * cr).clamp(0.0, 255.0) as u8;
p[2] = (481.816 - y - 1.772 * cb).clamp(0.0, 255.0) as u8;
}
for v in p.iter_mut() {
*v = 255 - *v;
}
}
Some((w, h, data))
}
/// Over CMYK and YCCK JPEGs of many sizes, samplings, qualities and pictures, with and without
/// Adobe's marker, baseline and progressive, the inks and component counts read the same as the
/// reference reader.
#[test]
fn matches_the_reference_reader() {
use jpeg_encoder::{ColorType, Encoder, SamplingFactor};
let (mut seen, mut refused) = (0, 0);
for (i, (w, h)) in [(1u16, 1u16), (7, 3), (8, 8), (17, 9), (33, 31), (64, 5), (100, 77)].into_iter().enumerate() {
let (wu, n) = (usize::from(w), usize::from(w) * usize::from(h));
let noise: Vec<u8> = (0..n * 4).map(|k| ((k as u64).wrapping_mul(2_654_435_761).wrapping_add(i as u64 * 97) >> 7) as u8).collect();
let ramp: Vec<u8> = (0..n * 4).map(|k| (k / 4 % wu * 255 / wu.max(2)) as u8).collect();
for samples in [&noise, &ramp] {
for ty in [ColorType::Cmyk, ColorType::CmykAsYcck] {
for (q, sampling, progressive) in
[(100, SamplingFactor::R_4_4_4, false), (75, SamplingFactor::R_4_2_0, false), (40, SamplingFactor::R_4_2_2, true)]
{
let mut out = Vec::new();
let mut enc = Encoder::new(&mut out, q);
enc.set_sampling_factor(sampling);
enc.set_progressive(progressive);
enc.encode(samples, w, h, ty).unwrap();
for file in [without_adobe(&out), out] {
let read = decode_jpeg_inks(&file, &Limits::default());
assert_eq!(jpeg_components(&file), Some(4));
let (dw, dh) = jpeg_dimensions(&file).unwrap();
assert_eq!((usize::from(w), usize::from(h)), (dw as usize, dh as usize));
let Some((rw, rh, inks)) = reference_inks(&file) else {
eprintln!("refused: {w}x{h} {ty:?} q{q} progressive {progressive}");
assert!(read.is_err(), "refused by both: {w}x{h} {ty:?} q{q}");
refused += 1;
continue;
};
let read = read.unwrap();
assert_eq!(read.dimensions(), (rw as u32, rh as u32));
assert_eq!(read.data(), inks.as_slice(), "{w}x{h} {ty:?} q{q}");
seen += 1;
}
}
}
}
}
eprintln!("read {seen}, refused {refused}");
assert_eq!(seen + refused, 7 * 2 * 2 * 3 * 2);
assert!(seen > refused);
}
/// zune-jpeg 0.5 panics on some subsampled CMYK frames whose first component isn't the largest
/// (here `jpeg-encoder`'s 2×2 K with C, M and Y at 1×1, progressive): an error, not a panic.
#[test]
fn frames_zune_panics_on_are_errors() {
for (w, h) in [(1u16, 1u16), (3, 3)] {
let mut out = Vec::new();
let mut enc = jpeg_encoder::Encoder::new(&mut out, 70);
enc.set_sampling_factor(jpeg_encoder::SamplingFactor::F_2_2);
enc.set_progressive(true);
enc.encode(&vec![9; usize::from(w * h) * 4], w, h, jpeg_encoder::ColorType::Cmyk).unwrap();
assert!(std::panic::catch_unwind(|| reference_inks(&out)).is_err(), "the reference reader panics on {w}x{h}");
assert!(matches!(decode_jpeg_inks(&out, &Limits::default()), Err(CodecError::Unsupported { .. })), "{w}x{h}");
assert_eq!((jpeg_components(&out), jpeg_dimensions(&out)), (Some(4), Some((u32::from(w), u32::from(h)))));
}
}
+2
View File
@@ -1,5 +1,6 @@
//! The crate's integration tests, linked into one test binary.
mod bmp_resolution;
mod camera_raw;
mod common;
mod dds;
@@ -14,6 +15,7 @@ mod exr_stream;
mod fidelity;
mod gif_frames;
mod heif;
mod jpeg_inks;
mod jpegls_oracle;
mod limits;
mod malformed;
+35
View File
@@ -759,3 +759,38 @@ fn bounded_png_stream_preserves_late_exif_across_metadata_yields() {
assert_eq!(extras.orientation, 6);
assert_eq!(extras.meta.text.len(), 150);
}
/// zune-jpeg 0.5 asserts on some subsampled CMYK frames whose first component isn't the largest
/// (jpeg-encoder writes C, M and Y at 1×1 and K larger); craft-jpeg reads those instead, and they
/// decode to the inks they were written with. The others of these files read as before.
#[test]
fn cmyk_frames_zune_jpeg_panics_on_decode() {
use jpeg_encoder::{ColorType, Encoder, SamplingFactor as S};
let mut seen = 0;
for (s, progressive) in
[(S::F_2_2, true), (S::R_4_2_0, true), (S::F_4_1, false), (S::F_4_2, false), (S::F_2_4, false), (S::R_4_1_1, false), (S::R_4_1_0, false)]
{
for (w, h) in [(1u16, 1u16), (9, 1), (1, 9), (17, 13), (40, 40)] {
let inks: Vec<u8> = (0..usize::from(w) * usize::from(h)).flat_map(|_| [30u8, 90, 150, 210]).collect();
let mut out = Vec::new();
let mut e = Encoder::new(&mut out, 100);
e.set_sampling_factor(s);
e.set_progressive(progressive);
e.encode(&inks, w, h, ColorType::Cmyk).unwrap();
let zune = std::panic::catch_unwind(|| {
let mut d = zune_jpeg::JpegDecoder::new(zune_core::bytestream::ZCursor::new(&out[..]));
d.decode().is_ok()
});
if zune.is_ok() {
// Read or refused by zune-jpeg as before; no panic either way.
let _ = decode(&out);
continue;
}
seen += 1;
let img = decode(&out).unwrap_or_else(|e| panic!("{s:?} {w}x{h}: {e}"));
assert_eq!((img.dimensions(), img.layout()), ((u32::from(w), u32::from(h)), ChannelLayout::Cmyk));
assert!(img.data().iter().zip(&inks).all(|(a, b)| a.abs_diff(*b) <= 3), "{s:?} progressive {progressive} {w}x{h}");
}
}
assert!(seen >= 10, "zune-jpeg panicked on {seen} of them");
}
@@ -0,0 +1,65 @@
use craft_image_codecs::legacy_tiff_raw_hint;
fn tiff(tag: u16, ty: u16, value: u32) -> Vec<u8> {
let mut b = b"II\x2a\0\x08\0\0\0\x01\0".to_vec();
b.extend_from_slice(&tag.to_le_bytes());
b.extend_from_slice(&ty.to_le_bytes());
b.extend_from_slice(&1u32.to_le_bytes());
b.extend_from_slice(&value.to_le_bytes());
b.extend_from_slice(&0u32.to_le_bytes());
b
}
#[test]
fn recognizes_legacy_raw_compression_and_broad_cr_marker() {
assert!(legacy_tiff_raw_hint(&tiff(0x0103, 3, 34713)));
assert!(!legacy_tiff_raw_hint(&tiff(0x0103, 3, 1)));
assert!(legacy_tiff_raw_hint(b"II*\0\x08\0\0\0CR"));
let mut cut = tiff(0xc612, 1, 1);
cut.truncate(17);
assert!(!legacy_tiff_raw_hint(&cut));
}
/// A little-endian TIFF with one IFD of (tag, type, count, value) entries and `tail` after it.
fn ifd(entries: &[(u16, u16, u32, [u8; 4])], tail: &[u8]) -> Vec<u8> {
let mut b = b"II\x2a\0\x08\0\0\0".to_vec();
b.extend_from_slice(&(entries.len() as u16).to_le_bytes());
for (tag, ty, count, value) in entries {
b.extend_from_slice(&tag.to_le_bytes());
b.extend_from_slice(&ty.to_le_bytes());
b.extend_from_slice(&count.to_le_bytes());
b.extend_from_slice(value);
}
b.extend_from_slice(&0u32.to_le_bytes());
b.extend_from_slice(tail);
b
}
#[test]
fn sony_lossless_ycbcr_needs_a_sony_make_and_every_layout_tag() {
let short = |v: u16| [v.to_le_bytes()[0], v.to_le_bytes()[1], 0, 0];
let layout = [(0x828e, 1, 4, [0, 1, 1, 2]), (0x0103, 3, 1, short(7)), (0x0106, 3, 1, short(6)), (0x0115, 3, 1, short(3))];
// Make stored after the IFD (ASCII, 5 bytes): 8 + 2 + 5 * 12 + 4 = 74.
let with_make = |make: (u16, u16, u32, [u8; 4]), tail: &[u8]| ifd(&[&[make], &layout[..]].concat(), tail);
assert!(legacy_tiff_raw_hint(&with_make((0x010f, 2, 5, 74u32.to_le_bytes()), b"SONY\0")));
assert!(legacy_tiff_raw_hint(&with_make((0x010f, 7, 4, *b"sony"), b"")));
assert!(!legacy_tiff_raw_hint(&with_make((0x010f, 2, 6, 74u32.to_le_bytes()), b"Nikon\0")));
assert!(!legacy_tiff_raw_hint(&with_make((0x010f, 99, 4, *b"SONY"), b"")));
assert!(!legacy_tiff_raw_hint(&with_make((0x010f, 2, 5, 9999u32.to_le_bytes()), b"")));
// A Sony file without the full lossless layout is not a RAW by this rule.
let partial = ifd(&[(0x010f, 7, 4, *b"SONY"), layout[0], layout[1], layout[2]], b"");
assert!(!legacy_tiff_raw_hint(&partial));
// Repeated tags: the first value wins.
let repeated = ifd(&[&[(0x010f, 7, 4, *b"SONY")], &layout[..], &[(0x0103, 3, 1, short(1))]].concat(), b"");
assert!(legacy_tiff_raw_hint(&repeated));
}
#[test]
fn oversized_or_cut_directories_are_skipped() {
let mut huge = b"II\x2a\0\x08\0\0\0".to_vec();
huge.extend_from_slice(&5000u16.to_le_bytes());
assert!(!legacy_tiff_raw_hint(&huge));
let mut cut = ifd(&[(0x010f, 7, 4, *b"SONY"), (0x0103, 3, 1, [7, 0, 0, 0])], b"");
cut.truncate(20);
assert!(!legacy_tiff_raw_hint(&cut));
}
@@ -0,0 +1,7 @@
# Seeds for failure cases proptest has generated in the past. It is
# automatically read and these particular cases re-run before any
# novel cases are generated.
#
# It is recommended to check this file in to source control so that
# everyone who runs the test benefits from these saved cases.
cc f92b7316155ea46006de7467faa6182a90782b7befd08ed047ebeb70be15c3f7 # shrinks to which = 0, pos = 0, val = 0, cut = 0
+2
View File
@@ -914,6 +914,8 @@ fn the_magnetic_lasso_shows_the_width_circle_under_the_precise_cursor() {
#[trial]
fn width_steps_by_one_and_shift_jumps_to_the_limits() {
let mut settings = MagneticSettings::default();
settings.step_width(1, false);
assert_eq!(settings.width, 11.0, "a step adds one to the default width");
settings.width = 10.4;
settings.step_width(1, false);
assert_eq!(settings.width, 11.0, "a step adds one to the rounded width");
+7 -2
View File
@@ -380,17 +380,22 @@ fn closed_styled_previews_leave_their_composites_to_eviction() {
// With nearest interpolation every zoom and scale gives level 0 with nearest filtering.
/// A preview with the scale it is painted at.
type PaintedPreview = (Rc<PreviewTextures>, f32);
/// The texture and size a painted preview showed.
type ShownPreview = (egui::TextureId, [f32; 2]);
/// Previews painted every frame of a fixture whose host runs jobs only when a trial says (upstream's 5 × 3 white
/// image).
struct Mips {
fixture: Fixture,
jobs: Arc<ManualJobs>,
/// The previews the next frames paint, each at its scale.
painted: Rc<RefCell<Vec<(Rc<PreviewTextures>, f32)>>>,
painted: Rc<RefCell<Vec<PaintedPreview>>>,
/// The display every painted preview is converted for.
display: Rc<RefCell<Option<PreviewDisplay>>>,
/// What each painted preview showed in the last frame.
shown: Rc<RefCell<Vec<Option<(egui::TextureId, [f32; 2])>>>>,
shown: Rc<RefCell<Vec<Option<ShownPreview>>>>,
/// The previews' uploads in the last frame, read in the frame.
uploads: Rc<Cell<usize>>,
}
+22
View File
@@ -0,0 +1,22 @@
[package]
name = "craft-macho"
description = "Mach-O header pieces from hostile bytes, from Craft Anything: universal (fat and fat64) slice tables, CPU and load command names, load command minimum sizes and the thread-state program counter. Pure Rust, no I/O."
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
thiserror = { workspace = true }
# `serde`: `Serialize` on `FatSlice`, for apps that report slice tables as JSON.
serde = { workspace = true, optional = true }
[features]
serde = ["dep:serde"]
[dev-dependencies]
serde_json = { workspace = true }
+39
View File
@@ -0,0 +1,39 @@
//! Bounds-checked fixed-width reads in either byte order: out of range is `None`.
#[derive(Debug, Clone, Copy)]
pub(crate) struct Reader<'a> {
bytes: &'a [u8],
}
impl<'a> Reader<'a> {
pub(crate) fn new(bytes: &'a [u8]) -> Self {
Self { bytes }
}
/// `len` bytes starting at `at`.
pub(crate) fn slice(&self, at: usize, len: usize) -> Option<&'a [u8]> {
let end = at.checked_add(len)?;
self.bytes.get(at..end)
}
fn word<const N: usize>(&self, at: usize) -> Option<[u8; N]> {
let (word, _) = self.bytes.get(at..)?.split_first_chunk::<N>()?;
Some(*word)
}
pub(crate) fn u32_le(&self, at: usize) -> Option<u32> {
self.word(at).map(u32::from_le_bytes)
}
pub(crate) fn u64_le(&self, at: usize) -> Option<u64> {
self.word(at).map(u64::from_le_bytes)
}
pub(crate) fn u32_be(&self, at: usize) -> Option<u32> {
self.word(at).map(u32::from_be_bytes)
}
pub(crate) fn u64_be(&self, at: usize) -> Option<u64> {
self.word(at).map(u64::from_be_bytes)
}
}
+192
View File
@@ -0,0 +1,192 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
//! Mach-O header pieces from untrusted bytes: universal (fat and fat64) slice tables
//! ([`fat_slices`]), CPU and load command names, the fixed size each known load command needs
//! before its fields can be read, and the program counter in a thread command.
//!
//! From Craft Anything (`crates/formats/src/macho.rs`), where the load command walker, symbol,
//! stub and Objective-C code built on these stays. With the `serde` feature, [`FatSlice`] is
//! `Serialize`.
mod bytes;
use crate::bytes::Reader;
/// Slices allowed in a universal binary's table.
pub const MAX_FAT_SLICES: usize = 64;
/// Thread-state records walked in one thread command.
pub const MAX_THREAD_STATES: usize = 4096;
/// Why [`fat_slices`] gave up.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum MachoError {
/// The header or a slice ends before the file does.
#[error("the file ends early ({0})")]
Truncated(String),
/// Not a universal binary, or an implausible slice count.
#[error("the file is malformed ({0})")]
Malformed(String),
}
/// One architecture in a universal binary's slice table.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize))]
pub struct FatSlice {
/// [`cpu_name`] of the slice's CPU type and subtype.
pub arch: String,
pub cputype: u32,
pub cpusubtype: u32,
/// File offset of the thin image.
pub offset: u64,
pub size: u64,
/// Alignment as a power of two.
pub align: u32,
}
/// A CPU type and subtype as the name `lipo` and `-arch` use (`x86_64h`, `arm64e`…), or
/// `cpu0x…` for others.
pub fn cpu_name(cputype: u32, sub: u32) -> String {
match cputype {
7 => "i386".into(),
0x0100_0007 if sub & 0x00ff_ffff == 8 => "x86_64h".into(),
0x0100_0007 => "x86_64".into(),
12 => "arm".into(),
0x0100_000c => {
if sub & 0xff == 2 {
"arm64e".into()
} else {
"arm64".into()
}
}
0x0200_000c => "arm64_32".into(),
18 => "ppc".into(),
0x0100_0012 => "ppc64".into(),
other => format!("cpu{other:#x}"),
}
}
/// The slices of a universal (fat, or fat64 with magic `0xcafebabf`) binary. Every slice must
/// lie inside `data`. More than [`MAX_FAT_SLICES`] slices is refused: Java class files share the
/// magic and carry their version there.
pub fn fat_slices(data: &[u8]) -> Result<Vec<FatSlice>, MachoError> {
let r = Reader::new(data);
let magic = r.u32_be(0).ok_or_else(|| MachoError::Truncated("fat header".into()))?;
let wide = match magic {
0xcafe_babe => false,
0xcafe_babf => true,
_ => return Err(MachoError::Malformed("not a universal binary".into())),
};
let n = r.u32_be(4).ok_or_else(|| MachoError::Truncated("fat header".into()))? as usize;
if n == 0 || n > MAX_FAT_SLICES {
return Err(MachoError::Malformed(format!("{n} slices (Java class files share this magic)")));
}
let mut out = Vec::new();
for i in 0..n {
let s = if wide {
let o = 8 + i * 32;
(r.u32_be(o), r.u32_be(o + 4), r.u64_be(o + 8), r.u64_be(o + 16), r.u32_be(o + 24))
} else {
let o = 8 + i * 20;
(r.u32_be(o), r.u32_be(o + 4), r.u32_be(o + 8).map(u64::from), r.u32_be(o + 12).map(u64::from), r.u32_be(o + 16))
};
let (Some(cpu), Some(sub), Some(offset), Some(size), Some(align)) = s else {
return Err(MachoError::Truncated("fat arch table".into()));
};
if offset.checked_add(size).is_none_or(|end| end > data.len() as u64) {
return Err(MachoError::Truncated(format!("slice {i} runs past the end of the file")));
}
out.push(FatSlice { arch: cpu_name(cpu, sub), cputype: cpu, cpusubtype: sub, offset, size, align });
}
Ok(out)
}
/// A load command's `LC_` name, or its number in hex for ones not listed.
pub fn load_command_name(cmd: u32) -> String {
match cmd {
0x1 => "LC_SEGMENT",
0x2 => "LC_SYMTAB",
0x5 => "LC_UNIXTHREAD",
0xb => "LC_DYSYMTAB",
0xc => "LC_LOAD_DYLIB",
0xd => "LC_ID_DYLIB",
0xe => "LC_LOAD_DYLINKER",
0x19 => "LC_SEGMENT_64",
0x1b => "LC_UUID",
0x1d => "LC_CODE_SIGNATURE",
0x1e => "LC_SEGMENT_SPLIT_INFO",
0x20 => "LC_LAZY_LOAD_DYLIB",
0x21 => "LC_ENCRYPTION_INFO",
0x22 => "LC_DYLD_INFO",
0x8000_0022 => "LC_DYLD_INFO_ONLY",
0x24 => "LC_VERSION_MIN_MACOSX",
0x25 => "LC_VERSION_MIN_IPHONEOS",
0x26 => "LC_FUNCTION_STARTS",
0x29 => "LC_DATA_IN_CODE",
0x2a => "LC_SOURCE_VERSION",
0x2c => "LC_ENCRYPTION_INFO_64",
0x32 => "LC_BUILD_VERSION",
0x8000_0018 => "LC_LOAD_WEAK_DYLIB",
0x8000_001c => "LC_RPATH",
0x8000_001f => "LC_REEXPORT_DYLIB",
0x8000_0028 => "LC_MAIN",
0x8000_0033 => "LC_DYLD_EXPORTS_TRIE",
0x8000_0034 => "LC_DYLD_CHAINED_FIXUPS",
other => return format!("{other:#x}"),
}
.into()
}
/// Fixed bytes required before decoding a known load command's fields (8, the command header,
/// for unknown ones).
pub fn load_command_min_size(cmd: u32) -> usize {
match cmd {
0x1 => 56,
0x2 | 0x1b | 0x2c | 0x32 | 0x8000_0028 => 24,
0x5 => 16,
0xb => 80,
0xc | 0xd | 0x20 | 0x8000_0018 | 0x8000_001f | 0x8000_0023 => 24,
0xe | 0xf | 0x12 | 0x13 | 0x14 | 0x15 | 0x27 | 0x8000_001c => 12,
0x19 => 72,
0x1d | 0x1e | 0x24 | 0x25 | 0x26 | 0x29 | 0x2a | 0x2b | 0x2e | 0x2f | 0x30 | 0x8000_0033 | 0x8000_0034 => 16,
0x21 => 20,
0x22 | 0x8000_0022 => 48,
_ => 8,
}
}
/// An `X.Y.Z` version packed as `xxxx.yy.zz` nibbles (`LC_VERSION_MIN_*`, `LC_BUILD_VERSION`).
pub fn nibble_version(v: u32) -> String {
format!("{}.{}.{}", v >> 16, (v >> 8) & 0xff, v & 0xff)
}
/// The program counter in an `LC_UNIXTHREAD` / `LC_THREAD` body (the bytes after its 8-byte
/// command header): the first complete x86_64 (flavor 4), arm64 (flavor 6) or i386 (flavor 1)
/// thread state for `cpu`. A command can carry several flavor/count/state records; counts are
/// 32-bit words. At most [`MAX_THREAD_STATES`] records are walked.
pub fn unixthread_pc(data: &[u8], cpu: u32, is64: bool) -> Option<u64> {
let r = Reader::new(data);
let mut pos = 0usize;
for _ in 0..MAX_THREAD_STATES {
let flavor = r.u32_le(pos)?;
let count = r.u32_le(pos.checked_add(4)?)?;
let state = pos.checked_add(8)?;
let size = usize::try_from(count).ok()?.checked_mul(4)?;
let record = Reader::new(r.slice(state, size)?);
let pc = match (cpu, is64, flavor) {
(0x0100_0007, true, 4) if count >= 42 => record.u64_le(16 * 8),
(0x0100_000c, true, 6) if count >= 68 => record.u64_le(32 * 8),
(7, false, 1) if count >= 16 => record.u32_le(10 * 4).map(u64::from),
_ => None,
};
if pc.is_some() {
return pc;
}
pos = state.checked_add(size)?;
}
None
}
#[cfg(test)]
mod tests;
+213
View File
@@ -0,0 +1,213 @@
//! Slice table, name and thread-state tests. The thread-state cases are Craft Anything's
//! `macho_thread_state` integration test, called on the command body directly instead of
//! through the app's program model.
use super::*;
const X86_64: u32 = 0x0100_0007;
const ARM64: u32 = 0x0100_000c;
/// A universal binary with `slices` of (cputype, cpusubtype, body), each at a 0x100 boundary.
fn fat(wide: bool, slices: &[(u32, u32, &[u8])]) -> Vec<u8> {
let entry = if wide { 32 } else { 20 };
let mut d = vec![0u8; 0x100];
d[0..4].copy_from_slice(&(if wide { 0xcafe_babfu32 } else { 0xcafe_babe }).to_be_bytes());
d[4..8].copy_from_slice(&(slices.len() as u32).to_be_bytes());
for (i, &(cpu, sub, body)) in slices.iter().enumerate() {
let offset = d.len();
let o = 8 + i * entry;
d[o..o + 4].copy_from_slice(&cpu.to_be_bytes());
d[o + 4..o + 8].copy_from_slice(&sub.to_be_bytes());
if wide {
d[o + 8..o + 16].copy_from_slice(&(offset as u64).to_be_bytes());
d[o + 16..o + 24].copy_from_slice(&(body.len() as u64).to_be_bytes());
d[o + 24..o + 28].copy_from_slice(&8u32.to_be_bytes());
} else {
d[o + 8..o + 12].copy_from_slice(&(offset as u32).to_be_bytes());
d[o + 12..o + 16].copy_from_slice(&(body.len() as u32).to_be_bytes());
d[o + 16..o + 20].copy_from_slice(&8u32.to_be_bytes());
}
d.extend_from_slice(body);
d.resize(d.len().next_multiple_of(0x100), 0);
}
d
}
#[test]
fn fat_and_fat64_tables_list_their_slices() {
for wide in [false, true] {
let d = fat(wide, &[(X86_64, 3, &[1; 16]), (ARM64, 2, &[2; 32])]);
let s = fat_slices(&d).unwrap();
assert_eq!(s.len(), 2, "wide={wide}");
assert_eq!(s[0], FatSlice { arch: "x86_64".into(), cputype: X86_64, cpusubtype: 3, offset: 0x100, size: 16, align: 8 });
assert_eq!((s[1].arch.as_str(), s[1].offset, s[1].size), ("arm64e", 0x200, 32));
}
}
#[test]
fn damaged_tables_are_errors() {
assert_eq!(fat_slices(b"\xca\xfe"), Err(MachoError::Truncated("fat header".into())));
assert_eq!(fat_slices(&[0xfe, 0xed, 0xfa, 0xcf, 0, 0, 0, 1]), Err(MachoError::Malformed("not a universal binary".into())));
assert_eq!(fat_slices(&[0xca, 0xfe, 0xba, 0xbe]), Err(MachoError::Truncated("fat header".into())));
// A Java 21 class file: minor 0, major 65.
let java = [0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 65];
assert_eq!(fat_slices(&java).unwrap_err().to_string(), "the file is malformed (65 slices (Java class files share this magic))");
assert!(matches!(fat_slices(&[0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 0]), Err(MachoError::Malformed(_))));
let d = fat(false, &[(X86_64, 3, &[1; 16])]);
assert_eq!(fat_slices(&d[..20]), Err(MachoError::Truncated("fat arch table".into())));
assert_eq!(fat_slices(&d[..0x108]), Err(MachoError::Truncated("slice 0 runs past the end of the file".into())));
let mut wrap = fat(true, &[(X86_64, 3, &[1; 16])]);
wrap[16..24].copy_from_slice(&u64::MAX.to_be_bytes());
assert!(matches!(fat_slices(&wrap), Err(MachoError::Truncated(_))), "offset + size must not wrap");
}
#[test]
fn cpu_names() {
let names: Vec<String> = [
(7, 3),
(X86_64, 3),
(X86_64, 8),
(X86_64, 0x8000_0008),
(X86_64, 0x108),
(12, 9),
(ARM64, 0),
(ARM64, 0x8000_0002),
(0x0200_000c, 1),
(18, 0),
(0x0100_0012, 0),
(0x99, 0),
]
.into_iter()
.map(|(cpu, sub)| cpu_name(cpu, sub))
.collect();
assert_eq!(names, ["i386", "x86_64", "x86_64h", "x86_64h", "x86_64", "arm", "arm64", "arm64e", "arm64_32", "ppc", "ppc64", "cpu0x99"]);
}
#[test]
fn load_command_names_and_sizes() {
let known = [
(0x1, "LC_SEGMENT", 56),
(0x2, "LC_SYMTAB", 24),
(0x5, "LC_UNIXTHREAD", 16),
(0xb, "LC_DYSYMTAB", 80),
(0xc, "LC_LOAD_DYLIB", 24),
(0xd, "LC_ID_DYLIB", 24),
(0xe, "LC_LOAD_DYLINKER", 12),
(0x19, "LC_SEGMENT_64", 72),
(0x1b, "LC_UUID", 24),
(0x1d, "LC_CODE_SIGNATURE", 16),
(0x1e, "LC_SEGMENT_SPLIT_INFO", 16),
(0x20, "LC_LAZY_LOAD_DYLIB", 24),
(0x21, "LC_ENCRYPTION_INFO", 20),
(0x22, "LC_DYLD_INFO", 48),
(0x8000_0022, "LC_DYLD_INFO_ONLY", 48),
(0x24, "LC_VERSION_MIN_MACOSX", 16),
(0x25, "LC_VERSION_MIN_IPHONEOS", 16),
(0x26, "LC_FUNCTION_STARTS", 16),
(0x29, "LC_DATA_IN_CODE", 16),
(0x2a, "LC_SOURCE_VERSION", 16),
(0x2c, "LC_ENCRYPTION_INFO_64", 24),
(0x32, "LC_BUILD_VERSION", 24),
(0x8000_0018, "LC_LOAD_WEAK_DYLIB", 24),
(0x8000_001c, "LC_RPATH", 12),
(0x8000_001f, "LC_REEXPORT_DYLIB", 24),
(0x8000_0028, "LC_MAIN", 24),
(0x8000_0033, "LC_DYLD_EXPORTS_TRIE", 16),
(0x8000_0034, "LC_DYLD_CHAINED_FIXUPS", 16),
];
for (cmd, name, size) in known {
assert_eq!(load_command_name(cmd), name);
assert_eq!(load_command_min_size(cmd), size, "{name}");
}
assert_eq!(load_command_name(0x8000_0023), "0x80000023");
assert_eq!(load_command_min_size(0x8000_0023), 24);
assert_eq!(load_command_min_size(0x27), 12);
assert_eq!(load_command_min_size(0x2e), 16);
assert_eq!(load_command_min_size(0x77), 8);
}
#[test]
fn nibble_versions() {
assert_eq!(nibble_version(0x000e_0500), "14.5.0");
assert_eq!(nibble_version(0x000a_0f01), "10.15.1");
}
fn state(flavor: u32, count: u32, bytes: usize, pc_offset: usize, pc: u64, wide: bool) -> Vec<u8> {
let mut b = vec![0u8; 8 + bytes];
b[0..4].copy_from_slice(&flavor.to_le_bytes());
b[4..8].copy_from_slice(&count.to_le_bytes());
if wide {
b[8 + pc_offset..16 + pc_offset].copy_from_slice(&pc.to_le_bytes());
} else {
b[8 + pc_offset..12 + pc_offset].copy_from_slice(&(pc as u32).to_le_bytes());
}
b
}
fn decode(cpu: u32, wide: bool, states: &[Vec<u8>]) -> Option<u64> {
unixthread_pc(&states.concat(), cpu, wide)
}
#[test]
fn a_program_counter_requires_a_complete_declared_state() {
for count in [0, 1, 34, 41] {
assert_eq!(decode(X86_64, true, &[state(4, count, 168, 128, 0x1234, true)]), None, "count {count}");
}
assert_eq!(decode(X86_64, true, &[state(4, 42, 136, 128, 0x1234, true)]), None);
assert_eq!(decode(ARM64, true, &[state(6, 68, 264, 256, 0x1234, true)]), None);
}
#[test]
fn flavor_numbers_are_interpreted_for_the_image_cpu() {
assert_eq!(decode(ARM64, true, &[state(4, 42, 168, 128, 0x1234, true)]), None);
assert_eq!(decode(X86_64, true, &[state(6, 68, 272, 256, 0x1234, true)]), None);
assert_eq!(decode(12, false, &[state(1, 17, 68, 40, 0x1234, false)]), None);
}
#[test]
fn a_later_matching_state_is_reached_using_word_counts() {
let unknown = state(99, 2, 8, 0, 0, true);
let matching = state(4, 42, 168, 128, 0x1234, true);
assert_eq!(decode(X86_64, true, &[unknown, matching]), Some(0x1234));
}
#[test]
fn complete_supported_states_keep_their_program_counters() {
assert_eq!(decode(X86_64, true, &[state(4, 42, 168, 128, 0x1234, true)]), Some(0x1234));
assert_eq!(decode(ARM64, true, &[state(6, 68, 272, 256, 0x1234, true)]), Some(0x1234));
assert_eq!(decode(7, false, &[state(1, 16, 64, 40, 0x1234, false)]), Some(0x1234));
}
#[test]
fn endless_empty_states_stop() {
assert_eq!(unixthread_pc(&vec![0; 8 * (MAX_THREAD_STATES + 4)], X86_64, true), None);
assert_eq!(unixthread_pc(&[4, 0, 0, 0, 0xff, 0xff, 0xff, 0xff], X86_64, true), None);
}
#[cfg(feature = "serde")]
#[test]
fn slices_serialize() {
let s = fat_slices(&fat(false, &[(ARM64, 0, &[0; 4])])).unwrap();
let value = serde_json::to_value(&s[0]).unwrap();
assert_eq!(value["arch"], "arm64");
assert_eq!(value["offset"], 0x100);
}
#[test]
fn damaged_input_never_panics() {
let sample = fat(true, &[(X86_64, 3, &[1; 16]), (ARM64, 2, &[2; 32])]);
let thread = state(4, 42, 168, 128, 0x1234, true);
for source in [&sample, &thread] {
for i in 0..source.len() {
let _ = fat_slices(&source[..i]);
let _ = unixthread_pc(&source[..i], X86_64, true);
for v in [0x00, 0x7f, 0xff] {
let mut m = source.clone();
m[i] = v;
let _ = fat_slices(&m);
let _ = unixthread_pc(&m, ARM64, true);
let _ = unixthread_pc(&m, 7, false);
}
}
}
}
+17
View File
@@ -0,0 +1,17 @@
[package]
name = "craft-mcp-server"
description = "MCP server machinery any app may share: JSON-RPC 2.0 over stdio, the MCP lifecycle, tool and resource methods, tool schemas and argument checks, image results"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
serde_json = { workspace = true }
[dev-dependencies]
proptest = { workspace = true }
+64
View File
@@ -0,0 +1,64 @@
//! The content blocks a tool result is made of (MCP `CallToolResult`), and the contents a resource
//! read sends (MCP `ReadResourceResult`).
use serde_json::{Value, json};
/// What `resources/read` sends for one resource: text of a MIME type, or bytes of a MIME type,
/// which go out base64-encoded.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Contents {
Text { mime_type: String, text: String },
Blob { mime_type: String, data: Vec<u8> },
}
impl Contents {
/// `value` as JSON text (`application/json`).
pub fn json(value: &Value) -> Self {
Self::Text { mime_type: "application/json".into(), text: value.to_string() }
}
/// The MCP contents block of resource `uri`.
pub(crate) fn block(&self, uri: &str) -> Value {
match self {
Self::Text { mime_type, text } => json!({"uri": uri, "mimeType": mime_type, "text": text}),
Self::Blob { mime_type, data } => json!({"uri": uri, "mimeType": mime_type, "blob": base64(data)}),
}
}
}
/// A tool result of one text block, flagged as an error when `is_error` (the model sees the
/// message; an error is still a result, not a JSON-RPC error).
pub fn text_result(text: impl Into<String>, is_error: bool) -> Value {
let mut r = json!({"content": [{"type": "text", "text": text.into()}]});
if is_error {
r["isError"] = json!(true);
}
r
}
/// A tool result of an image (`data` of type `mime`, base64-encoded here) followed by a text
/// block saying what it shows.
pub fn image_result(data: &[u8], mime: &str, text: impl Into<String>) -> Value {
json!({"content": [
{"type": "image", "data": base64(data), "mimeType": mime},
{"type": "text", "text": text.into()},
]})
}
/// Standard base64 with padding (RFC 4648), as MCP image content carries it.
pub fn base64(data: &[u8]) -> String {
const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
let mut out = String::with_capacity(data.len().div_ceil(3) * 4);
for chunk in data.chunks(3) {
let b = [chunk.first().copied().unwrap_or(0), chunk.get(1).copied().unwrap_or(0), chunk.get(2).copied().unwrap_or(0)];
let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
for (i, shift) in [18u32, 12, 6, 0].into_iter().enumerate() {
if i <= chunk.len() {
out.push(char::from(T[((n >> shift) & 63) as usize]));
} else {
out.push('=');
}
}
}
out
}
+21
View File
@@ -0,0 +1,21 @@
//! MCP server machinery any app may share: JSON-RPC 2.0 over stdio (newline-delimited), the MCP
//! lifecycle, the tools, resources, prompts and completion methods, tool schemas with the check of
//! a call's arguments against them, and the content blocks tools reply with (text, images).
//!
//! An app implements [`Tools`]: what its tools are, how each one runs, its instructions, and the
//! resources, resource templates and prompts it serves, if any. [`Server`] speaks the protocol
//! around them and tells a client at `initialize` only what the app serves. Nothing here knows any
//! app's design, engine or renderer.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
mod content;
mod schema;
mod server;
pub use content::{Contents, base64, image_result, text_result};
pub use schema::{Effect, check_arguments, object, string, tool};
pub use server::{Incoming, Info, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, read_message, response};
#[cfg(test)]
mod tests;
+132
View File
@@ -0,0 +1,132 @@
//! Tool definitions (MCP `Tool`: name, title, description, input schema, annotations) and the
//! check of a call's arguments against the definition it was listed with.
use serde_json::{Value, json};
/// What a tool may do, for the client's approval prompts (MCP tool annotations). Clients that
/// auto-approve on these hints must not be told less than the truth.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Effect {
/// Only looks.
ReadOnly,
/// Changes the app's document in memory, undoably.
Edits,
/// Replaces the app's document in memory (unsaved changes are lost).
Replaces,
/// May read, write, overwrite or delete files on disk.
Files,
}
/// A tool's definition, its annotations from `effect`.
pub fn tool(name: &str, title: &str, desc: &str, schema: Value, effect: Effect) -> Value {
let (read_only, destructive, open_world) = match effect {
Effect::ReadOnly => (true, false, false),
Effect::Edits => (false, false, false),
Effect::Replaces => (false, true, false),
Effect::Files => (false, true, true),
};
json!({
"name": name,
"title": title,
"description": desc,
"inputSchema": schema,
"annotations": {"title": title, "readOnlyHint": read_only, "destructiveHint": destructive, "idempotentHint": read_only, "openWorldHint": open_world},
})
}
/// An object schema with these `props`, the `required` ones, and no other keys.
pub fn object(props: Value, required: &[&str]) -> Value {
let mut o = json!({"type": "object", "properties": props, "additionalProperties": false});
if !required.is_empty() {
o["required"] = json!(required);
}
o
}
/// A string property described by `desc`.
pub fn string(desc: &str) -> Value {
json!({"type": "string", "description": desc})
}
fn type_matches(t: &str, v: &Value) -> bool {
match t {
"string" => v.is_string(),
"number" => v.is_number(),
"integer" => v.is_i64() || v.is_u64(),
"boolean" => v.is_boolean(),
"array" => v.is_array(),
"object" => v.is_object(),
"null" => v.is_null(),
_ => true,
}
}
/// Check `args` against tool `name`'s input schema as `defs` list it (top level): unknown tool,
/// missing required arguments, unknown arguments, type and enum mismatches.
pub fn check_arguments(defs: &[Value], name: &str, args: &Value) -> Result<(), String> {
let def = defs.iter().find(|d| d["name"] == name).ok_or_else(|| format!("unknown tool `{name}`"))?;
let schema = &def["inputSchema"];
let Some(a) = args.as_object() else { return Err(format!("{name}: arguments must be an object")) };
for r in schema["required"].as_array().into_iter().flatten().filter_map(Value::as_str) {
if a.get(r).is_none_or(Value::is_null) {
return Err(format!("{name}: missing required argument `{r}`"));
}
}
let props = schema["properties"].as_object();
for (k, v) in a {
let Some(p) = props.and_then(|p| p.get(k)) else {
let valid: Vec<&str> = props.map(|p| p.keys().map(String::as_str).collect()).unwrap_or_default();
let valid = if valid.is_empty() { "none".to_string() } else { valid.join(", ") };
let k: String = k.chars().take(64).collect();
return Err(format!("{name}: unknown argument `{k}`; its arguments are: {valid}"));
};
if v.is_null() {
continue;
}
// An argument of several types lists them as `anyOf` (Gemini refuses a list of types).
let types: Vec<&Value> = match p["anyOf"].as_array() {
Some(alts) => alts.iter().map(|alt| &alt["type"]).collect(),
None => vec![&p["type"]],
};
let ok = types.iter().any(|t| match t {
Value::String(t) => type_matches(t, v),
Value::Array(ts) => ts.iter().filter_map(Value::as_str).any(|t| type_matches(t, v)),
_ => true,
});
if !ok {
let expected = if types.len() == 1 { types[0].clone() } else { Value::Array(types.into_iter().cloned().collect()) };
return Err(format!("{name}: argument `{k}` must be of type {expected}"));
}
if let Some(allowed) = p["enum"].as_array()
&& !allowed.contains(v)
{
return Err(format!("{name}: `{k}` must be one of {}", Value::Array(allowed.clone())));
}
}
Ok(())
}
/// Check `args` against prompt `name` as `defs` list it (MCP `Prompt`): unknown prompt, missing
/// required arguments, unknown arguments, values that are not strings.
pub(crate) fn check_prompt_arguments(defs: &[Value], name: &str, args: &Value) -> Result<(), String> {
let def = defs.iter().find(|d| d["name"] == name).ok_or_else(|| format!("unknown prompt `{name}`"))?;
let declared = def["arguments"].as_array().map(Vec::as_slice).unwrap_or_default();
let Some(a) = args.as_object() else { return Err(format!("{name}: arguments must be an object")) };
for r in declared.iter().filter(|d| d["required"] == true).filter_map(|d| d["name"].as_str()) {
if !a.contains_key(r) {
return Err(format!("{name}: missing required argument `{r}`"));
}
}
for (k, v) in a {
if !declared.iter().any(|d| d["name"] == k.as_str()) {
let valid: Vec<&str> = declared.iter().filter_map(|d| d["name"].as_str()).collect();
let valid = if valid.is_empty() { "none".to_string() } else { valid.join(", ") };
let k: String = k.chars().take(64).collect();
return Err(format!("{name}: unknown argument `{k}`; its arguments are: {valid}"));
}
if !v.is_string() {
return Err(format!("{name}: argument `{k}` must be a string"));
}
}
Ok(())
}
+330
View File
@@ -0,0 +1,330 @@
//! JSON-RPC 2.0 framing and the MCP lifecycle / tools / resources / prompts / completion methods.
use std::io::{BufRead, Write};
use serde_json::{Value, json};
use crate::content::Contents;
use crate::schema::{check_arguments, check_prompt_arguments};
/// The newest protocol revision we speak; offered when the client asks for one we don't know.
pub const PROTOCOL_VERSION: &str = "2025-06-18";
pub const SUPPORTED_VERSIONS: &[&str] = &[PROTOCOL_VERSION, "2025-03-26", "2024-11-05"];
const PARSE_ERROR: i64 = -32700;
const INVALID_REQUEST: i64 = -32600;
const METHOD_NOT_FOUND: i64 = -32601;
const INVALID_PARAMS: i64 = -32602;
const INTERNAL_ERROR: i64 = -32603;
const RESOURCE_NOT_FOUND: i64 = -32002;
/// Most messages in one JSON-RPC batch.
const MAX_BATCH: usize = 1000;
/// Most suggestions in one `completion/complete` reply (the MCP limit).
const MAX_COMPLETIONS: usize = 100;
/// What `completion/complete` asks about: a prompt by its name, or a resource by its URI or URI
/// template.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Reference<'a> {
Prompt(&'a str),
Resource(&'a str),
}
/// An app's tools, served by [`Server`].
pub trait Tools {
/// What every client is told at `initialize`.
fn instructions(&self) -> String;
/// Every tool's definition ([`crate::tool`]), as `tools/list` serves it now.
fn list(&mut self) -> Vec<Value>;
/// Run tool `name`. The server calls it right after the [`Tools::list`] its arguments were
/// checked against ([`crate::check_arguments`]), so it runs the tool as that listing defined
/// it. Failures are tool results with `isError: true` ([`crate::text_result`]); it never
/// panics on hostile arguments.
fn call(&mut self, name: &str, args: &Value) -> Value;
/// The resources `resources/list` serves (MCP `Resource`: uri, name, description, mimeType).
fn resources(&self) -> Vec<Value> {
Vec::new()
}
/// The resource templates `resources/templates/list` serves (MCP `ResourceTemplate`:
/// uriTemplate, name, description, mimeType). A URI made from one is read with
/// [`Tools::read_resource`].
fn resource_templates(&self) -> Vec<Value> {
Vec::new()
}
/// The contents of resource `uri`: `None` when there is no such resource, `Err` when it could
/// not be read now.
fn read_resource(&mut self, uri: &str) -> Option<Result<Contents, String>> {
let _ = uri;
None
}
/// The prompts `prompts/list` serves (MCP `Prompt`: name, title, description, arguments).
fn prompts(&self) -> Vec<Value> {
Vec::new()
}
/// Prompt `name` filled in with `args` (MCP `GetPromptResult`: description, messages). The
/// server calls it only for a listed prompt, with `args` checked against that listing: an
/// object of strings with every required argument and no other. `Err` is an internal error.
fn get_prompt(&mut self, name: &str, args: &Value) -> Result<Value, String> {
let _ = args;
Err(format!("prompt `{name}` cannot be filled in"))
}
/// What argument `argument` of `reference` could be, given what has been typed (`value`).
/// Asked only of an app that serves prompts or resource templates; a prompt reference is
/// always a listed prompt. The reply holds the first 100.
fn complete(&mut self, reference: Reference<'_>, argument: &str, value: &str) -> Vec<String> {
let _ = (reference, argument, value);
Vec::new()
}
}
/// Who the server says it is at `initialize`, and the longest request line it reads.
#[derive(Clone, Copy, Debug)]
pub struct Info {
pub name: &'static str,
pub title: &'static str,
pub version: &'static str,
/// Longest accepted request line, in bytes; a longer one is refused in bounded space.
pub max_line: usize,
}
/// An MCP server over the app's [`Tools`].
pub struct Server<T: Tools> {
info: Info,
tools: T,
initialized: bool,
}
fn error(id: Value, code: i64, message: impl Into<String>) -> Value {
json!({"jsonrpc": "2.0", "id": id, "error": {"code": code, "message": message.into()}})
}
/// One message read from a newline-delimited stream by [`read_message`].
#[derive(Debug, PartialEq, Eq)]
pub enum Incoming {
/// A line to handle ([`Server::handle_line`]), without its newline.
Line(String),
/// A line refused unread (too long, not UTF-8): the error reply to send for it.
Refused(String),
}
/// Read the next line of newline-delimited JSON-RPC from `input`; `None` once it closes. A line
/// longer than `max_line` is consumed in bounded space, so a client cannot exhaust memory by
/// sending an arbitrarily long request without a newline, and an unterminated last line counts.
pub fn read_message(input: &mut impl BufRead, max_line: usize) -> std::io::Result<Option<Incoming>> {
let mut line = Vec::new();
let mut oversized = false;
let mut received = false;
loop {
let chunk = input.fill_buf()?;
if chunk.is_empty() {
break;
}
let count = chunk.iter().position(|&byte| byte == b'\n').map_or(chunk.len(), |i| i + 1);
let newline = chunk[count - 1] == b'\n';
let content_len = count - usize::from(newline);
if !oversized {
if line.len().saturating_add(content_len) > max_line {
oversized = true;
line.clear();
} else {
line.extend_from_slice(&chunk[..content_len]);
}
}
input.consume(count);
received = true;
if newline {
break;
}
}
if !received {
return Ok(None);
}
if oversized {
return Ok(Some(Incoming::Refused(error(Value::Null, INVALID_REQUEST, "request too large").to_string())));
}
Ok(Some(match String::from_utf8(line) {
Ok(line) => Incoming::Line(line),
// Not UTF-8: answer with a parse error and keep serving.
Err(_) => Incoming::Refused(error(Value::Null, PARSE_ERROR, "parse error: input is not UTF-8").to_string()),
}))
}
/// The JSON-RPC response to request `id`, as [`Server::handle`] sends it.
pub fn response(id: Value, result: Value) -> Value {
json!({"jsonrpc": "2.0", "id": id, "result": result})
}
impl<T: Tools> Server<T> {
pub fn new(info: Info, tools: T) -> Self {
Self { info, tools, initialized: false }
}
pub fn tools(&mut self) -> &mut T {
&mut self.tools
}
pub fn is_initialized(&self) -> bool {
self.initialized
}
/// Serve newline-delimited JSON-RPC until `input` closes. Only protocol messages go to
/// `output`; diagnostics belong on stderr. Consume oversized lines in bounded space so a
/// client cannot exhaust memory by sending an arbitrarily long request without a newline.
pub fn serve(&mut self, mut input: impl BufRead, mut output: impl Write) -> std::io::Result<()> {
while let Some(incoming) = read_message(&mut input, self.info.max_line)? {
let reply = match incoming {
Incoming::Line(line) => self.handle_line(&line),
Incoming::Refused(reply) => Some(reply),
};
if let Some(reply) = reply {
output.write_all(reply.as_bytes())?;
output.write_all(b"\n")?;
output.flush()?;
}
}
Ok(())
}
/// One line in, at most one line out (notifications get no reply).
pub fn handle_line(&mut self, line: &str) -> Option<String> {
// Check the raw input before trimming: a huge whitespace prefix is still a huge line.
if line.len() > self.info.max_line {
return Some(error(Value::Null, INVALID_REQUEST, "request too large").to_string());
}
let line = line.trim();
if line.is_empty() {
return None;
}
let reply = match serde_json::from_str::<Value>(line) {
Ok(Value::Array(batch)) if batch.is_empty() => Some(error(Value::Null, INVALID_REQUEST, "empty batch")),
Ok(Value::Array(batch)) if batch.len() > MAX_BATCH => Some(error(Value::Null, INVALID_REQUEST, "batch too large")),
Ok(Value::Array(batch)) => {
let replies: Vec<Value> = batch.into_iter().filter_map(|m| self.handle(m)).collect();
(!replies.is_empty()).then_some(Value::Array(replies))
}
Ok(msg) => self.handle(msg),
Err(e) => Some(error(Value::Null, PARSE_ERROR, format!("parse error: {e}"))),
};
reply.map(|r| r.to_string())
}
pub fn handle(&mut self, msg: Value) -> Option<Value> {
let Value::Object(o) = &msg else { return Some(error(Value::Null, INVALID_REQUEST, "message must be an object")) };
let id = o.get("id").cloned();
let Some(method) = o.get("method").and_then(Value::as_str) else {
// A response to a request of ours (we send none) or junk.
if o.contains_key("result") || o.contains_key("error") {
return None;
}
return Some(error(id.unwrap_or(Value::Null), INVALID_REQUEST, "missing `method`"));
};
let params = o.get("params").cloned().unwrap_or(Value::Null);
let Some(id) = id else {
if method == "notifications/initialized" {
self.initialized = true;
}
return None;
};
Some(match self.request(method, &params) {
Ok(r) => response(id, r),
Err((code, m)) => error(id, code, m),
})
}
fn request(&mut self, method: &str, params: &Value) -> Result<Value, (i64, String)> {
match method {
"initialize" => {
let asked = params.get("protocolVersion").and_then(Value::as_str).unwrap_or(PROTOCOL_VERSION);
let version = if SUPPORTED_VERSIONS.contains(&asked) { asked } else { PROTOCOL_VERSION };
let Info { name, title, version: server_version, .. } = self.info;
Ok(json!({
"protocolVersion": version,
"capabilities": self.capabilities(),
"serverInfo": {"name": name, "title": title, "version": server_version},
"instructions": self.tools.instructions(),
}))
}
"ping" => Ok(json!({})),
"tools/list" => Ok(json!({"tools": self.tools.list()})),
"tools/call" => {
let name = params.get("name").and_then(Value::as_str).ok_or((INVALID_PARAMS, "missing tool `name`".to_string()))?;
let args = match params.get("arguments") {
None | Some(Value::Null) => json!({}),
Some(a) => a.clone(),
};
check_arguments(&self.tools.list(), name, &args).map_err(|e| (INVALID_PARAMS, e))?;
Ok(self.tools.call(name, &args))
}
"resources/list" => Ok(json!({"resources": self.tools.resources()})),
"resources/templates/list" => Ok(json!({"resourceTemplates": self.tools.resource_templates()})),
"prompts/list" => Ok(json!({"prompts": self.tools.prompts()})),
"resources/read" => {
let uri = params.get("uri").and_then(Value::as_str).unwrap_or("");
let contents = self
.tools
.read_resource(uri)
.ok_or_else(|| (RESOURCE_NOT_FOUND, format!("unknown resource `{uri}`")))?
.map_err(|e| (INTERNAL_ERROR, e))?;
Ok(json!({"contents": [contents.block(uri)]}))
}
"prompts/get" if !self.tools.prompts().is_empty() => {
let name = params.get("name").and_then(Value::as_str).ok_or((INVALID_PARAMS, "missing prompt `name`".to_string()))?;
let args = match params.get("arguments") {
None | Some(Value::Null) => json!({}),
Some(a) => a.clone(),
};
check_prompt_arguments(&self.tools.prompts(), name, &args).map_err(|e| (INVALID_PARAMS, e))?;
self.tools.get_prompt(name, &args).map_err(|e| (INTERNAL_ERROR, e))
}
"completion/complete" if self.completes() => {
let reference = params.get("ref").unwrap_or(&Value::Null);
let reference = match (reference.get("type").and_then(Value::as_str), reference.get("name"), reference.get("uri")) {
(Some("ref/prompt"), Some(Value::String(name)), _) => {
if !self.tools.prompts().iter().any(|p| p["name"] == name.as_str()) {
return Err((INVALID_PARAMS, format!("unknown prompt `{name}`")));
}
Reference::Prompt(name)
}
(Some("ref/resource"), _, Some(Value::String(uri))) => Reference::Resource(uri),
_ => {
return Err((
INVALID_PARAMS,
"`ref` must be a prompt's {type: ref/prompt, name} or a resource's {type: ref/resource, uri}".into(),
));
}
};
let argument = params.get("argument");
let (Some(argument), Some(value)) =
(argument.and_then(|a| a.get("name")).and_then(Value::as_str), argument.and_then(|a| a.get("value")).and_then(Value::as_str))
else {
return Err((INVALID_PARAMS, "`argument` must have a `name` and a `value`, both strings".into()));
};
let mut values = self.tools.complete(reference, argument, value);
let total = values.len();
values.truncate(MAX_COMPLETIONS);
Ok(json!({"completion": {"values": values, "total": total, "hasMore": total > MAX_COMPLETIONS}}))
}
_ => Err((METHOD_NOT_FOUND, format!("method `{method}` not found"))),
}
}
/// What the server tells a client at `initialize` that it serves: the tools always, and the
/// rest only when the app has some.
fn capabilities(&self) -> Value {
let mut capabilities = json!({"tools": {"listChanged": false}});
if !self.tools.resources().is_empty() || !self.tools.resource_templates().is_empty() {
capabilities["resources"] = json!({"listChanged": false});
}
if !self.tools.prompts().is_empty() {
capabilities["prompts"] = json!({"listChanged": false});
}
if self.completes() {
capabilities["completions"] = json!({});
}
capabilities
}
/// Whether the app has arguments to complete: those of its prompts and resource templates.
fn completes(&self) -> bool {
!self.tools.prompts().is_empty() || !self.tools.resource_templates().is_empty()
}
}
+368
View File
@@ -0,0 +1,368 @@
use serde_json::{Value, json};
use crate::{
Contents, Effect, Info, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, check_arguments, object, string, text_result, tool,
};
/// One app's tools: `echo` and `level`, nothing behind them (no engine, no design). `log` records
/// every listing and call in order.
#[derive(Default)]
struct Echo {
calls: Vec<Value>,
log: Vec<String>,
}
impl Tools for Echo {
fn instructions(&self) -> String {
"Echo test tools.".into()
}
fn list(&mut self) -> Vec<Value> {
self.log.push("list".into());
vec![
tool("echo", "Echo", "Say `text` back.", object(json!({"text": string("What to say")}), &["text"]), Effect::ReadOnly),
tool("level", "Level", "Set the level.", object(json!({"level": {"type": "string", "enum": ["low", "high"]}}), &[]), Effect::Edits),
]
}
fn call(&mut self, name: &str, args: &Value) -> Value {
self.log.push(format!("call {name}"));
self.calls.push(json!({"name": name, "args": args}));
text_result(args["text"].as_str().unwrap_or_default(), false)
}
}
fn info() -> Info {
Info { name: "echo", title: "Echo", version: "1.0.0", max_line: 1 << 16 }
}
fn serve(lines: &[Value]) -> (Server<Echo>, Vec<Value>) {
serve_with(Echo::default(), lines)
}
fn serve_with<T: Tools>(tools: T, lines: &[Value]) -> (Server<T>, Vec<Value>) {
let input: String = lines.iter().map(|l| format!("{l}\n")).collect();
let mut output = Vec::new();
let mut s = Server::new(info(), tools);
s.serve(std::io::Cursor::new(input), &mut output).unwrap();
let replies = String::from_utf8(output).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect();
(s, replies)
}
/// A registered tool is listed and called over the stdio transport, with no engine behind it.
#[test]
fn a_registered_tool_is_served_over_stdio() {
let (mut s, replies) = serve(&[
json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": "2025-03-26"}}),
json!({"jsonrpc": "2.0", "method": "notifications/initialized"}),
json!({"jsonrpc": "2.0", "id": 2, "method": "tools/list"}),
json!({"jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": {"name": "echo", "arguments": {"text": "hi"}}}),
]);
assert_eq!(replies.len(), 3, "the notification gets no reply: {replies:?}");
let init = &replies[0]["result"];
assert_eq!(init["protocolVersion"], "2025-03-26");
assert_eq!(init["serverInfo"], json!({"name": "echo", "title": "Echo", "version": "1.0.0"}));
assert_eq!(init["instructions"], "Echo test tools.");
assert!(s.is_initialized());
assert_eq!(replies[1]["result"]["tools"][0]["name"], "echo");
assert_eq!(replies[1]["result"]["tools"][0]["annotations"]["readOnlyHint"], true);
assert_eq!(replies[2], json!({"jsonrpc": "2.0", "id": 3, "result": {"content": [{"type": "text", "text": "hi"}]}}));
assert_eq!(s.tools().calls, vec![json!({"name": "echo", "args": {"text": "hi"}})]);
}
/// Arguments are checked against the tool's schema before the tool runs; unknown tools and
/// methods are JSON-RPC errors.
#[test]
fn arguments_break_the_schema_before_the_tool_runs() {
let (mut s, replies) = serve(&[
json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": "echo", "arguments": {}}}),
json!({"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "echo", "arguments": {"text": 3}}}),
json!({"jsonrpc": "2.0", "id": 3, "method": "tools/call", "params": {"name": "echo", "arguments": {"text": "a", "loud": true}}}),
json!({"jsonrpc": "2.0", "id": 4, "method": "tools/call", "params": {"name": "shout", "arguments": {}}}),
json!({"jsonrpc": "2.0", "id": 5, "method": "nope"}),
json!({"jsonrpc": "2.0", "id": 6, "method": "resources/read", "params": {"uri": "x://y"}}),
]);
let codes: Vec<&Value> = replies.iter().map(|r| &r["error"]["code"]).collect();
assert_eq!(codes, [&json!(-32602), &json!(-32602), &json!(-32602), &json!(-32602), &json!(-32601), &json!(-32002)], "{replies:?}");
assert_eq!(replies[0]["error"]["message"], "echo: missing required argument `text`");
assert_eq!(replies[2]["error"]["message"], "echo: unknown argument `loud`; its arguments are: text");
assert!(s.tools().calls.is_empty());
let defs = s.tools().list();
assert!(check_arguments(&defs, "echo", &json!({"text": "a"})).is_ok());
}
/// Hostile framing: oversized lines, bad UTF-8 and bad JSON are answered and the server goes on.
#[test]
fn hostile_lines_are_answered_and_serving_goes_on() {
let mut input = vec![b'x'; (1 << 16) + 10];
input.push(b'\n');
input.extend_from_slice(b"\xff\xfe\n{not json\n[]\n");
input.extend_from_slice(json!({"jsonrpc": "2.0", "id": 9, "method": "ping"}).to_string().as_bytes());
let mut output = Vec::new();
Server::new(info(), Echo::default()).serve(std::io::Cursor::new(input), &mut output).unwrap();
let replies: Vec<Value> = String::from_utf8(output).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect();
let codes: Vec<&Value> = replies.iter().map(|r| r.get("error").map_or(&r["result"], |e| &e["code"])).collect();
assert_eq!(codes, [&json!(-32600), &json!(-32700), &json!(-32700), &json!(-32600), &json!({})], "{replies:?}");
}
#[test]
fn base64_matches_rfc4648() {
for (i, o) in [("", ""), ("f", "Zg=="), ("fo", "Zm8="), ("foo", "Zm9v"), ("foob", "Zm9vYg=="), ("fooba", "Zm9vYmE="), ("foobar", "Zm9vYmFy")] {
assert_eq!(crate::base64(i.as_bytes()), o);
}
let r = crate::image_result(b"foo", "image/png", "info");
assert_eq!(r, json!({"content": [{"type": "image", "data": "Zm9v", "mimeType": "image/png"}, {"type": "text", "text": "info"}]}));
}
fn ping(id: usize) -> Value {
json!({"jsonrpc": "2.0", "id": id, "method": "ping"})
}
fn notification() -> Value {
json!({"jsonrpc": "2.0", "method": "notifications/progress"})
}
fn reply<T: Tools>(s: &mut Server<T>, msg: &Value) -> Option<Value> {
s.handle_line(&msg.to_string()).map(|r| serde_json::from_str(&r).unwrap())
}
/// A batch answers each request in it and none of its notifications; a batch of notifications
/// only gets no reply at all; a batch holds at most 1000 messages.
#[test]
fn batches_answer_their_requests_up_to_the_limit() {
let mut s = Server::new(info(), Echo::default());
let mixed =
json!([ping(1), notification(), {"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "echo", "arguments": {"text": "b"}}}]);
let replies = reply(&mut s, &mixed).unwrap();
assert_eq!(
replies,
json!([{"jsonrpc": "2.0", "id": 1, "result": {}}, {"jsonrpc": "2.0", "id": 2, "result": {"content": [{"type": "text", "text": "b"}]}}])
);
assert_eq!(reply(&mut s, &json!([notification(), notification()])), None);
let full: Vec<Value> = (0..1000).map(ping).collect();
assert_eq!(reply(&mut s, &Value::Array(full)).unwrap().as_array().map(Vec::len), Some(1000));
let over: Vec<Value> = (0..1001).map(ping).collect();
let r = reply(&mut s, &Value::Array(over)).unwrap();
assert_eq!(r, json!({"jsonrpc": "2.0", "id": null, "error": {"code": -32600, "message": "batch too large"}}));
}
/// We send no requests, so a response is never answered, alone or in a batch; a message with
/// neither a method nor a result, or one that is not an object, is refused; a blank line is skipped.
#[test]
fn a_response_we_never_asked_for_is_ignored() {
let mut s = Server::new(info(), Echo::default());
assert_eq!(reply(&mut s, &json!({"jsonrpc": "2.0", "id": 1, "result": {}})), None);
assert_eq!(reply(&mut s, &json!({"jsonrpc": "2.0", "id": 2, "error": {"code": -1, "message": "no"}})), None);
assert_eq!(
reply(&mut s, &json!([{"jsonrpc": "2.0", "id": 3, "result": {}}, ping(4)])).unwrap(),
json!([{"jsonrpc": "2.0", "id": 4, "result": {}}])
);
let r = reply(&mut s, &json!({"jsonrpc": "2.0", "id": 5})).unwrap();
assert_eq!(r, json!({"jsonrpc": "2.0", "id": 5, "error": {"code": -32600, "message": "missing `method`"}}));
let r = reply(&mut s, &json!(42)).unwrap();
assert_eq!(r, json!({"jsonrpc": "2.0", "id": null, "error": {"code": -32600, "message": "message must be an object"}}));
assert_eq!(s.handle_line(" "), None);
assert!(s.tools().log.is_empty());
}
/// A version we speak is echoed; one we don't, or none, gets ours.
#[test]
fn an_unknown_protocol_version_gets_ours() {
let mut s = Server::new(info(), Echo::default());
let version = |s: &mut Server<Echo>, params: Value| {
reply(s, &json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": params})).unwrap()["result"]["protocolVersion"].clone()
};
for v in SUPPORTED_VERSIONS {
assert_eq!(version(&mut s, json!({"protocolVersion": v})), json!(v));
}
assert_eq!(version(&mut s, json!({"protocolVersion": "1999-01-01"})), json!(PROTOCOL_VERSION));
assert_eq!(version(&mut s, json!({"protocolVersion": 7})), json!(PROTOCOL_VERSION));
assert_eq!(version(&mut s, json!({})), json!(PROTOCOL_VERSION));
assert!(!s.is_initialized(), "initialized only by the client's notification");
}
/// `arguments: null` is no arguments; a value outside a property's `enum` is refused before the
/// tool runs.
#[test]
fn null_arguments_are_none_and_enums_are_checked() {
let mut s = Server::new(info(), Echo::default());
let call = |s: &mut Server<Echo>, id: u64, args: Value| {
reply(s, &json!({"jsonrpc": "2.0", "id": id, "method": "tools/call", "params": {"name": "level", "arguments": args}})).unwrap()
};
assert!(call(&mut s, 1, Value::Null).get("result").is_some());
let none = reply(&mut s, &json!({"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "level"}})).unwrap();
assert!(none.get("result").is_some(), "{none}");
assert!(call(&mut s, 3, json!({"level": "high"})).get("result").is_some());
let r = call(&mut s, 4, json!({"level": "loud"}));
assert_eq!(r["error"], json!({"code": -32602, "message": "level: `level` must be one of [\"low\",\"high\"]"}));
let r = call(&mut s, 5, json!([]));
assert_eq!(r["error"], json!({"code": -32602, "message": "level: arguments must be an object"}));
let args: Vec<&Value> = s.tools().calls.iter().map(|c| &c["args"]).collect();
assert_eq!(args, [&json!({}), &json!({}), &json!({"level": "high"})]);
}
/// An app that serves no resources, templates or prompts lists none of each, says it serves only
/// tools, and has no prompt to fill in or argument to complete.
#[test]
fn the_default_lists_are_empty() {
let mut s = Server::new(info(), Echo::default());
let result = |s: &mut Server<Echo>, method: &str| reply(s, &json!({"jsonrpc": "2.0", "id": 1, "method": method})).unwrap()["result"].clone();
assert_eq!(result(&mut s, "resources/list"), json!({"resources": []}));
assert_eq!(result(&mut s, "resources/templates/list"), json!({"resourceTemplates": []}));
assert_eq!(result(&mut s, "prompts/list"), json!({"prompts": []}));
assert_eq!(result(&mut s, "initialize")["capabilities"], json!({"tools": {"listChanged": false}}));
for method in ["prompts/get", "completion/complete"] {
let r = reply(&mut s, &json!({"jsonrpc": "2.0", "id": 2, "method": method, "params": {"name": "describe"}})).unwrap();
assert_eq!(r["error"]["code"], -32601, "{method}: {r}");
}
}
/// An app that serves more than tools: a prompt with a required and an optional argument, a
/// resource template, a Markdown resource and a PNG, and suggestions for any argument. `asked`
/// records every prompt filled in and every completion asked for.
#[derive(Default)]
struct Library {
asked: Vec<String>,
}
impl Tools for Library {
fn instructions(&self) -> String {
"Library test.".into()
}
fn list(&mut self) -> Vec<Value> {
Vec::new()
}
fn call(&mut self, name: &str, _: &Value) -> Value {
text_result(format!("no tool `{name}`"), true)
}
fn resources(&self) -> Vec<Value> {
vec![
json!({"uri": "lib://readme", "name": "Readme", "mimeType": "text/markdown"}),
json!({"uri": "lib://logo", "name": "Logo", "mimeType": "image/png"}),
]
}
fn resource_templates(&self) -> Vec<Value> {
vec![json!({"uriTemplate": "lib://parts/{name}", "name": "Part", "mimeType": "text/markdown"})]
}
fn read_resource(&mut self, uri: &str) -> Option<Result<Contents, String>> {
let markdown = |text: String| Contents::Text { mime_type: "text/markdown".into(), text };
Some(Ok(match uri {
"lib://readme" => markdown("# Parts\n\nA \"quoted\" line.".into()),
"lib://logo" => Contents::Blob { mime_type: "image/png".into(), data: b"foo".to_vec() },
"lib://parts/broken" => return Some(Err("the part is being rebuilt".into())),
_ => markdown(format!("# {}", uri.strip_prefix("lib://parts/")?)),
}))
}
fn prompts(&self) -> Vec<Value> {
vec![json!({"name": "describe", "title": "Describe a part", "arguments": [{"name": "part", "required": true}, {"name": "tone"}]})]
}
fn get_prompt(&mut self, name: &str, args: &Value) -> Result<Value, String> {
self.asked.push(format!("{name} {args}"));
let text = format!("Describe {} in a {} tone.", args["part"].as_str().unwrap_or_default(), args["tone"].as_str().unwrap_or("plain"));
Ok(json!({"messages": [{"role": "user", "content": {"type": "text", "text": text}}]}))
}
fn complete(&mut self, reference: Reference<'_>, argument: &str, value: &str) -> Vec<String> {
self.asked.push(format!("{reference:?} {argument}={value}"));
(0..150).map(|i| format!("{value}{i}")).collect()
}
}
/// The shared crate serves an app's prompt, resource template and Markdown and binary resources,
/// and tells the client at `initialize` that it does (#13336).
#[test]
fn an_app_serves_prompts_templates_and_markdown_or_binary_resources() {
let request = |id: u64, method: &str, params: Value| json!({"jsonrpc": "2.0", "id": id, "method": method, "params": params});
let (mut s, replies) = serve_with(
Library::default(),
&[
request(1, "initialize", json!({"protocolVersion": PROTOCOL_VERSION})),
request(2, "prompts/list", json!({})),
request(3, "prompts/get", json!({"name": "describe", "arguments": {"part": "bracket"}})),
request(4, "resources/templates/list", json!({})),
request(5, "resources/read", json!({"uri": "lib://readme"})),
request(6, "resources/read", json!({"uri": "lib://logo"})),
request(7, "resources/read", json!({"uri": "lib://parts/bracket"})),
request(8, "resources/read", json!({"uri": "lib://parts/broken"})),
request(9, "resources/read", json!({"uri": "lib://nope"})),
],
);
let result = |i: usize| &replies[i]["result"];
assert_eq!(
result(0)["capabilities"],
json!({"tools": {"listChanged": false}, "resources": {"listChanged": false}, "prompts": {"listChanged": false}, "completions": {}})
);
assert_eq!(result(1)["prompts"][0]["name"], "describe");
assert_eq!(result(2), &json!({"messages": [{"role": "user", "content": {"type": "text", "text": "Describe bracket in a plain tone."}}]}));
assert_eq!(result(3), &json!({"resourceTemplates": [{"uriTemplate": "lib://parts/{name}", "name": "Part", "mimeType": "text/markdown"}]}));
// The Markdown goes out as it is, not as a JSON string of it.
assert_eq!(result(4), &json!({"contents": [{"uri": "lib://readme", "mimeType": "text/markdown", "text": "# Parts\n\nA \"quoted\" line."}]}));
assert_eq!(result(5), &json!({"contents": [{"uri": "lib://logo", "mimeType": "image/png", "blob": "Zm9v"}]}));
assert_eq!(result(6), &json!({"contents": [{"uri": "lib://parts/bracket", "mimeType": "text/markdown", "text": "# bracket"}]}));
assert_eq!(replies[7]["error"], json!({"code": -32603, "message": "the part is being rebuilt"}));
assert_eq!(replies[8]["error"]["code"], -32002);
assert_eq!(s.tools().asked, [r#"describe {"part":"bracket"}"#]);
}
/// A prompt is filled in only with the arguments its listing declares, each a string, the
/// required ones present; anything else is invalid params and the app is never asked.
#[test]
fn prompt_arguments_are_checked_before_the_prompt_is_filled_in() {
let mut s = Server::new(info(), Library::default());
let get =
|s: &mut Server<Library>, params: Value| reply(s, &json!({"jsonrpc": "2.0", "id": 1, "method": "prompts/get", "params": params})).unwrap();
for (params, message) in [
(json!({"arguments": {"part": "a"}}), "missing prompt `name`"),
(json!({"name": "summarise"}), "unknown prompt `summarise`"),
(json!({"name": "describe"}), "describe: missing required argument `part`"),
(json!({"name": "describe", "arguments": ["a"]}), "describe: arguments must be an object"),
(json!({"name": "describe", "arguments": {"part": 3}}), "describe: argument `part` must be a string"),
(json!({"name": "describe", "arguments": {"part": null}}), "describe: argument `part` must be a string"),
(json!({"name": "describe", "arguments": {"part": "a", "size": "big"}}), "describe: unknown argument `size`; its arguments are: part, tone"),
] {
assert_eq!(get(&mut s, params.clone())["error"], json!({"code": -32602, "message": message}), "{params}");
}
assert!(s.tools().asked.is_empty());
let filled = get(&mut s, json!({"name": "describe", "arguments": {"part": "hinge", "tone": "dry"}}));
assert_eq!(filled["result"]["messages"][0]["content"]["text"], "Describe hinge in a dry tone.");
}
/// Completion asks the app about a listed prompt or a resource, and sends the first 100
/// suggestions with how many there are; a bad reference or argument is invalid params.
#[test]
fn completion_suggests_up_to_a_hundred_values() {
let mut s = Server::new(info(), Library::default());
let complete = |s: &mut Server<Library>, params: Value| {
reply(s, &json!({"jsonrpc": "2.0", "id": 1, "method": "completion/complete", "params": params})).unwrap()
};
let r = complete(&mut s, json!({"ref": {"type": "ref/prompt", "name": "describe"}, "argument": {"name": "part", "value": "br"}}));
let completion = &r["result"]["completion"];
assert_eq!(completion["values"].as_array().map(Vec::len), Some(100), "{r}");
assert_eq!((&completion["values"][0], &completion["total"], &completion["hasMore"]), (&json!("br0"), &json!(150), &json!(true)));
let r = complete(&mut s, json!({"ref": {"type": "ref/resource", "uri": "lib://parts/{name}"}, "argument": {"name": "name", "value": ""}}));
assert_eq!(r["result"]["completion"]["values"][99], "99");
assert_eq!(s.tools().asked, [r#"Prompt("describe") part=br"#, r#"Resource("lib://parts/{name}") name="#]);
let bad_ref = "`ref` must be a prompt's {type: ref/prompt, name} or a resource's {type: ref/resource, uri}";
let bad_argument = "`argument` must have a `name` and a `value`, both strings";
for (params, message) in [
(json!({"ref": {"type": "ref/prompt", "name": "summarise"}, "argument": {"name": "a", "value": ""}}), "unknown prompt `summarise`"),
(json!({"ref": {"type": "ref/tool", "name": "x"}, "argument": {"name": "a", "value": ""}}), bad_ref),
(json!({"ref": {"type": "ref/resource"}, "argument": {"name": "a", "value": ""}}), bad_ref),
(json!({"argument": {"name": "a", "value": ""}}), bad_ref),
(json!({"ref": {"type": "ref/prompt", "name": "describe"}, "argument": {"name": "part"}}), bad_argument),
(json!({"ref": {"type": "ref/prompt", "name": "describe"}}), bad_argument),
] {
assert_eq!(complete(&mut s, params.clone())["error"], json!({"code": -32602, "message": message}), "{params}");
}
assert_eq!(s.tools().asked.len(), 2, "a refused completion never reaches the app");
}
/// `Tools::call` runs right after the listing its arguments were checked against, every time,
/// whatever was listed before it; a refused call is listed and never run.
#[test]
fn every_call_runs_right_after_its_listing() {
let mut s = Server::new(info(), Echo::default());
let call = |name: &str, args: Value| json!({"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": name, "arguments": args}});
reply(&mut s, &json!({"jsonrpc": "2.0", "id": 1, "method": "tools/list"}));
reply(&mut s, &call("echo", json!({"text": "a"})));
reply(&mut s, &call("echo", json!({})));
reply(&mut s, &json!([call("level", json!({})), call("echo", json!({"text": "b"}))]));
assert_eq!(s.tools().log, ["list", "list", "call echo", "list", "list", "call level", "list", "call echo"]);
}
+114
View File
@@ -0,0 +1,114 @@
//! Tolerant fields from AVC and HEVC configuration records.
/// NAL length prefix width from AVC configuration, defaulting to four on a short record.
pub fn avcc_length_size(avcc: &[u8]) -> usize {
avcc.get(4).map_or(4, |b| (b & 3) as usize + 1)
}
/// NAL length prefix width and highest temporal ID from HEVC configuration.
pub fn hvcc_length_size_and_tid(hvcc: &[u8]) -> (usize, Option<u8>) {
let length_size = hvcc.get(21).map_or(4, |b| (b & 3) as usize + 1);
let highest_tid = hvcc.get(21).map(|b| (b >> 3) & 7).filter(|&n| n > 0).map(|n| n - 1);
(length_size, highest_tid)
}
/// objectTypeIndication of the DecoderConfigDescriptor inside an `esds` box in `raw`.
pub fn esds_object_type(raw: &[u8]) -> Option<u8> {
let at = raw.windows(4).position(|w| w == b"esds")? + 8;
let mut p = at;
let size = |p: &mut usize| -> Option<usize> {
let mut n = 0usize;
for _ in 0..4 {
let b = *raw.get(*p)?;
*p += 1;
n = (n << 7) | (b & 0x7F) as usize;
if b & 0x80 == 0 {
break;
}
}
Some(n)
};
if *raw.get(p)? != 0x03 {
return None;
}
p += 1;
size(&mut p)?;
let flags = *raw.get(p + 2)?;
p += 3;
if flags & 0x80 != 0 {
p += 2;
}
if flags & 0x40 != 0 {
p += 1 + *raw.get(p)? as usize;
}
if flags & 0x20 != 0 {
p += 2;
}
if *raw.get(p)? != 0x04 {
return None;
}
p += 1;
size(&mut p)?;
raw.get(p).copied()
}
/// E-AC-3 `ec-3` entry: the program-layout fields are the 24 bits starting at byte 2 of the
/// descriptor, then `bsmod(5) | acmod(3) | lfeon(1)`. Matches FFmpeg's `dec3` demuxer.
pub fn dec3_layout(payload: &[u8]) -> Option<(u32, u32)> {
if payload.len() < 5 {
return None;
}
let bits = u32::from(payload[2]) << 16 | u32::from(payload[3]) << 8 | u32::from(payload[4]);
let acmod = ((bits >> 9) & 0x7) as u8;
let lfeon = bits & 0x100 != 0;
let channels = dac3_channels(acmod, lfeon)?;
Some((0, channels))
}
/// AC-3 `ac-3` entry: `dac3` begins with the syncframe bitstream information: sample rate code,
/// bit stream id, bit stream mode, channel mode and LFE flag. The descriptor has no sample rate,
/// so the caller keeps the sample-entry rate.
pub fn dac3_layout(payload: &[u8]) -> Option<(u32, u32)> {
if payload.len() < 3 {
return None;
}
// The descriptor begins with the bitstream information: fscod(2), bsid(5), bsmod(3),
// acmod(3) and lfeon(1), continuing in the second payload byte.
let bits = u32::from_be_bytes([0, 0, payload[1], payload[2]]);
let acmod = ((bits >> 11) & 0x7) as u8;
let lfeon = bits & 0x400 != 0;
let channels = dac3_channels(acmod, lfeon)?;
Some((0, channels))
}
/// Full-bandwidth channels per AC-3 channel mode (Table 5.8) plus the optional LFE channel.
fn dac3_channels(acmod: u8, lfeon: bool) -> Option<u32> {
let n = [2u32, 1, 2, 3, 3, 4, 4, 5].get(usize::from(acmod & 7))?.checked_add(u32::from(lfeon))?;
(n > 0).then_some(n)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn audio_layout_and_elementary_descriptor_are_bounded() {
assert_eq!(dac3_layout(&[0, 0, 0]), Some((0, 2)));
assert_eq!(dec3_layout(&[0, 0, 0, 0, 0]), Some((0, 2)));
assert_eq!(dac3_layout(&[0, 0]), None);
assert_eq!(dec3_layout(&[0; 4]), None);
let esds = [b'e', b's', b'd', b's', 0, 0, 0, 0, 0x03, 0x04, 0, 0, 0, 0x04, 0x01, 0x60];
assert_eq!(esds_object_type(&esds), Some(0x60));
for end in 0..esds.len() {
assert_eq!(esds_object_type(&esds[..end]), None);
}
}
#[test]
fn short_records_and_temporal_layers() {
assert_eq!(avcc_length_size(&[]), 4);
assert_eq!(avcc_length_size(&[0, 0, 0, 0, 1]), 2);
assert_eq!(hvcc_length_size_and_tid(&[]), (4, None));
let mut h = [0; 22];
h[21] = (3 << 3) | 2;
assert_eq!(hvcc_length_size_and_tid(&h), (3, Some(2)));
}
}

Some files were not shown because too many files have changed in this diff Show More