Merge remote-tracking branch 'origin/codex/fcp7-bounded-report' into w1693

This commit is contained in:
FredApps
2026-11-16 08:54:47 +01:00
371 changed files with 43192 additions and 3460 deletions
+3
View File
@@ -14,3 +14,6 @@ crates/font-store/index/sources.json -text
# The build fixture verifies these license files are copied byte-for-byte.
crates/fonts-build/tests/fixture/craft-fonts/fonts/**/OFL.txt -text
# DXF drawing goldens assert exact writer bytes, including LF on every platform.
crates/dxf/tests/data/drawing_*.dxf -text
+1 -1
View File
@@ -23,5 +23,5 @@ jobs:
# The codec and format code behind features that are off by default. Without these the
# coverage build never compiles it, and new code there reads as untested.
cargo-features: >-
craft-video/dnx,craft-video/vp8,craft-video/prores,craft-video/h264,craft-video/hevc,craft-video/vp9,craft-video/av1,craft-video/threads,craft-audio/decode,craft-audio/encode,craft-audio/peaks,craft-audio/rayon,craft-audio/serde,craft-media/matroska,craft-media/ogg,craft-media/bitstream,craft-media/serde,craft-mesh/stl,craft-mesh/obj,craft-mesh/ply,craft-mesh/gltf,craft-metafile/write
craft-image-codecs/avif,craft-image-codecs/avif-decode,craft-video/dnx,craft-video/vp8,craft-video/prores,craft-video/h264,craft-video/hevc,craft-video/vp9,craft-video/av1,craft-video/threads,craft-audio/decode,craft-audio/encode,craft-audio/peaks,craft-audio/rayon,craft-audio/serde,craft-media/matroska,craft-media/ogg,craft-media/bitstream,craft-media/serde,craft-mesh/stl,craft-mesh/obj,craft-mesh/ply,craft-mesh/gltf,craft-metafile/write,craft-image-codecs/sgi,craft-image-codecs/sunraster,craft-jsonrpc/async-lines
secrets: inherit
+3
View File
@@ -100,3 +100,6 @@ and the exact transfer revision are recorded in each vendored crate's SOURCE.jso
## Binary Apple property lists
`crates/plist` is extracted from `storytold/craft-anything` (`crates/formats/src/plist.rs` and checked byte reads), at source main `88d1d9cac852aec1c7fd97886db451e005e49d2a`. MIT OR Apache-2.0; source license texts are retained in the crate. Primary import Git authors preserve Brandon Thomas, BFlat, RRFZ and Afloat. Regression imports preserve BFlat. XML, JSON projection, calendar formatting and app policy were not imported into the public API.
`crates/tga/src/metadata_tests.rs` and metadata-boundary helpers in `crates/tga/src/lib.rs` are adapted from nuclearspike's FilmCraft #1932 at `e7842f76f887005c460c64a8ab51d59d086e6f27` (MIT OR Apache-2.0). Synthetic fixtures are original contributor test data; primary Git author retained.
The XML reader in `crates/plist/src/xml.rs` and its base64 compatibility helper are ported from Craft Anything `8f3b775157d331dd31cfa4da911e26d8856b8c77` (same MIT OR Apache-2.0 source licenses). Brandon Thomas is the primary import author; retained hardening credits include Afloat, BFlat, Rahul, RRFZ and NetterSilver. The finite-real fix preserves gmrmad1 as author from Craft Anything #1905 (`d85e104f`); quoted-attribute/line-ending regressions preserve RRFZ. Bounds/adaptation and bounded entity scanning are subsequent Dylan M. Taylor changes. App-specific JSON projections and keyed-archive interpretation remain outside this crate.
Generated
+133 -38
View File
@@ -222,16 +222,6 @@ version = "1.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
[[package]]
name = "aotuv_lancer_vorbis_sys"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5bc4fd1a61860d2f1198b60bedd30910eaffa978f1ee6214dfb24ac70d589225"
dependencies = [
"cc",
"ogg_next_sys",
]
[[package]]
name = "arbitrary"
version = "1.5.0"
@@ -340,6 +330,12 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "astc-decode"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d6058e6202e20b7fa445692ec4b77ca8eb02508091fcd479ea753d9f60e376b"
[[package]]
name = "async-broadcast"
version = "0.7.2"
@@ -1414,7 +1410,6 @@ dependencies = [
"serde_json",
"symphonia",
"thiserror 2.0.21",
"vorbis_rs",
]
[[package]]
@@ -1425,6 +1420,7 @@ version = "0.1.0"
name = "craft-captions"
version = "0.1.0"
dependencies = [
"craft-media-testkit",
"craft-timeline",
"craft-xml",
"proptest",
@@ -1455,6 +1451,14 @@ dependencies = [
"encoding_rs",
]
[[package]]
name = "craft-control-protocol"
version = "0.1.0"
dependencies = [
"serde_json",
"subtle",
]
[[package]]
name = "craft-crs"
version = "0.1.0"
@@ -1490,6 +1494,17 @@ dependencies = [
"proptest",
]
[[package]]
name = "craft-deckcraft-format"
version = "0.1.0"
dependencies = [
"craft-zip",
"serde",
"serde_json",
"thiserror 2.0.21",
"zip",
]
[[package]]
name = "craft-designcraft-format"
version = "0.1.0"
@@ -1670,6 +1685,15 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-fcproj-format"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-fig"
version = "0.1.0"
@@ -1761,8 +1785,12 @@ name = "craft-font-catalog"
version = "0.1.0"
dependencies = [
"fontique",
"harfrust 0.13.3",
"kurbo 0.13.1",
"skrifa 0.47.0",
"thiserror 2.0.21",
"unicode-bidi",
"unicode-segmentation",
"windows-registry",
"write-fonts 0.54.1",
]
@@ -1821,6 +1849,16 @@ dependencies = [
"craft-fonts-build",
]
[[package]]
name = "craft-format"
version = "0.1.0"
dependencies = [
"craft-zip",
"proptest",
"serde_json",
"sha2 0.10.9",
]
[[package]]
name = "craft-gainmap"
version = "0.1.0"
@@ -1843,6 +1881,15 @@ dependencies = [
"image",
]
[[package]]
name = "craft-gridcraft-format"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-gzip"
version = "0.1.0"
@@ -1862,6 +1909,7 @@ name = "craft-heif"
version = "0.1.0"
dependencies = [
"heic-rs",
"proptest",
"rayon",
]
@@ -1878,6 +1926,10 @@ dependencies = [
"proptest",
]
[[package]]
name = "craft-http-url"
version = "0.1.0"
[[package]]
name = "craft-i18n"
version = "0.1.0"
@@ -1928,6 +1980,7 @@ dependencies = [
"craft-jpeg",
"craft-media-testkit",
"craft-metadata",
"craft-tiff",
"craft-video",
"crc32fast",
"exr",
@@ -1983,6 +2036,7 @@ name = "craft-jsonrpc"
version = "0.1.0"
dependencies = [
"serde_json",
"tokio",
]
[[package]]
@@ -2125,6 +2179,7 @@ version = "0.1.0"
dependencies = [
"craft-opc",
"craft-xml",
"craft-zip",
"miniz_oxide 0.9.1",
"serde_json",
]
@@ -2618,6 +2673,10 @@ dependencies = [
name = "craft-sheet"
version = "0.1.0"
[[package]]
name = "craft-shx"
version = "0.1.0"
[[package]]
name = "craft-silhouette-studio"
version = "0.1.0"
@@ -2745,6 +2804,7 @@ version = "0.1.0"
dependencies = [
"hypher",
"icu_segmenter",
"kurbo 0.13.1",
"miniz_oxide 0.8.9",
"unicode-bidi",
"unicode-bidi-mirroring",
@@ -2757,6 +2817,8 @@ dependencies = [
name = "craft-texture"
version = "0.1.0"
dependencies = [
"bytemuck",
"dds",
"rayon",
"thiserror 2.0.21",
]
@@ -2854,6 +2916,14 @@ dependencies = [
"zip",
]
[[package]]
name = "craft-usd"
version = "0.1.0"
dependencies = [
"lz4_flex",
"thiserror 2.0.21",
]
[[package]]
name = "craft-vcard"
version = "0.1.0"
@@ -3294,6 +3364,19 @@ dependencies = [
"generic-array",
]
[[package]]
name = "dds"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e917a58e17b27d2f15626c490754ccced7f46592dc5b5f39a3013b89cba5f068"
dependencies = [
"astc-decode",
"bitflags 2.13.2",
"glam",
"resize",
"zerocopy",
]
[[package]]
name = "der"
version = "0.7.10"
@@ -4542,6 +4625,12 @@ dependencies = [
"xml-rs",
]
[[package]]
name = "glam"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "151665d9be52f9bb40fc7966565d39666f2d1e69233571b71b87791c7e0528b3"
[[package]]
name = "glifo"
version = "0.1.1"
@@ -6053,6 +6142,12 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f"
[[package]]
name = "lz4_flex"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a"
[[package]]
name = "maybe-owned"
version = "0.3.4"
@@ -6066,6 +6161,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519"
dependencies = [
"cfg-if",
"rayon",
]
[[package]]
@@ -6811,15 +6907,6 @@ dependencies = [
"phf 0.11.3",
]
[[package]]
name = "ogg_next_sys"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed2d7a48e247c2bb07e633aefb65a38648ea58c7eedd4e4408a5861721ab049b"
dependencies = [
"cc",
]
[[package]]
name = "oid-registry"
version = "0.8.1"
@@ -7912,6 +7999,7 @@ dependencies = [
"loop9",
"quick-error 2.0.1",
"rav1e",
"rayon",
"rgb",
]
@@ -8135,6 +8223,15 @@ dependencies = [
"web-sys",
]
[[package]]
name = "resize"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71725ecd5e0197b54fe859055b108688472ab6a358f8fbe5cee4a556b1b5bfea"
dependencies = [
"rgb",
]
[[package]]
name = "resvg"
version = "0.45.1"
@@ -8454,9 +8551,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f"
[[package]]
name = "rustls-webpki"
version = "0.103.15"
version = "0.103.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281"
dependencies = [
"aws-lc-rs",
"ring",
@@ -9846,9 +9943,21 @@ dependencies = [
"mio",
"pin-project-lite",
"socket2",
"tokio-macros",
"windows-sys 0.61.2",
]
[[package]]
name = "tokio-macros"
version = "2.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.6",
]
[[package]]
name = "tokio-rustls"
version = "0.26.6"
@@ -10638,20 +10747,6 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "vorbis_rs"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49c5da94d280f7a27e8c937e9b73df2da3e23a2583f48471fd8fb4c72f9c1933"
dependencies = [
"aotuv_lancer_vorbis_sys",
"errno",
"getrandom 0.4.3",
"ogg_next_sys",
"thiserror 2.0.21",
"tinyvec",
]
[[package]]
name = "wait-timeout"
version = "0.2.1"
@@ -11881,9 +11976,9 @@ dependencies = [
[[package]]
name = "zip"
version = "9.1.0"
version = "9.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2886b00003a72c4ccee3f42cfc34908d33b2508a74d827dc2a4244706475c704"
checksum = "1156b39d142a34b999c2310a50455e0cb3760bf7efad0122f796d813bee26ccb"
dependencies = [
"crc32fast",
"flate2",
+7
View File
@@ -11,6 +11,9 @@ rust-version = "1.95"
repository = "https://github.com/storytold/craft-libs"
[workspace.dependencies]
craft-shx = { path = "crates/shx" }
craft-http-url = { path = "crates/http-url" }
craft-usd = { path = "crates/usd" }
craft-smime = { path = "crates/smime" }
craft-ber = { path = "crates/ber" }
craft-proto = { path = "crates/proto" }
@@ -39,6 +42,7 @@ bytemuck = "1"
kurbo = "0.13"
thiserror = "2"
craft-gzip = { path = "crates/gzip" }
craft-gridcraft-format = { path = "crates/gridcraft-format" }
craft-aaf = { path = "crates/aaf" }
craft-atomic = { path = "crates/atomic" }
craft-captions = { path = "crates/captions" }
@@ -54,6 +58,7 @@ craft-doc-binary = { path = "crates/doc-binary" }
craft-docx = { path = "crates/docx" }
craft-document-bundle = { path = "crates/document-bundle" }
craft-mcmat = { path = "crates/mcmat" }
craft-deckcraft-format = { path = "crates/deckcraft-format" }
craft-designcraft-format = { path = "crates/designcraft-format" }
craft-dxf = { path = "crates/dxf" }
craft-drawingml = { path = "crates/drawingml" }
@@ -68,6 +73,7 @@ craft-doc = { path = "crates/doc" }
craft-docpath = { path = "crates/docpath" }
craft-dpx = { path = "crates/dpx" }
craft-dwg = { path = "crates/dwg" }
craft-fcproj-format = { path = "crates/fcproj-format" }
craft-ecproj-format = { path = "crates/ecproj-format" }
craft-figma-rest = { path = "crates/figma-rest" }
craft-exr = { path = "crates/exr" }
@@ -91,6 +97,7 @@ craft-indd = { path = "crates/indd" }
craft-jmap = { path = "crates/jmap" }
craft-ical = { path = "crates/ical" }
craft-jpeg = { path = "crates/jpeg" }
craft-control-protocol = { path = "crates/control-protocol" }
craft-jsonrpc = { path = "crates/jsonrpc" }
craft-mcp = { path = "crates/mcp" }
craft-jxl = { path = "crates/jxl" }
+7
View File
@@ -120,6 +120,10 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
pages and version detection, and written back; and `model`, the drawing above the tags (header
variables, tables, blocks, layouts, entities) and its writer (R12 to 2018), merged from CADCraft's and VectorCraft's readers. Bounded
on hostile input. Mapping the model onto a document stays in each app.
- [`craft-deckcraft-format`](crates/deckcraft-format/README.md): the DeckCraft `.deckcraft` package:
identity and sniffing, the `presentation.json` envelope with the presentation as a type
parameter, NaN-safe JSON and the reader for older files with `null` numbers, member names and
compression, and bounded, budgeted member reads. The presentation model and its mapping stay in DeckCraft.
- [`craft-dwg`](crates/dwg/README.md): bounded byte-level DWG↔DXF conversion with diagnostics,
capped output and an optional child-process worker; the `acadrust` dependency remains isolated
behind bytes and is unavailable on wasm32 and FreeBSD/aarch64.
@@ -172,6 +176,9 @@ See the [release notes](CHANGELOG.md) for included functionality and known limit
every frame's position, delay, disposal and transparent colour, the loop count and the first
frame's colour table, without decompressing pixels. Tells an animation from a still and whether
a file can hold the picture it declares before a decoder runs.
- [`craft-gridcraft-format`](crates/gridcraft-format/README.md): GridCraft's native JSON workbook
(`.json`, `.scjson`): the extensions, the reader, the pretty writer, and newer entries kept
unread and written back into a list. The workbook model and what follows a read stay in GridCraft.
- [`craft-hdr`](crates/hdr/src/lib.rs): Radiance HDR (RGBE) picture reader, from FilmCraft: flat,
old and new run-length scan lines and all eight axis orders, to linear float RGB. Scan lines are
validated before the picture is allocated.
+45 -2
View File
@@ -57,13 +57,56 @@ pub struct WriteOptions<'a> {
/// reported.
pub fn write(doc: &Document, opts: &WriteOptions<'_>, report: &mut Report) -> Result<Vec<u8>> {
let version = if opts.small_sectors { craft_cfb::Version::V3 } else { craft_cfb::Version::V4 };
write::write(doc, version, &opts.identity, report)
write::write(doc, version, &opts.identity, &ReferenceOptions::default(), report)
}
/// Optional interpretation choices for applications retaining older interchange behavior.
#[derive(Clone, Copy, Debug)]
pub struct ReadOptions {
/// Interpret a source clip's start using its target slot's edit rate.
pub source_slot_rate: bool,
/// Use the referenced mob name when the clip has no explicit name.
pub infer_clip_name: bool,
/// Decode percent escapes in relative network locators.
pub decode_relative_locator: bool,
/// Report selector and nested-scope flattening.
pub report_flattening: bool,
/// Accept truncated PCM WAVE essence summaries.
pub permissive_pcm: bool,
}
impl Default for ReadOptions {
fn default() -> Self {
Self { source_slot_rate: true, infer_clip_name: true, decode_relative_locator: true, report_flattening: true, permissive_pcm: false }
}
}
/// Read using explicitly selected interpretation choices.
pub fn read_with_options(bytes: &[u8], options: &ReadOptions, report: &mut Report) -> Result<Document> {
read::read(bytes, options, report)
}
/// Optional compatibility choices for writing source references and locators.
#[derive(Clone, Copy, Debug)]
pub struct ReferenceOptions {
/// Write source offsets in target-slot units.
pub source_slot_rate: bool,
/// Percent-encode relative locators.
pub encode_relative_locator: bool,
}
impl Default for ReferenceOptions {
fn default() -> Self {
Self { source_slot_rate: true, encode_relative_locator: true }
}
}
/// Write with explicitly selected reference and locator conventions.
pub fn write_with_references(doc: &Document, opts: &WriteOptions<'_>, references: &ReferenceOptions, report: &mut Report) -> Result<Vec<u8>> {
let version = if opts.small_sectors { craft_cfb::Version::V3 } else { craft_cfb::Version::V4 };
write::write(doc, version, &opts.identity, references, report)
}
/// Read an AAF file: its top-level compositions, the compositions nested in them and their
/// sources. What cannot be represented is skipped and named in `report`.
pub fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
read::read(bytes, report)
read::read(bytes, &ReadOptions::default(), report)
}
/// Whether `bytes` look like an AAF file (a compound file whose root has the AAF root class or a
+16
View File
@@ -108,3 +108,19 @@ fn objects_nested_too_deeply_are_an_error() {
let e = store::read(&chain(250)).unwrap_err();
assert!(e.contains("nested too deeply"), "{e}");
}
/// Many distinct property IDs still reject a duplicate without quadratic scanning.
#[test]
fn duplicate_after_many_distinct_properties_is_rejected() {
let mut props = vec![0x4C, 0x20];
props.extend_from_slice(&8193u16.to_le_bytes());
for id in (0..8192u16).chain(std::iter::once(7)) {
props.extend_from_slice(&id.to_le_bytes());
props.extend_from_slice(&0x82u16.to_le_bytes());
props.extend_from_slice(&0u16.to_le_bytes());
}
let mut writer = Writer::new(Version::V3);
writer.stream(Writer::ROOT, "properties", props).unwrap();
let e = store::read(&writer.finish()).unwrap_err();
assert!(e.contains("duplicate property id"), "{e}");
}
+25 -7
View File
@@ -67,6 +67,7 @@ fn indirect_f64(d: &[u8]) -> Option<f64> {
struct R<'a> {
report: &'a mut Report,
options: super::ReadOptions,
mobs: HashMap<MobId, &'a Obj>,
essence: HashMap<MobId, &'a [u8]>,
doc: Document,
@@ -90,7 +91,7 @@ struct NestClip<'a> {
/// Compositions nested deeper than this are read as gaps.
const MAX_NESTING: usize = 16;
pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
pub(crate) fn read(bytes: &[u8], options: &super::ReadOptions, report: &mut Report) -> Result<Document> {
let root = store::read(bytes).map_err(|e| Error::Parse { format: "AAF", message: e })?;
let header = root.strong(pid::ROOT_HEADER).ok_or_else(|| Error::Parse { format: "AAF", message: "no header object".into() })?;
let content = header.strong(pid::CONTENT).ok_or_else(|| Error::Parse { format: "AAF", message: "no content storage".into() })?;
@@ -106,7 +107,8 @@ pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
essence.insert(id, d);
}
}
let mut r = R { report, mobs, essence, doc: Document::default(), sources: HashMap::new(), nests: HashMap::new(), open: Vec::new() };
let mut r =
R { report, options: *options, mobs, essence, doc: Document::default(), sources: HashMap::new(), nests: HashMap::new(), open: Vec::new() };
// Top-level compositions: those no other composition uses, among the ones tagged top-level
// when any are. (Premiere Pro tags a nested sequence's composition top-level too; it is still
// only a nested sequence.) Compositions that only use each other in a circle all count. The
@@ -331,7 +333,9 @@ impl R<'_> {
items
}
x if x == cls::SELECTOR => {
self.report.info("a selector was flattened to its selected input");
if self.options.report_flattening {
self.report.info("a selector was flattened to its selected input");
}
match c.strong(pid::SELECTED) {
Some(s) => {
let mut items = self.segment(s, rate, kind, depth);
@@ -342,7 +346,9 @@ impl R<'_> {
}
}
x if x == cls::NESTED_SCOPE => {
self.report.info("a nested scope was flattened to its final input");
if self.options.report_flattening {
self.report.info("a nested scope was flattened to its final input");
}
match c.objs(pid::NESTED_SLOTS).last() {
Some(s) => {
let mut items = self.segment(s, rate, kind, depth);
@@ -388,7 +394,12 @@ impl R<'_> {
let len = units_to_ticks(len_of(c), rate.0, rate.1);
let tagged_name = tagged_values(c, pid::COMPONENT_USER_COMMENTS).into_iter().find(|(k, _)| k == "Clip Name").map(|(_, v)| v);
let (Some(id), Some(slot)) = (c.mob_id(pid::SOURCE_ID), c.u32(pid::SOURCE_MOB_SLOT_ID)) else { return CItem::Filler(len) };
let name = tagged_name.or_else(|| self.mobs.get(&id).and_then(|mob| mob.string(pid::MOB_NAME))).unwrap_or_default();
let name = if self.options.infer_clip_name {
tagged_name.or_else(|| self.mobs.get(&id).and_then(|mob| mob.string(pid::MOB_NAME)))
} else {
tagged_name
}
.unwrap_or_default();
if id == [0; 32] {
return CItem::Filler(len);
}
@@ -401,6 +412,7 @@ impl R<'_> {
.and_then(|mob| mob.objs(pid::SLOTS).iter().find(|s| s.u32(pid::SLOT_ID) == Some(slot)))
.and_then(|s| rate_of(s, pid::EDIT_RATE))
.unwrap_or(rate);
let source_rate = if self.options.source_slot_rate { source_rate } else { rate };
let start = units_to_ticks(c.i64(pid::START_TIME).unwrap_or(0), source_rate.0, source_rate.1);
match self.resolve(id, slot, start, kind, master_key, 0) {
Some((source, t)) => CItem::Clip(CClip { len, source, start: t, gain: None, name }),
@@ -506,6 +518,7 @@ impl R<'_> {
.and_then(|m| m.objs(pid::SLOTS).iter().find(|s| s.u32(pid::SLOT_ID) == Some(next_slot)))
.and_then(|s| rate_of(s, pid::EDIT_RATE))
.unwrap_or(rate);
let source_rate = if self.options.source_slot_rate { source_rate } else { rate };
let inner = units_to_ticks(clip.i64(pid::START_TIME).unwrap_or(0), source_rate.0, source_rate.1);
if next == [0; 32] {
return None;
@@ -545,7 +558,11 @@ impl R<'_> {
DataKind::Sound => CKind::Sound,
_ => kind,
};
let path = d.objs(pid::LOCATOR).iter().find_map(|l| l.string(pid::URL_STRING)).map(|u| locator_path(&u));
let path = d
.objs(pid::LOCATOR)
.iter()
.find_map(|l| l.string(pid::URL_STRING))
.map(|u| if self.options.decode_relative_locator { locator_path(&u) } else { craft_timeline::path::file_url_to_path(&u) });
let file_rate = rate_of(d, pid::SAMPLE_RATE).unwrap_or(rate);
let length = d
.i64(pid::FILE_LENGTH)
@@ -604,7 +621,8 @@ impl R<'_> {
if let Some(data) = self.essence.get(&id) {
s.embedded = Some(data.to_vec());
// embedded essence may be a whole WAVE / AIFF file
if let Some((pcm, ch, sr, bits)) = crate::pcm::parse_wav(data) {
let pcm = if self.options.permissive_pcm { craft_audio::essence_pcm::parse_wav(data) } else { crate::pcm::parse_wav(data) };
if let Some((pcm, ch, sr, bits)) = pcm {
s.embedded = Some(pcm);
s.file_channels = ch as u32;
s.channels = if s.channel.is_some() { 1 } else { ch as u32 };
+3 -1
View File
@@ -461,11 +461,13 @@ impl R<'_> {
if at > props.len() {
return Err(bad("truncated property index"));
}
// A properties stream can name tens of thousands of ids: do not rescan all earlier ids.
let mut seen = std::collections::HashSet::with_capacity(n);
for i in 0..n {
let (Some(pid), Some(sf), Some(len)) = (rd_u16(&props, 4 + i * 6), rd_u16(&props, 6 + i * 6), rd_u16(&props, 8 + i * 6)) else {
return Err(bad("truncated property index"));
};
if o.props.iter().any(|(previous, _)| *previous == pid) {
if !seen.insert(pid) {
return Err(bad("duplicate property id"));
}
let v = props.get(at..at + len as usize).ok_or_else(|| bad("truncated property value"))?;
+29
View File
@@ -557,3 +557,32 @@ fn relative_media_locator_round_trips_escaped_characters() {
let got = read_clean(&bytes);
assert!(got.sources.iter().any(|s| s.path.as_deref() == Some("Audio Files/100%20 #1.wav")));
}
#[test]
fn compatibility_options_keep_relative_paths_and_parent_rate() {
let mut doc = crate::testdoc::sound_only(None);
doc.compositions[0].sample_rate = 44_100;
doc.sources[0].path = Some("Audio Files/100%20 #1.wav".into());
let mut report = Report::default();
let references = ReferenceOptions { source_slot_rate: false, encode_relative_locator: false };
let bytes = write_with_references(&doc, &WriteOptions { identity: IDENTITY, small_sectors: false }, &references, &mut report).unwrap();
let root = store::read(&bytes).unwrap();
let mut all = Vec::new();
walk(&root, &mut all);
assert!(
all.iter().filter(|o| o.class == cls::NETWORK_LOCATOR).any(|o| o.string(pid::URL_STRING).as_deref() == Some("Audio Files/100%20 #1.wav"))
);
let starts: Vec<i64> = all.iter().filter(|o| o.class == cls::SOURCE_CLIP).filter_map(|o| o.i64(pid::START_TIME)).collect();
assert!(starts.contains(&17_640), "parent rate start: {starts:?}");
let opts = ReadOptions {
source_slot_rate: false,
infer_clip_name: false,
decode_relative_locator: false,
report_flattening: false,
permissive_pcm: true,
};
let back = read_with_options(&bytes, &opts, &mut report).unwrap();
assert!(back.sources.iter().any(|s| s.path.as_deref() == Some("Audio Files/100%20 #1.wav")));
let clip = back.compositions[0].tracks[0].items.iter().find_map(|i| if let CItem::Clip(c) = i { Some(c) } else { None }).unwrap();
assert_eq!(clip.start, FrameRate::FPS_25.tick_of(10));
}
+14 -4
View File
@@ -75,6 +75,7 @@ fn definition(class: Auid, id: Auid, name: &str) -> Obj {
struct W<'a> {
report: &'a mut Report,
inexact: bool,
references: super::ReferenceOptions,
}
struct IdGen {
@@ -350,9 +351,15 @@ fn add_tape_mob(first: &Source, tape_id: [u8; 32], timecode: Option<(i64, FrameR
}
/// Serialise a composition document.
pub(crate) fn write(doc: &Document, version: craft_cfb::Version, identity: &Identity<'_>, report: &mut Report) -> Result<Vec<u8>> {
pub(crate) fn write(
doc: &Document,
version: craft_cfb::Version,
identity: &Identity<'_>,
references: &super::ReferenceOptions,
report: &mut Report,
) -> Result<Vec<u8>> {
let comp = doc.compositions.first().ok_or(Error::Empty)?;
let mut w = W { report, inexact: false };
let mut w = W { report, inexact: false, references: *references };
let mut ids = IdGen::for_document(doc, comp);
let MediaOutput { mut mobs, essence, master_ref } = write_media(doc, &mut w, &mut ids);
let mut used_ops: Vec<Auid> = Vec::new();
@@ -360,7 +367,8 @@ pub(crate) fn write(doc: &Document, version: craft_cfb::Version, identity: &Iden
// ---- nested compositions: a clip refers to one as it does to a master mob, through the
// composition's first track of the clip's kind (a broken-out channel: that channel's track)
let mut refs = Refs { mobs: master_ref, rates: doc.sources.iter().enumerate().map(|(i, s)| (i, source_rate(s))).collect() };
let rates = if references.source_slot_rate { doc.sources.iter().enumerate().map(|(i, s)| (i, source_rate(s))).collect() } else { HashMap::new() };
let mut refs = Refs { mobs: master_ref, rates };
let nested_ids: Vec<[u8; 32]> = doc.nested.iter().map(|_| ids.next()).collect();
for (si, s) in doc.sources.iter().enumerate() {
let Some((comp, id)) = s.nested.and_then(|n| Some((doc.nested.get(n)?, nested_ids.get(n)?))) else { continue };
@@ -795,8 +803,10 @@ fn descriptor(w: &mut W, s: &Source) -> Obj {
if let Some(p) = &s.path {
let url = if craft_timeline::path::is_absolute(p) {
craft_timeline::path::path_to_file_url(p, false)
} else {
} else if w.references.encode_relative_locator {
craft_timeline::path::encode_uri_path(p)
} else {
p.clone()
};
d.set(pid::LOCATOR, Value::StrongVec(vec![Obj::new(cls::NETWORK_LOCATOR).data_prop(pid::URL_STRING, utf16z(&url))]));
}
+4
View File
@@ -29,7 +29,11 @@ mod roundtrip_tests;
#[cfg(test)]
mod testdoc;
/// Diagnostic types used by readers and writers.
pub use craft_timeline::report;
pub use craft_timeline::report::Report;
/// Exact time types used by the composition model.
pub use craft_timeline::time;
/// Error from a reader or writer. Unsupported *features* never error; they go in the [`Report`].
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
+1 -1
View File
@@ -21,7 +21,7 @@
//! transition's name) are not in the specification; they were first written by FilmCraft and keep
//! their names so that existing files read back.
pub(crate) mod bento;
pub mod bento;
mod read;
mod write;
+18 -7
View File
@@ -35,8 +35,8 @@
use std::collections::{BTreeMap, HashMap};
pub(crate) const MAGIC: [u8; 8] = [0xA4, b'C', b'M', 0xA5, b'H', b'd', b'r', 0xD7];
pub(crate) const LABEL_SIZE: usize = 24;
pub const MAGIC: [u8; 8] = [0xA4, b'C', b'M', 0xA5, b'H', b'd', b'r', 0xD7];
pub const LABEL_SIZE: usize = 24;
// standard object ids
const TOC_OBJECT: u32 = 1;
@@ -82,7 +82,7 @@ const MAX_VALUE_BYTES: usize = 16 << 20;
const MAX_NAME_BYTES: usize = 4 << 20;
/// Builds a Bento container.
pub(crate) struct Writer {
pub struct Writer {
data: Vec<u8>,
names: HashMap<(u32, String), u32>,
next: u32,
@@ -90,6 +90,12 @@ pub(crate) struct Writer {
entries: Vec<(u32, u32, u32, Vec<Seg>)>,
}
impl Default for Writer {
fn default() -> Self {
Self::new()
}
}
impl Writer {
pub fn new() -> Writer {
Writer { data: Vec::new(), names: HashMap::new(), next: MIN_USER_ID, entries: Vec::new() }
@@ -217,9 +223,9 @@ pub(crate) struct Prop {
}
/// A parsed Bento container.
pub(crate) struct Container<'a> {
pub struct Container<'a> {
data: &'a [u8],
pub objects: BTreeMap<u32, Vec<Prop>>,
pub(crate) objects: BTreeMap<u32, Vec<Prop>>,
names: HashMap<u32, String>,
by_name: HashMap<String, u32>,
}
@@ -229,11 +235,16 @@ fn be32(b: &[u8], at: usize) -> Option<u32> {
}
/// Whether `b` ends with a Bento label.
pub(crate) fn sniff(b: &[u8]) -> bool {
pub fn sniff(b: &[u8]) -> bool {
b.len() >= LABEL_SIZE && b[b.len() - LABEL_SIZE..b.len() - LABEL_SIZE + 8] == MAGIC
}
impl<'a> Container<'a> {
/// Object identifiers in ascending order.
pub fn object_ids(&self) -> impl Iterator<Item = u32> + '_ {
self.objects.keys().copied()
}
pub fn open(data: &'a [u8]) -> Result<Container<'a>, String> {
if !sniff(data) {
return Err("no Bento label at the end of the file".into());
@@ -324,7 +335,7 @@ impl<'a> Container<'a> {
}
/// The bytes of a value.
pub fn value(&self, p: &Prop) -> Result<Vec<u8>, String> {
pub(crate) fn value(&self, p: &Prop) -> Result<Vec<u8>, String> {
let total = p.segs.iter().try_fold(0usize, |total, s| {
let len = match s {
Seg::Imm(b) => b.len(),
+340 -10
View File
@@ -12,6 +12,10 @@ type Uid = [u8; 12];
const MAX_REFERENCE_VISITS: usize = 65_536;
/// A source-clip search stopped at its work limit.
#[derive(Clone, Copy)]
struct SearchLimit;
struct R<'a, 'b> {
c: &'b Container<'a>,
report: &'b mut Report,
@@ -21,15 +25,49 @@ struct R<'a, 'b> {
media: HashMap<Uid, Vec<u8>>,
doc: Document,
sources: HashMap<(Uid, u32, String), usize>,
/// Component visits left for [`R::segment`] over the whole document.
budget: usize,
/// Outcome of the source-clip search under each segment: deterministic, so it is made once.
clip_search: HashMap<u32, std::result::Result<Option<u32>, SearchLimit>>,
/// Visits and queued references left for source-clip searches over the whole document:
/// distinct roots over one shared graph would otherwise each search it in full.
search_budget: usize,
}
/// Components one document may expand into timeline items; a small file can reference the same
/// sequence from many places and would otherwise multiply without bound.
const MAX_COMPONENT_VISITS: usize = 1_000_000;
/// Reported when the shared component budget runs out.
const BUDGET_WARNING: &str = "OMF composition is too large to import completely (the remaining components are dropped)";
/// Most visited components, and most queued references, one source-clip search may spend.
const MAX_CLIP_SEARCH_VISITS: usize = 100_000;
const MAX_CLIP_SEARCH_PENDING: usize = 1_000_000;
/// Work all source-clip searches in one document may spend together.
const MAX_DOCUMENT_SEARCH_WORK: usize = 2_000_000;
pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
read_with_budget(bytes, report, MAX_COMPONENT_VISITS)
}
fn read_with_budget(bytes: &[u8], report: &mut Report, budget: usize) -> Result<Document> {
let perr = |m: String| Error::Parse { format: "OMF", message: m };
let c = Container::open(bytes).map_err(perr)?;
let class_of = |o: u32| c.get(o, "OMFI:OOBJ:ObjClass").and_then(|v| v.get(..4).map(|b| [b[0], b[1], b[2], b[3]]));
let head = c.objects.keys().copied().find(|&o| class_of(o) == Some(*b"HEAD")).ok_or_else(|| perr("no OMF header object".into()))?;
let be = c.get(head, "OMFI:HEAD:ByteOrder").is_none_or(|v| v.first() != Some(&0x49));
let mut r = R { c: &c, report, be, mobs: HashMap::new(), media: HashMap::new(), doc: Document::default(), sources: HashMap::new() };
let mut r = R {
c: &c,
report,
be,
mobs: HashMap::new(),
media: HashMap::new(),
doc: Document::default(),
sources: HashMap::new(),
budget,
clip_search: HashMap::new(),
search_budget: MAX_DOCUMENT_SEARCH_WORK,
};
let mob_list = r.refs(head, "OMFI:HEAD:Mobs");
let mob_list = if mob_list.is_empty() {
c.objects.keys().copied().filter(|&o| matches!(class_of(o).as_ref(), Some(b"CMOB" | b"MMOB" | b"SMOB"))).collect()
@@ -66,8 +104,14 @@ pub(crate) fn read(bytes: &[u8], report: &mut Report) -> Result<Document> {
cmobs.iter().copied().filter(|m| r.uid(*m, "OMFI:MOBJ:MobID").is_none_or(|id| !referenced.contains(&id))).collect()
};
for m in top {
// every composition spends from the budget too: a list of up to 65,535 repeated entries must not yield that many
if r.budget == 0 && !r.doc.compositions.is_empty() {
r.report.warn(BUDGET_WARNING);
break;
}
let comp = r.composition(m);
r.doc.compositions.push(comp);
r.budget = r.budget.saturating_sub(1);
}
if r.doc.compositions.is_empty() {
return Err(Error::Empty);
@@ -166,6 +210,13 @@ impl R<'_, '_> {
let slots = self.refs(m, "OMFI:MOBJ:Slots");
let mut na = 0;
for s in slots {
if self.budget == 0 {
self.report.warn(BUDGET_WARNING);
break;
}
// Every slot spends a unit, skipped ones too: a list of up to 65,535 repeated slots
// must not cost that much work in each of 65,535 repeated compositions.
self.budget -= 1;
let Some(seg) = self.r#ref(s, "OMFI:MSLT:Segment") else { continue };
let rate = self.rational(s, "OMFI:MSLT:EditRate").filter(|r| r.0 > 0 && r.1 > 0).unwrap_or((25, 1));
let kind = self.data_kind(seg);
@@ -213,13 +264,32 @@ impl R<'_, '_> {
fn segment(&mut self, seg: u32, rate: (i64, i64), depth: usize) -> Vec<CItem> {
let len = units_to_ticks(self.len(seg), rate.0, rate.1);
// Depth alone does not bound a graph that references the same component many times at
// every level, so each visit also spends from a per-document budget.
// A visit stopped by depth spends from it too.
if self.budget == 0 {
self.report.warn(BUDGET_WARNING);
return vec![CItem::Filler(len)];
}
self.budget -= 1;
if depth > 32 {
return vec![CItem::Filler(len)];
}
match &self.class(seg) {
b"SEQU" => {
let mut out = Vec::new();
let mut expanded = 0i64;
for c in self.refs(seg, "OMFI:SEQU:Components") {
if self.budget == 0 {
// the rest of the sequence keeps its duration as one gap, not a gap per reference
self.report.warn(BUDGET_WARNING);
let rest = self.len(seg).saturating_sub(expanded);
if rest > 0 {
out.push(CItem::Filler(units_to_ticks(rest, rate.0, rate.1)));
}
break;
}
expanded = expanded.saturating_add(self.len(c));
out.extend(self.segment(c, rate, depth + 1));
}
out
@@ -306,12 +376,66 @@ impl R<'_, '_> {
.find(|&s| self.r#ref(s, "OMFI:MSLT:TrackDesc").and_then(|t| self.u32(t, "OMFI:TRKD:TrackID")) == Some(track))
}
fn first_clip(&self, seg: u32) -> Option<u32> {
match &self.class(seg) {
b"SCLP" => Some(seg),
b"SEQU" => self.refs(seg, "OMFI:SEQU:Components").into_iter().find_map(|c| self.first_clip(c)),
_ => None,
/// First source clip under `seg`, depth-first; a sequence referencing itself (directly or
/// through a cycle) ends the search rather than the stack. `Err` when the search spent its
/// budget of visited components or queued references before finishing (the answer is unknown).
fn first_clip(&mut self, seg: u32) -> std::result::Result<Option<u32>, SearchLimit> {
if let Some(found) = self.clip_search.get(&seg) {
return *found;
}
let mut spent = 0;
let found = if self.search_budget == 0 {
Err(SearchLimit)
} else {
let visits = MAX_CLIP_SEARCH_VISITS.min(self.search_budget);
let pending = MAX_CLIP_SEARCH_PENDING.min(self.search_budget);
self.first_clip_within(seg, visits, pending, &mut spent)
};
self.search_budget = self.search_budget.saturating_sub(spent);
self.clip_search.insert(seg, found);
found
}
/// `spent` receives the visits plus queued references the search used.
fn first_clip_within(&self, seg: u32, max_visits: usize, max_pending: usize, spent: &mut usize) -> std::result::Result<Option<u32>, SearchLimit> {
let mut visited = HashSet::new();
let mut todo = vec![seg];
let mut queued = 1usize;
let found = Self::search_from(self, &mut visited, &mut todo, &mut queued, max_visits, max_pending);
*spent = visited.len().saturating_add(queued);
found
}
fn search_from(
&self,
visited: &mut HashSet<u32>,
todo: &mut Vec<u32>,
queued: &mut usize,
max_visits: usize,
max_pending: usize,
) -> std::result::Result<Option<u32>, SearchLimit> {
while let Some(seg) = todo.pop() {
if !visited.insert(seg) {
continue;
}
if visited.len() > max_visits {
return Err(SearchLimit);
}
match &self.class(seg) {
b"SCLP" => return Ok(Some(seg)),
b"SEQU" => {
let comps = self.refs(seg, "OMFI:SEQU:Components");
// sequences may share one component list, so queued references grow with expansions, not visits
*queued = queued.saturating_add(comps.len());
if *queued > max_pending {
return Err(SearchLimit);
}
todo.extend(comps.into_iter().rev());
}
_ => {}
}
}
Ok(None)
}
fn resolve(&mut self, id: Uid, track: u32, start: Tick, master: Option<String>, depth: usize) -> Option<(usize, Tick)> {
@@ -328,7 +452,13 @@ impl R<'_, '_> {
match &self.class(mob) {
b"MMOB" => {
let key = master.unwrap_or_else(|| id.iter().map(|b| format!("{b:02x}")).collect());
let clip = self.first_clip(seg)?;
let clip = match self.first_clip(seg) {
Ok(c) => c?,
Err(SearchLimit) => {
self.report.warn("a master mob is too large to search for its source clip (imported as a gap)");
return None;
}
};
let inner = units_to_ticks(self.i32(clip, "OMFI:SCLP:StartTime").unwrap_or(0), rate.0, rate.1);
let next = self.uid(clip, "OMFI:SCLP:SourceID")?;
let next_track = self.u32(clip, "OMFI:SCLP:SourceTrackID")?;
@@ -353,7 +483,9 @@ impl R<'_, '_> {
let file_rate = sr.filter(|r| r.0 > 0 && r.1 > 0).unwrap_or(rate);
let length = units_to_ticks(self.i32(desc, "OMFI:MDFL:Length").unwrap_or(0), file_rate.0, file_rate.1);
let summary = self.c.get(desc, "OMFI:WAVD:Summary").or_else(|| self.c.get(desc, "OMFI:AIFD:Summary")).unwrap_or_default();
let fmt = crate::pcm::parse_wav(&summary).or_else(|| crate::pcm::parse_aiff(&summary)).map(|(_, ch, sr, bits)| (ch as u32, sr, bits));
let fmt = craft_audio::essence_pcm::parse_wav(&summary)
.or_else(|| craft_audio::essence_pcm::parse_aiff(&summary))
.map(|(_, ch, sr, bits)| (ch as u32, sr, bits));
let mut path = None;
for l in self.refs(desc, "OMFI:MDES:Locator") {
let p = match &self.class(l) {
@@ -373,7 +505,8 @@ impl R<'_, '_> {
let (file_channels, sample_rate, bits) = fmt.unwrap_or((1, file_rate.0 as u32, 16));
let channel = (file_channels > 1 && sound_slots > 1).then(|| track.saturating_sub(1).min(file_channels - 1));
// timecode of the physical source
let tc = self.first_clip(seg).and_then(|clip| {
let first = self.first_clip(seg).ok().flatten();
let tc = first.and_then(|clip| {
let tid = self.uid(clip, "OMFI:SCLP:SourceID")?;
let tape = *self.mobs.get(&tid)?;
let (start_tc, fps, trate) = self.refs(tape, "OMFI:MOBJ:Slots").into_iter().find_map(|s| {
@@ -415,7 +548,7 @@ impl R<'_, '_> {
nested: None,
};
if let Some(d) = self.media.get(&id) {
if let Some((pcm, ch, sr, b)) = crate::pcm::parse_wav(d).or_else(|| crate::pcm::parse_aiff(d)) {
if let Some((pcm, ch, sr, b)) = craft_audio::essence_pcm::parse_wav(d).or_else(|| craft_audio::essence_pcm::parse_aiff(d)) {
s.embedded = Some(pcm);
s.file_channels = ch as u32;
s.channels = if s.channel.is_some() { 1 } else { ch as u32 };
@@ -437,3 +570,200 @@ impl R<'_, '_> {
}
}
}
#[cfg(test)]
mod tests {
use super::super::bento::Writer;
use super::*;
fn obj(w: &mut Writer, class: &[u8; 4]) -> u32 {
let id = w.new_object();
w.set(id, "OMFI:OOBJ:ObjClass", "omfi:ClassID", class);
id
}
fn list(w: &mut Writer, o: u32, p: &str, targets: &[u32]) {
let mut v = (targets.len() as u16).to_be_bytes().to_vec();
for t in targets {
v.extend_from_slice(&t.to_be_bytes());
}
w.set(o, p, "omfi:ObjRefArray", &v);
}
fn search(max_visits: usize, max_pending: usize, build: impl FnOnce(&mut Writer) -> u32) -> std::result::Result<Option<u32>, SearchLimit> {
let mut w = Writer::new();
let root = build(&mut w);
let bytes = w.finish();
let c = Container::open(&bytes).unwrap();
let mut report = Report::default();
let r = R {
c: &c,
report: &mut report,
be: true,
mobs: HashMap::new(),
media: HashMap::new(),
doc: Document::default(),
sources: HashMap::new(),
budget: 0,
clip_search: HashMap::new(),
search_budget: MAX_DOCUMENT_SEARCH_WORK,
};
r.first_clip_within(root, max_visits, max_pending, &mut 0)
}
#[test]
fn clip_search_reports_its_limits() {
// found within the limits
assert!(matches!(
search(10, 10, |w| {
let (s, c) = (obj(w, b"SEQU"), obj(w, b"SCLP"));
list(w, s, "OMFI:SEQU:Components", &[c]);
s
}),
Ok(Some(_))
));
// too many distinct components before any clip
assert!(
search(5, 1000, |w| {
let (s, f) = (obj(w, b"SEQU"), (0..20).map(|_| obj(w, b"FILL")).collect::<Vec<_>>());
list(w, s, "OMFI:SEQU:Components", &f);
s
})
.is_err()
);
// few visits, but sequences that each queue a long list of references
assert!(
search(1000, 500, |w| {
let f = obj(w, b"FILL");
let inner: Vec<u32> = (0..20).map(|_| obj(w, b"SEQU")).collect();
for &i in &inner {
list(w, i, "OMFI:SEQU:Components", &[f; 100]);
}
let s = obj(w, b"SEQU");
list(w, s, "OMFI:SEQU:Components", &inner);
s
})
.is_err()
);
// a search that ends without a clip is not a limit
assert!(matches!(search(10, 10, |w| obj(w, b"FILL")), Ok(None)));
}
#[test]
fn skipped_slots_spend_the_budget() {
let mut w = Writer::new();
let head = obj(&mut w, b"HEAD");
let cm = obj(&mut w, b"CMOB");
let slot = obj(&mut w, b"MSLT");
list(&mut w, cm, "OMFI:MOBJ:Slots", &[slot; 200]);
list(&mut w, head, "OMFI:HEAD:Mobs", &[cm]);
list(&mut w, head, "OMFI:HEAD:PrimaryMobs", &[cm]);
let bytes = w.finish();
let mut report = Report::default();
read_with_budget(&bytes, &mut report, 50).unwrap();
assert!(report.mentions(BUDGET_WARNING), "slots with no segment cost nothing before");
}
#[test]
fn a_source_clip_search_is_made_once_per_segment() {
let mut w = Writer::new();
let (s, c) = (obj(&mut w, b"SEQU"), obj(&mut w, b"SCLP"));
list(&mut w, s, "OMFI:SEQU:Components", &[c]);
let bytes = w.finish();
let c = Container::open(&bytes).unwrap();
let mut report = Report::default();
let mut r = R {
c: &c,
report: &mut report,
be: true,
mobs: HashMap::new(),
media: HashMap::new(),
doc: Document::default(),
sources: HashMap::new(),
budget: 0,
clip_search: HashMap::new(),
search_budget: MAX_DOCUMENT_SEARCH_WORK,
};
let first = r.first_clip(s).ok();
assert!(matches!(first, Some(Some(_))));
assert_eq!(r.clip_search.len(), 1);
assert_eq!(r.first_clip(s).ok(), first);
assert_eq!(r.clip_search.len(), 1);
}
#[test]
fn repeated_slots_and_compositions_spend_the_budget() {
let mut w = Writer::new();
let head = obj(&mut w, b"HEAD");
let cm = obj(&mut w, b"CMOB");
let dd = obj(&mut w, b"DDEF");
w.set(dd, "OMFI:DDEF:DataKindID", "omfi:UniqueName", b"omfi:data:Sound\0");
let fill = obj(&mut w, b"FILL");
w.set(fill, "OMFI:CPNT:DataKind", "omfi:ObjRef", &dd.to_be_bytes());
let slot = obj(&mut w, b"MSLT");
w.set(slot, "OMFI:MSLT:Segment", "omfi:ObjRef", &fill.to_be_bytes());
list(&mut w, cm, "OMFI:MOBJ:Slots", &[slot; 200]);
list(&mut w, head, "OMFI:HEAD:Mobs", &[cm]);
list(&mut w, head, "OMFI:HEAD:PrimaryMobs", &[cm; 200]);
let bytes = w.finish();
let mut report = Report::default();
let doc = read_with_budget(&bytes, &mut report, 50).unwrap();
let tracks: usize = doc.compositions.iter().map(|c| c.tracks.len()).sum();
assert!(tracks <= 50, "{tracks} tracks from a budget of 50");
assert!(doc.compositions.len() <= 2, "{} compositions", doc.compositions.len());
assert!(report.has_warnings());
}
fn with_reader<T>(bytes: &[u8], budget: usize, search_budget: usize, f: impl FnOnce(&mut R) -> T) -> (T, Report) {
let c = Container::open(bytes).unwrap();
let mut report = Report::default();
let mut r = R {
c: &c,
report: &mut report,
be: true,
mobs: HashMap::new(),
media: HashMap::new(),
doc: Document::default(),
sources: HashMap::new(),
budget,
clip_search: HashMap::new(),
search_budget,
};
let out = f(&mut r);
(out, report)
}
#[test]
fn depth_limited_components_spend_the_budget() {
// a chain 33 sequences deep whose last one repeats a depth-limited sequence many times
let mut w = Writer::new();
let deep = obj(&mut w, b"SEQU");
let chain: Vec<u32> = (0..33).map(|_| obj(&mut w, b"SEQU")).collect();
for pair in chain.windows(2) {
list(&mut w, pair[0], "OMFI:SEQU:Components", &[pair[1]]);
}
list(&mut w, chain[32], "OMFI:SEQU:Components", &[deep; 1000]);
let bytes = w.finish();
let (items, report) = with_reader(&bytes, 100, MAX_DOCUMENT_SEARCH_WORK, |r| r.segment(chain[0], (48_000, 1), 0));
assert!(items.len() <= 100, "{} items from a budget of 100", items.len());
assert!(report.mentions(BUDGET_WARNING));
}
#[test]
fn source_clip_searches_share_a_document_budget() {
// distinct roots over one shared clip-free graph
let mut w = Writer::new();
let fills: Vec<u32> = (0..50).map(|_| obj(&mut w, b"FILL")).collect();
let shared = obj(&mut w, b"SEQU");
list(&mut w, shared, "OMFI:SEQU:Components", &fills);
let roots: Vec<u32> = (0..30).map(|_| obj(&mut w, b"SEQU")).collect();
for &root in &roots {
list(&mut w, root, "OMFI:SEQU:Components", &[shared]);
}
let bytes = w.finish();
let (results, _) = with_reader(&bytes, 0, 500, |r| roots.iter().map(|&root| r.first_clip(root)).collect::<Vec<_>>());
assert!(matches!(results[0], Ok(None)), "the first search fits");
assert!(results.iter().any(|r| r.is_err()), "later searches run out of the shared budget");
let (results, _) = with_reader(&bytes, 0, MAX_DOCUMENT_SEARCH_WORK, |r| roots.iter().map(|&root| r.first_clip(root)).collect::<Vec<_>>());
assert!(results.iter().all(|r| matches!(r, Ok(None))));
}
}
-5
View File
@@ -4,8 +4,3 @@
pub(crate) fn parse_wav(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
craft_audio::wav_pcm(b).map(|p| (p.data, p.channels, p.sample_rate, p.bits))
}
/// The same for an AIFF / AIFF-C (`NONE`, `twos`, `sowt`) file.
pub(crate) fn parse_aiff(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
craft_audio::aiff_pcm(b).map(|p| (p.data, p.channels, p.sample_rate, p.bits))
}
+27
View File
@@ -0,0 +1,27 @@
use craft_aaf::omf::bento::{Container, Writer, sniff};
#[test]
fn external_consumer_reads_named_values_and_references() {
let mut writer = Writer::new();
let first = writer.new_object();
let second = writer.new_object();
writer.set(first, "test:value", "test:type", b"long value");
writer.set_references(first, &[second]);
let bytes = writer.finish();
assert!(sniff(&bytes));
let container = Container::open(&bytes).unwrap();
assert!(container.object_ids().any(|id| id == first));
assert_eq!(container.get(first, "test:value").as_deref(), Some(b"long value".as_slice()));
assert_eq!(container.resolve(first, second), second);
assert!(Container::open(&bytes[..bytes.len() - 1]).is_err());
}
#[test]
fn default_writer_builds_a_readable_empty_container() -> Result<(), String> {
let bytes = Writer::default().finish();
let container = Container::open(&bytes)?;
let ids: Vec<_> = container.object_ids().collect();
assert!(!ids.is_empty());
assert!(ids.windows(2).all(|pair| pair[0] < pair[1]));
Ok(())
}
+10 -6
View File
@@ -1,6 +1,6 @@
[package]
name = "craft-audio"
description = "Audio file I/O and codecs: WAV/AIFF read and write, FLAC encoder, AAC and ALAC decoders and encoders, AC-3/E-AC-3 decoder and encoder, MP3 decoder and encoder, Opus decoder and encoder, MP1/MP2 decoders."
description = "Audio file I/O and codecs: WAV/AIFF read and write, FLAC encoder, AAC and ALAC decoders and encoders, AC-3/E-AC-3 decoder and encoder, MP3 decoder and encoder, Opus decoder and encoder, Vorbis encoder, MP1/MP2 decoders."
version.workspace = true
edition.workspace = true
license.workspace = true
@@ -29,10 +29,13 @@ encode = ["lossless-encode", "aac-encode", "alac-encode", "ac3-encode", "eac3-en
lossless-encode = ["pcm-encode", "flac-encode"]
# Encode WAV/BWF/RF64 and AIFF/AIFF-C files.
pcm-encode = []
# Decode FLAC files and frames natively (`flac::decode`, `flac::Decoder`).
flac-decode = []
# Encode FLAC files.
flac-encode = ["craft-media/bitstream"]
# Encode Ogg Vorbis files (desktop only: builds the reference libvorbis from its C sources).
vorbis-encode = ["dep:vorbis_rs"]
# The pure-Rust Vorbis encoder (`vorbis::encoder`) and Ogg Vorbis files through `encode` and
# `encode_ogg_vorbis`.
vorbis-encode = ["craft-media/ogg"]
# Codec modules on their own, for packet-level use.
aac-decode = ["craft-media/bitstream"]
# The AAC encoder (`aac::Encoder`), and AAC in M4A through `encode` and `FileEncoder`.
@@ -72,9 +75,6 @@ rayon = { workspace = true, optional = true }
# symphonia keeps its AAC decoder for AAC in CAF, a container craft-media does not read.
symphonia = { version = "0.6.1", optional = true, default-features = false, features = ["mp3", "flac", "vorbis", "ogg", "wav", "pcm", "aiff", "isomp4", "aac", "alac", "adpcm", "caf", "mkv"] }
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
vorbis_rs = { version = "0.5", optional = true }
[dev-dependencies]
craft-media-testkit = { workspace = true }
proptest = { workspace = true }
@@ -88,6 +88,10 @@ required-features = ["decode", "lossless-encode", "peaks"]
name = "codecs"
required-features = ["decode", "encode"]
[[test]]
name = "flac_decode"
required-features = ["flac-decode", "flac-encode"]
[[test]]
name = "qoa_oracle"
required-features = ["qoa"]
+31 -3
View File
@@ -29,14 +29,16 @@ No default features. The shared types (`AudioBuffer`, `AudioInfo`, `ChannelLayou
| `decode` | `decode`, `probe`, `Decoder`, `tracks`, `TrackDecoder`; every format in [Format support](#format-support) | `decode-native`, `symphonia` |
| `decode-native` | the same, native formats only (others are `Unsupported`); `wav_pcm`, `aiff_pcm` (a file's integer PCM bit for bit, as `RawPcm`) | the six decoders below, craft-media `isobmff`/`matroska`/`ogg`, `log` |
| `symphonia` | the fallback for formats with no native decoder; turning it on is the same as `decode` | `decode-native`, symphonia |
| `encode` | every encoder, and every format of `encode` and `FileEncoder` | `lossless-encode`, `aac-encode`, `alac-encode`, `ac3-encode`, `eac3-encode`, `mp3-encode`, `opus-encode` |
| `encode` | every encoder, and every format of `encode` and `FileEncoder` | `lossless-encode`, `aac-encode`, `alac-encode`, `ac3-encode`, `eac3-encode`, `mp3-encode`, `opus-encode`, `vorbis-encode` |
| `lossless-encode` | WAV, AIFF and FLAC writing | `pcm-encode`, `flac-encode` |
| `pcm-encode` | `encode`, `FileEncoder` for WAV and AIFF; `wav_float_header`; `pcm_wav_file`, `pcm_wav_header`, `pcm_aiff_file`, `pcm_aiff_header` (minimal files around integer PCM bytes, for essence embedded in AAF / OMF) | none |
| `flac-decode` | `flac::decode` (whole `.flac` files) and `flac::Decoder` (single frames, for callers that demux FLAC themselves); native, CRC-checked, 4 to 32 bits. `decode` still uses symphonia for FLAC files | none |
| `flac-encode` | `encode`, `FileEncoder` for FLAC; `flac::Encoder`, the packet encoder for callers that mux FLAC themselves | craft-media `bitstream` |
| `aac-decode`, `aac-encode` | `aac::Decoder`, `aac::Encoder`; with `aac-decode`, `aac::latm_config` / `latm_split` for LOAS / LATM frames (AAC in MPEG transport streams); with `aac-encode`, `encode`, `FileEncoder` for AAC in M4A | craft-media `bitstream`; `aac-encode` also `isobmff` |
| `alac-decode` | `alac::Decoder` | none |
| `alac-encode` | `alac::Encoder`; `encode`, `FileEncoder` for ALAC in M4A | craft-media `isobmff` |
| `opus-decode`, `opus-encode` | `opus::Decoder`/`StreamDecoder`, `opus::Encoder` | none |
| `vorbis-encode` | `vorbis::encoder::Encoder` (pure Rust); `encode_ogg_vorbis`, `encode_ogg` and `encode` for Ogg Vorbis files (whole buffer; not `FileEncoder`) | craft-media `ogg` |
| `ac3-decode` | `ac3::Decoder`, `ac3::ProgrammeDecoder` | craft-media `bitstream` |
| `ac3-encode`, `eac3-encode` | `ac3::encoder::Encoder`; `encode`, `FileEncoder` for AC-3 and E-AC-3 streams | craft-media `bitstream` |
| `mpa-decode` | `mpa::Decoder` | craft-media `bitstream` |
@@ -355,8 +357,12 @@ let pre_skip = enc.pre_skip(); // samples the player drops at the start
The encoder takes 48 kHz only, and craft-audio has no resampler.
craft-media has no Ogg muxer, so `.opus` files cannot be written yet; Opus goes into
Matroska/WebM or MP4:
Opus goes into Ogg, Matroska/WebM or MP4:
- **Ogg (`.opus`):** `craft_media::ogg::PageWriter` with `head` and `opus_tags` as the two header
packets, each packet's granule position the samples decoded so far (pre-skip included, as
the input above starts with it), and `finish(Some(pre_skip + length))` to trim the end
(RFC 7845 §4).
- **Matroska/WebM:** `head` is the track's `CodecPrivate`, and the pre-skip goes in
`TrackSpec::codec_delay_ns` (in nanoseconds at 48 kHz). The Matroska muxer cannot write
@@ -368,6 +374,27 @@ Matroska/WebM or MP4:
timescale), `segment_duration` the audio's length (movie timescale), `media_rate`
`0x0001_0000`.
### Encode Ogg Vorbis
```rust
use craft_audio::{OggVorbisOptions, encode_ogg_vorbis};
// 1–8 channels in Vorbis order (5.1: L C R Ls Rs LFE), any sample rate.
let opts = OggVorbisOptions::new(6.0).with_tag("TITLE", "Take 3").with_tag("ARTIST", "Me");
let ogg: Vec<u8> = encode_ogg_vorbis(&buf, &opts)?;
```
The encoder is pure Rust (`vorbis::encoder`, from FilmCraft #4481) and builds for wasm32. Quality
runs from -1 to 10 at a variable bit rate: about 18 kbit/s per channel at -1, 100 at 6 and 140
at 10 on music. `encode` with `EncodeFormat::Ogg` takes a WAV-order layout and writes it in
Vorbis order; its `RateControl::Bitrate` (64–500 kbit/s) picks the quality whose nominal rate
that is (`quality_for_bitrate`), and `RateControl::Quality` 0.0–1.0 is quality -1 to 10. For
streaming, feed `vorbis::encoder::Encoder` and write its packets with `craft_media::ogg::PageWriter`
(identification, `vorbis_comment`, setup, then the audio with each packet's granule position).
Not yet: short blocks (transients may pre-echo), channel coupling (stereo costs
about twice mono) and bit-rate management.
### Waveform overviews
```rust
@@ -704,6 +731,7 @@ the full tests with ffmpeg and the vectors required.
| MP3 encoder | FilmCraft `filmcraft-mp3enc` [4bec22b](https://github.com/storytold/filmcraft/commit/4bec22bd03bd1d995c9cf72e02a284f96d3d72cd) |
| ALAC | Apple's reference implementation [macosforge/alac@c38887c](https://github.com/macosforge/alac/tree/c38887c5c5e64a4b31108733bd79ca9b2496d987/codec), Copyright © 2011 Apple Inc., Apache-2.0; ported to Rust and modified (see the `alac` module docs and [NOTICE](../../NOTICE)) |
| Opus encoder | EffectCraft `effectcraft-opusenc` [c618429](https://github.com/storytold/effectcraft/commit/c618429252587525a2a6d4b5bccbe4ec5261e1b9) |
| Vorbis encoder | FilmCraft `filmcraft-vorbis` by @JordyUSA ([storytold/filmcraft#4481](https://github.com/storytold/filmcraft/pull/4481)), with partitions of whole sample frames so libvorbis reads surround |
Where apps had overlapping implementations, the one that measured better was kept, with the
tests of both:
+54
View File
@@ -343,6 +343,51 @@ impl AudioSpecificConfig {
}
}
/// Why an AudioSpecificConfig prefix cannot be written as an ADTS header.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AdtsConfigError {
/// Fewer than two bytes were supplied.
Short,
/// The prefix names a profile, indexed rate or channel configuration ADTS cannot carry.
Unsupported {
/// Audio object type from the prefix.
object_type: u8,
/// Sampling frequency index from the prefix.
sf_index: u8,
/// Channel configuration from the prefix.
channel_config: u8,
},
}
/// ADTS fixed fields taken from the prefix of an AudioSpecificConfig.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct AdtsConfig {
/// Audio object type, 1 through 4.
pub object_type: u8,
/// Indexed sample rate, 0 through 12.
pub sf_index: u8,
/// Channel configuration, 1 through 7.
pub channel_config: u8,
}
impl AdtsConfig {
/// Parse only the first two ASC bytes. Unlike a decoder's full ASC parser, this accepts all
/// four object types representable by ADTS and rejects configurations needing a PCE.
pub fn from_asc_prefix(asc: &[u8]) -> std::result::Result<Self, AdtsConfigError> {
let (Some(&a), Some(&b)) = (asc.first(), asc.get(1)) else { return Err(AdtsConfigError::Short) };
let object_type = a >> 3;
let sf_index = ((a & 7) << 1) | (b >> 7);
let channel_config = (b >> 3) & 15;
if !(1..=4).contains(&object_type) || sf_index > 12 || !(1..=7).contains(&channel_config) {
return Err(AdtsConfigError::Unsupported { object_type, sf_index, channel_config });
}
Ok(Self { object_type, sf_index, channel_config })
}
/// Write a no-CRC ADTS header with variable-rate buffer fullness for one payload.
pub fn header(self, payload_len: usize) -> Result<[u8; 7]> {
AdtsHeader::write(self.object_type, self.sf_index, self.channel_config, payload_len, 0x7ff)
}
}
/// A parsed ADTS header.
#[non_exhaustive]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -519,6 +564,15 @@ mod tests {
assert_eq!(AudioSpecificConfig::parse(&asc.to_bytes().unwrap()).unwrap(), asc);
}
#[test]
fn adts_config_preserves_prefix_and_frame_limits() {
let config = AdtsConfig::from_asc_prefix(&[0x12, 0x10]).unwrap();
assert_eq!(config.header(100).unwrap(), AdtsHeader::write(2, 4, 2, 100, 0x7ff).unwrap());
assert!(config.header(8185).is_err());
assert_eq!(AdtsConfig::from_asc_prefix(&[]), Err(AdtsConfigError::Short));
assert!(matches!(AdtsConfig::from_asc_prefix(&[0x2a, 0x10]), Err(AdtsConfigError::Unsupported { object_type: 5, .. })));
}
#[test]
fn adts_headers_refuse_fields_they_cannot_hold() {
for (object_type, sf_index, channel_config) in [(0, 4, 2), (5, 4, 2), (29, 4, 2), (2, 13, 2), (2, 15, 2), (2, 4, 8), (2, 4, 11), (2, 4, 12)] {
+1 -1
View File
@@ -60,7 +60,7 @@ mod tns;
#[cfg(feature = "aac-decode")]
pub(crate) use config::split_adts_upto;
pub use config::{AdtsHeader, AudioSpecificConfig, ElementType, ProgramConfig, config_layout, split_adts};
pub use config::{AdtsConfig, AdtsConfigError, AdtsHeader, AudioSpecificConfig, ElementType, ProgramConfig, config_layout, split_adts};
#[cfg(feature = "aac-decode")]
pub use decoder::Decoder;
#[cfg(feature = "aac-encode")]
+189
View File
@@ -0,0 +1,189 @@
//! Compatibility WAVE and AIFF wrappers for embedded PCM essence in interchange files.
//! PCM inside the model is interleaved little-endian signed integers (8-bit unsigned for WAVE).
pub use crate::{pcm_aiff_header as aiff_header, pcm_wav_file as wav_file, pcm_wav_header as wav_header};
/// PCM, channels, sample rate and bits of a WAVE file (integer PCM only).
pub fn parse_wav(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
if b.len() < 12 || &b[..4] != b"RIFF" || &b[8..12] != b"WAVE" {
return None;
}
let mut at = 12;
let mut fmt = None;
while at + 8 <= b.len() {
let id = &b[at..at + 4];
let len = u32::from_le_bytes(b[at + 4..at + 8].try_into().ok()?) as usize;
let body = &b[at + 8..(at + 8).saturating_add(len).min(b.len())];
if id == b"fmt " && body.len() >= 16 {
let tag = u16::from_le_bytes([body[0], body[1]]);
let ch = u16::from_le_bytes([body[2], body[3]]);
let sr = u32::from_le_bytes(body[4..8].try_into().ok()?);
let bits = u16::from_le_bytes([body[14], body[15]]);
if tag != 1 && tag != 0xFFFE {
return None;
}
if tag == 0xFFFE {
// The first two bytes of the sub-format GUID carry the real format code.
let sub = body.get(24..26)?;
if u16::from_le_bytes([sub[0], sub[1]]) != 1 {
return None;
}
}
fmt = Some((ch, sr, bits));
} else if id == b"data" {
let (ch, sr, bits) = fmt?;
return Some((body.to_vec(), ch, sr, bits));
}
at = at.checked_add(8usize.checked_add(len)?.checked_add(len & 1)?)?;
}
None
}
/// A complete AIFF file (big-endian PCM) from little-endian PCM.
pub fn aiff_file(pcm: &[u8], channels: u16, sample_rate: u32, bits: u16) -> Vec<u8> {
let bps = bits.div_ceil(8) as usize;
let frames = pcm.len() / (bps * channels.max(1) as usize).max(1);
let mut v = aiff_header(frames as u32, channels, sample_rate, bits);
for s in pcm.chunks_exact(bps.max(1)) {
if bps == 1 {
v.push(s[0] ^ 0x80); // WAVE 8-bit is unsigned, AIFF signed
} else {
v.extend(s.iter().rev());
}
}
if (v.len() - 8) & 1 == 1 {
v.push(0);
}
v
}
/// PCM (converted to little-endian), channels, sample rate and bits of an AIFF / AIFF-C (`NONE`
/// / `twos` / `sowt`) file.
pub fn parse_aiff(b: &[u8]) -> Option<(Vec<u8>, u16, u32, u16)> {
if b.len() < 12 || &b[..4] != b"FORM" || !(&b[8..12] == b"AIFF" || &b[8..12] == b"AIFC") {
return None;
}
let aifc = &b[8..12] == b"AIFC";
let mut at = 12;
let mut comm = None;
while at + 8 <= b.len() {
let id = &b[at..at + 4];
let len = u32::from_be_bytes(b[at + 4..at + 8].try_into().ok()?) as usize;
let body = &b[at + 8..(at + 8).saturating_add(len).min(b.len())];
if id == b"COMM" && body.len() >= 18 {
let ch = u16::from_be_bytes([body[0], body[1]]);
let bits = u16::from_be_bytes([body[6], body[7]]);
let sr = from_extended(body[8..18].try_into().ok()?).round() as u32;
// AIFF has no compression code (trailing COMM bytes mean nothing); AIFF-C must name one
// of the integer PCM codes, or the samples are not integer PCM at all
let little = if aifc {
match body.get(18..22)? {
b"NONE" | b"twos" => false,
b"sowt" => true,
_ => return None,
}
} else {
false
};
comm = Some((ch, sr, bits, little));
} else if id == b"SSND" && body.len() >= 8 {
let (ch, sr, bits, little) = comm?;
let off = u32::from_be_bytes(body[..4].try_into().ok()?) as usize;
let data = body.get(8usize.checked_add(off)?..)?;
let bps = bits.div_ceil(8) as usize;
let mut pcm = Vec::with_capacity(data.len());
for s in data.chunks_exact(bps.max(1)) {
if bps == 1 {
pcm.push(s[0] ^ 0x80);
} else if little {
pcm.extend_from_slice(s);
} else {
pcm.extend(s.iter().rev());
}
}
return Some((pcm, ch, sr, bits));
}
at = at.checked_add(8usize.checked_add(len)?.checked_add(len & 1)?)?;
}
None
}
fn from_extended(b: [u8; 10]) -> f64 {
let e = (u16::from_be_bytes([b[0], b[1]]) & 0x7FFF) as i32;
let m = u64::from_be_bytes(b[2..10].try_into().unwrap_or([0; 8]));
if e == 0 && m == 0 {
return 0.0;
}
m as f64 / (1u64 << 63) as f64 * 2f64.powi(e - 16383)
}
#[cfg(test)]
mod tests {
/// A mutated EDL whose clip ran for days produced an OMF descriptor for more than 4 GiB of
/// PCM: the RIFF size field overflowed ("attempt to add with overflow").
#[test]
fn huge_wav_header_saturates() {
let h = super::wav_header(u32::MAX, 2, u32::MAX, 24);
assert_eq!(&h[4..8], &u32::MAX.to_le_bytes());
}
use super::*;
#[test]
fn accepts_truncated_essence_summaries() {
let mut wav = wav_file(&[1, 2, 3, 4], 1, 48_000, 16);
wav.truncate(46);
assert_eq!(parse_wav(&wav), Some((vec![1, 2], 1, 48_000, 16)));
let mut aiff = aiff_file(&[1, 2, 3, 4], 1, 48_000, 16);
aiff.truncate(56);
assert_eq!(parse_aiff(&aiff), Some((vec![1, 2], 1, 48_000, 16)));
}
#[test]
fn wav_and_aiff_round_trip() {
let pcm: Vec<u8> = (0..48u8).collect(); // 8 stereo 24-bit frames
let w = wav_file(&pcm, 2, 48_000, 24);
assert_eq!(parse_wav(&w), Some((pcm.clone(), 2, 48_000, 24)));
let a = aiff_file(&pcm, 2, 44_100, 24);
assert_eq!(parse_aiff(&a), Some((pcm.clone(), 2, 44_100, 24)));
let p16: Vec<u8> = (0..32u8).collect();
assert_eq!(parse_aiff(&aiff_file(&p16, 1, 96_000, 16)), Some((p16, 1, 96_000, 16)));
assert!(parse_wav(b"RIFF").is_none());
assert!(parse_aiff(&a[..30]).is_none());
for cut in 0..w.len() {
let _ = parse_wav(&w[..cut]);
let _ = parse_aiff(&a[..cut.min(a.len())]);
}
}
#[test]
fn aifc_needs_an_integer_pcm_compression_code() {
let file = |kind: &[u8; 4], code: &[u8]| {
let mut comm = vec![0, 1, 0, 0, 0, 1, 0, 16];
comm.extend_from_slice(&[0x40, 0x0E, 0xBB, 0x80, 0, 0, 0, 0, 0, 0]);
comm.extend_from_slice(code);
let mut body = kind.to_vec();
body.extend_from_slice(b"COMM");
body.extend_from_slice(&(comm.len() as u32).to_be_bytes());
body.extend_from_slice(&comm);
if comm.len() & 1 == 1 {
body.push(0);
}
body.extend_from_slice(b"SSND");
body.extend_from_slice(&10u32.to_be_bytes());
body.extend_from_slice(&[0; 8]);
body.extend_from_slice(&[0x12, 0x34]);
let mut v = b"FORM".to_vec();
v.extend_from_slice(&(body.len() as u32).to_be_bytes());
v.extend_from_slice(&body);
v
};
assert_eq!(parse_aiff(&file(b"AIFC", b"NONE")).map(|p| p.0), Some(vec![0x34, 0x12]));
assert_eq!(parse_aiff(&file(b"AIFC", b"twos")).map(|p| p.0), Some(vec![0x34, 0x12]));
assert_eq!(parse_aiff(&file(b"AIFC", b"sowt")).map(|p| p.0), Some(vec![0x12, 0x34]));
assert!(parse_aiff(&file(b"AIFC", b"fl32")).is_none(), "float samples are not integer PCM");
assert!(parse_aiff(&file(b"AIFC", b"")).is_none(), "AIFF-C without a compression code");
// plain AIFF ignores trailing COMM bytes
assert_eq!(parse_aiff(&file(b"AIFF", b"sowt")).map(|p| p.0), Some(vec![0x34, 0x12]));
}
}
+603
View File
@@ -0,0 +1,603 @@
//! A native FLAC decoder (RFC 9639): [`decode`] for whole `.flac` files and [`Decoder`] for single
//! frames (the packets of FLAC in MP4, Matroska or Ogg).
//!
//! Every frame is checked against its CRC-8 and CRC-16. Subframe types constant, verbatim, fixed
//! and LPC; Rice partitions with 4- and 5-bit parameters and escaped partitions; wasted bits;
//! left/side, side/right and mid/side stereo; fixed and variable block sizes; 4 to 32 bits per
//! sample. The STREAMINFO MD5 is not verified.
//!
//! Samples come back as `i32`, one `Vec` per channel, in the FLAC channel order.
/// What went wrong decoding FLAC.
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum DecodeError {
/// The data ends early.
#[error("truncated FLAC data")]
Truncated,
/// The data is not valid FLAC.
#[error("invalid FLAC data: {0}")]
Invalid(&'static str),
/// A frame or header failed its CRC.
#[error("FLAC CRC mismatch in {0}")]
Crc(&'static str),
/// Valid FLAC that is not decoded.
#[error("unsupported FLAC feature: {0}")]
Unsupported(&'static str),
/// The stream is longer than the sample limit.
#[error("FLAC stream exceeds the sample limit")]
TooLarge,
}
/// `Result` of this module.
pub type Result<T> = std::result::Result<T, DecodeError>;
/// The STREAMINFO metadata block (RFC 9639 section 8.2).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct StreamInfo {
/// Smallest block size in samples.
pub min_block_size: u16,
/// Largest block size in samples.
pub max_block_size: u16,
/// Smallest frame in bytes, 0 if unknown.
pub min_frame_size: u32,
/// Largest frame in bytes, 0 if unknown.
pub max_frame_size: u32,
/// Sample rate in Hz.
pub sample_rate: u32,
/// Channel count, 1 to 8.
pub channels: u8,
/// Bits per sample, 4 to 32.
pub bits_per_sample: u8,
/// Samples per channel in the stream, 0 if unknown.
pub total_samples: u64,
/// MD5 of the unencoded audio, all zero if not set.
pub md5: [u8; 16],
}
impl StreamInfo {
/// Parse the 34 bytes of a STREAMINFO block body.
pub fn parse(b: &[u8]) -> Result<StreamInfo> {
let b = b.get(..34).ok_or(DecodeError::Truncated)?;
let packed = u64::from_be_bytes([b[10], b[11], b[12], b[13], b[14], b[15], b[16], b[17]]);
let info = StreamInfo {
min_block_size: u16::from_be_bytes([b[0], b[1]]),
max_block_size: u16::from_be_bytes([b[2], b[3]]),
min_frame_size: u32::from_be_bytes([0, b[4], b[5], b[6]]),
max_frame_size: u32::from_be_bytes([0, b[7], b[8], b[9]]),
sample_rate: (packed >> 44) as u32,
channels: ((packed >> 41) & 7) as u8 + 1,
bits_per_sample: ((packed >> 36) & 31) as u8 + 1,
total_samples: packed & 0xF_FFFF_FFFF,
md5: b[18..34].try_into().map_err(|_| DecodeError::Truncated)?,
};
if info.bits_per_sample < 4 {
return Err(DecodeError::Invalid("bits per sample below 4"));
}
if info.min_block_size != 0
&& (info.min_block_size < 16 && info.min_block_size != info.max_block_size || info.max_block_size < info.min_block_size)
{
return Err(DecodeError::Invalid("inconsistent block sizes"));
}
Ok(info)
}
}
/// One decoded frame.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Frame {
/// Sample rate in Hz.
pub sample_rate: u32,
/// Bits per sample.
pub bits_per_sample: u8,
/// `true` when `number` is the first sample's index (variable block size), `false` when it is
/// the frame's index (fixed block size).
pub variable_block_size: bool,
/// The coded number of the frame header.
pub number: u64,
/// Samples per channel in the frame.
pub block_size: usize,
/// One `Vec` per channel, after undoing any stereo decorrelation.
pub channels: Vec<Vec<i32>>,
/// Bytes the frame took, including the header and CRC footer.
pub size: usize,
}
// ---- checksums ----
fn crc8(data: &[u8]) -> u8 {
let mut crc = 0u8;
for &b in data {
crc ^= b;
for _ in 0..8 {
crc = if crc & 0x80 != 0 { crc << 1 ^ 0x07 } else { crc << 1 };
}
}
crc
}
fn crc16(data: &[u8]) -> u16 {
let mut crc = 0u16;
for &b in data {
crc ^= u16::from(b) << 8;
for _ in 0..8 {
crc = if crc & 0x8000 != 0 { crc << 1 ^ 0x8005 } else { crc << 1 };
}
}
crc
}
// ---- bits ----
struct Bits<'a> {
data: &'a [u8],
/// Position in bits.
pos: usize,
}
impl<'a> Bits<'a> {
fn new(data: &'a [u8]) -> Self {
Bits { data, pos: 0 }
}
fn left(&self) -> usize {
self.data.len() * 8 - self.pos
}
fn bit(&mut self) -> Result<u32> {
let byte = *self.data.get(self.pos / 8).ok_or(DecodeError::Truncated)?;
let v = (byte >> (7 - self.pos % 8)) & 1;
self.pos += 1;
Ok(u32::from(v))
}
/// `n` bits (0 to 32) as an unsigned number.
fn bits(&mut self, n: u32) -> Result<u64> {
if n as usize > self.left() {
return Err(DecodeError::Truncated);
}
let mut v = 0u64;
let mut n = n as usize;
while n > 0 {
let byte = u64::from(self.data[self.pos / 8]);
let avail = 8 - self.pos % 8;
let take = avail.min(n);
v = v << take | (byte >> (avail - take)) & ((1 << take) - 1);
self.pos += take;
n -= take;
}
Ok(v)
}
/// `n` bits (0 to 33) as a two's complement number.
fn signed(&mut self, n: u32) -> Result<i64> {
if n == 0 {
return Ok(0);
}
let v = self.bits(n)? as i64;
Ok(if v >> (n - 1) & 1 == 1 { v - (1 << n) } else { v })
}
/// Zero bits up to the next one bit (which is consumed).
fn unary(&mut self) -> Result<u64> {
let mut n = 0u64;
loop {
// Skip whole zero bytes when aligned.
if self.pos.is_multiple_of(8) {
while self.data.get(self.pos / 8) == Some(&0) {
self.pos += 8;
n += 8;
}
}
if self.bit()? == 1 {
return Ok(n);
}
n += 1;
}
}
fn align(&mut self) {
self.pos = self.pos.div_ceil(8) * 8;
}
}
// ---- frame ----
const BLOCK_SIZES: [usize; 16] = [0, 192, 576, 1152, 2304, 4608, 0, 0, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768];
/// A frame decoder. `info` supplies what a frame header leaves to the STREAMINFO.
#[derive(Debug, Clone, Copy, Default)]
pub struct Decoder {
info: Option<StreamInfo>,
}
impl Decoder {
/// A decoder with the stream's STREAMINFO (needed for frames whose header defers the sample
/// rate or bit depth to it).
pub fn new(info: StreamInfo) -> Decoder {
Decoder { info: Some(info) }
}
/// A decoder without STREAMINFO: frames that defer to it are an error.
pub fn without_streaminfo() -> Decoder {
Decoder { info: None }
}
/// Decode the frame at the start of `data`.
pub fn decode_frame(&self, data: &[u8]) -> Result<Frame> {
// StreamInfo has public fields: container callers can construct it without
// parse(), so validate before it can control shifts and sample reads.
if self.info.is_some_and(|i| !(4..=32).contains(&i.bits_per_sample) || !(1..=8).contains(&i.channels) || i.sample_rate == 0) {
return Err(DecodeError::Invalid("invalid decoder STREAMINFO"));
}
let mut r = Bits::new(data);
if r.bits(15)? != 0x7FFC {
return Err(DecodeError::Invalid("missing frame sync code"));
}
let variable = r.bit()? == 1;
let (bs_code, sr_code) = (r.bits(4)? as usize, r.bits(4)? as usize);
let (ch_code, bps_code) = (r.bits(4)? as usize, r.bits(3)? as usize);
if r.bit()? != 0 {
return Err(DecodeError::Invalid("reserved bit set"));
}
let number = coded_number(&mut r)?;
let block_size = match bs_code {
0 => return Err(DecodeError::Invalid("reserved block size")),
6 => r.bits(8)? as usize + 1,
7 => r.bits(16)? as usize + 1,
c => BLOCK_SIZES[c],
};
let sample_rate = match sr_code {
0 => self.info.map(|i| i.sample_rate).ok_or(DecodeError::Invalid("sample rate only in STREAMINFO"))?,
1 => 88_200,
2 => 176_400,
3 => 192_000,
4 => 8_000,
5 => 16_000,
6 => 22_050,
7 => 24_000,
8 => 32_000,
9 => 44_100,
10 => 48_000,
11 => 96_000,
12 => r.bits(8)? as u32 * 1000,
13 => r.bits(16)? as u32,
14 => r.bits(16)? as u32 * 10,
_ => return Err(DecodeError::Invalid("forbidden sample rate code")),
};
let header_len = r.pos.div_ceil(8);
let stored_crc = r.bits(8)? as u8;
if crc8(&data[..header_len]) != stored_crc {
return Err(DecodeError::Crc("a frame header"));
}
let bits = match bps_code {
0 => self.info.map(|i| u32::from(i.bits_per_sample)).ok_or(DecodeError::Invalid("bit depth only in STREAMINFO"))?,
1 => 8,
2 => 12,
4 => 16,
5 => 20,
6 => 24,
7 => 32,
_ => return Err(DecodeError::Invalid("reserved bit depth")),
};
let (channels, mode) = match ch_code {
0..=7 => (ch_code + 1, Stereo::Independent),
8 => (2, Stereo::LeftSide),
9 => (2, Stereo::SideRight),
10 => (2, Stereo::MidSide),
_ => return Err(DecodeError::Invalid("reserved channel assignment")),
};
let mut out: Vec<Vec<i64>> = Vec::with_capacity(channels);
for c in 0..channels {
let extra = match (mode, c) {
(Stereo::LeftSide, 1) | (Stereo::SideRight, 0) | (Stereo::MidSide, 1) => 1,
_ => 0,
};
out.push(subframe(&mut r, block_size, bits + extra)?);
}
r.align();
let size = r.pos / 8;
let crc = r.bits(16)? as u16;
if crc16(&data[..size]) != crc {
return Err(DecodeError::Crc("a frame"));
}
let size = size + 2;
if mode != Stereo::Independent {
let (first, rest) = out.split_at_mut(1);
let (a, b) = (&mut first[0], &mut rest[0]);
for (x, y) in a.iter_mut().zip(b.iter_mut()) {
match mode {
Stereo::LeftSide => *y = x.wrapping_sub(*y),
Stereo::SideRight => *x = x.wrapping_add(*y),
_ => {
let m = x.wrapping_shl(1) | (*y & 1);
let side = *y;
*x = m.wrapping_add(side) >> 1;
*y = m.wrapping_sub(side) >> 1;
}
}
}
}
let mut chans = Vec::with_capacity(channels);
for c in out {
let v: Result<Vec<i32>> = c.into_iter().map(|s| checked_sample(s, bits)).collect();
chans.push(v?);
}
Ok(Frame { sample_rate, bits_per_sample: bits as u8, variable_block_size: variable, number, block_size, channels: chans, size })
}
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum Stereo {
Independent,
LeftSide,
SideRight,
MidSide,
}
/// The UTF-8-like coded number of a frame header (up to 36 bits, 7 bytes).
fn coded_number(r: &mut Bits) -> Result<u64> {
let first = r.bits(8)? as u8;
let extra = match first {
0x00..=0x7F => return Ok(u64::from(first)),
0xC0..=0xDF => 1,
0xE0..=0xEF => 2,
0xF0..=0xF7 => 3,
0xF8..=0xFB => 4,
0xFC..=0xFD => 5,
0xFE => 6,
_ => return Err(DecodeError::Invalid("bad coded number")),
};
let mut v = if extra == 6 { 0 } else { u64::from(first & (0x3F >> extra)) };
for _ in 0..extra {
let b = r.bits(8)?;
if b & 0xC0 != 0x80 {
return Err(DecodeError::Invalid("bad coded number"));
}
v = v << 6 | (b & 0x3F);
}
Ok(v)
}
/// One subframe: `bits` is the frame bit depth (plus 1 for a side channel).
fn subframe(r: &mut Bits, n: usize, bits: u32) -> Result<Vec<i64>> {
if r.bit()? != 0 {
return Err(DecodeError::Invalid("subframe header padding bit set"));
}
let kind = r.bits(6)? as u32;
let wasted = if r.bit()? == 1 { r.unary()? as u32 + 1 } else { 0 };
if wasted >= bits {
return Err(DecodeError::Invalid("wasted bits leave no sample bits"));
}
let bps = bits - wasted;
let mut s = vec![0i64; n];
match kind {
0 => {
let v = r.signed(bps)?;
s.fill(v);
}
1 => {
for v in s.iter_mut() {
*v = r.signed(bps)?;
}
}
8..=12 => {
let order = (kind - 8) as usize;
warmup(r, &mut s, order, bps)?;
residual(r, &mut s, order)?;
for i in order..n {
let at = |k: usize| s[i - k];
// Wrapping: hostile residuals can overflow; such samples fail the range check later.
let p = match order {
0 => 0,
1 => at(1),
2 => at(1).wrapping_mul(2).wrapping_sub(at(2)),
3 => at(1).wrapping_mul(3).wrapping_sub(at(2).wrapping_mul(3)).wrapping_add(at(3)),
_ => at(1).wrapping_mul(4).wrapping_sub(at(2).wrapping_mul(6)).wrapping_add(at(3).wrapping_mul(4)).wrapping_sub(at(4)),
};
s[i] = s[i].wrapping_add(p);
}
}
32..=63 => {
let order = (kind - 31) as usize;
warmup(r, &mut s, order, bps)?;
let precision = r.bits(4)? as u32 + 1;
if precision == 16 {
return Err(DecodeError::Invalid("forbidden coefficient precision"));
}
let shift = r.signed(5)?;
if shift < 0 {
return Err(DecodeError::Invalid("negative prediction shift"));
}
let mut coefs = [0i64; 32];
for c in coefs.iter_mut().take(order) {
*c = r.signed(precision)?;
}
residual(r, &mut s, order)?;
for i in order..n {
let mut sum = 0i64;
for (k, &c) in coefs.iter().take(order).enumerate() {
sum = sum.wrapping_add(c.wrapping_mul(s[i - 1 - k]));
}
s[i] = s[i].wrapping_add(sum >> shift);
}
}
_ => return Err(DecodeError::Invalid("reserved subframe type")),
}
if wasted > 0 {
for v in s.iter_mut() {
*v = v.wrapping_shl(wasted);
}
}
Ok(s)
}
fn warmup(r: &mut Bits, s: &mut [i64], order: usize, bps: u32) -> Result<()> {
if order > s.len() {
return Err(DecodeError::Invalid("predictor order above the block size"));
}
for v in s.iter_mut().take(order) {
*v = r.signed(bps)?;
}
Ok(())
}
/// The coded residual (RFC 9639 section 9.2.7), into `s[order..]`.
fn residual(r: &mut Bits, s: &mut [i64], order: usize) -> Result<()> {
let method = r.bits(2)?;
let (param_bits, escape) = match method {
0 => (4, 15),
1 => (5, 31),
_ => return Err(DecodeError::Invalid("reserved residual coding method")),
};
let part_order = r.bits(4)? as u32;
let n = s.len();
let parts = 1usize << part_order;
if !n.is_multiple_of(parts) {
return Err(DecodeError::Invalid("partition order does not divide the block size"));
}
let per = n >> part_order;
if per < order {
return Err(DecodeError::Invalid("partition shorter than the predictor order"));
}
let mut i = order;
for p in 0..parts {
let count = if p == 0 { per - order } else { per };
let k = r.bits(param_bits)? as u32;
if k == escape {
let width = r.bits(5)? as u32;
for v in s[i..i + count].iter_mut() {
*v = r.signed(width)?;
}
} else {
for v in s[i..i + count].iter_mut() {
let q = r.unary()?;
if q > (1 << 32) {
return Err(DecodeError::Invalid("Rice quotient too large"));
}
let low = r.bits(k)?;
let folded = q << k | low;
*v = if folded & 1 == 0 { (folded >> 1) as i64 } else { -((folded >> 1) as i64) - 1 };
}
}
i += count;
}
Ok(())
}
// ---- files ----
/// A decoded file.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Decoded {
/// The STREAMINFO block.
pub info: StreamInfo,
/// One `Vec` of samples per channel.
pub channels: Vec<Vec<i32>>,
}
/// Most sample values (samples x channels) [`decode`] will produce.
pub const DEFAULT_MAX_VALUES: u64 = 1 << 28;
/// Decode a whole `.flac` file (an ID3v2 tag in front is skipped).
pub fn decode(bytes: &[u8]) -> Result<Decoded> {
decode_with_limit(bytes, DEFAULT_MAX_VALUES)
}
/// [`decode`] with a limit on the sample values (samples x channels) produced.
pub fn decode_with_limit(bytes: &[u8], max_values: u64) -> Result<Decoded> {
let mut pos = 0usize;
if bytes.starts_with(b"ID3") {
let size = bytes.get(6..10).ok_or(DecodeError::Truncated)?;
if size.iter().any(|&b| b & 0x80 != 0) {
return Err(DecodeError::Invalid("bad ID3v2 size"));
}
pos = 10 + size.iter().fold(0usize, |a, &b| a << 7 | usize::from(b));
}
if bytes.get(pos..pos + 4) != Some(b"fLaC") {
return Err(DecodeError::Invalid("missing fLaC marker"));
}
pos += 4;
let mut info: Option<StreamInfo> = None;
loop {
let head = bytes.get(pos..pos + 4).ok_or(DecodeError::Truncated)?;
let last = head[0] & 0x80 != 0;
let kind = head[0] & 0x7F;
let len = usize::from(head[1]) << 16 | usize::from(head[2]) << 8 | usize::from(head[3]);
let body = bytes.get(pos + 4..pos + 4 + len).ok_or(DecodeError::Truncated)?;
match (kind, &info) {
(0, None) => info = Some(StreamInfo::parse(body)?),
(0, Some(_)) => return Err(DecodeError::Invalid("a second STREAMINFO block")),
(_, None) => return Err(DecodeError::Invalid("STREAMINFO is not the first block")),
(127, _) => return Err(DecodeError::Invalid("forbidden metadata block type")),
_ => {}
}
pos += 4 + len;
if last {
break;
}
}
let info = info.ok_or(DecodeError::Invalid("no STREAMINFO"))?;
let dec = Decoder::new(info);
let mut channels: Vec<Vec<i32>> = vec![Vec::new(); usize::from(info.channels)];
let mut total = 0u64;
while pos + 2 <= bytes.len() && bytes[pos] == 0xFF && bytes[pos + 1] & 0xFE == 0xF8 {
let frame = dec.decode_frame(&bytes[pos..])?;
if frame.channels.len() != channels.len() {
return Err(DecodeError::Invalid("the channel count changes inside the stream"));
}
// One file has one sample rate and depth. Otherwise callers would play
// samples under incompatible STREAMINFO metadata.
if frame.sample_rate != info.sample_rate || frame.bits_per_sample != info.bits_per_sample {
return Err(DecodeError::Invalid("frame format differs from STREAMINFO"));
}
total += frame.block_size as u64;
if total.saturating_mul(channels.len() as u64) > max_values {
return Err(DecodeError::TooLarge);
}
for (dst, src) in channels.iter_mut().zip(frame.channels) {
dst.extend_from_slice(&src);
}
pos += frame.size;
}
// Unknown total_samples must not turn a corrupt/truncated next frame into
// a successfully decoded shorter recording.
if pos != bytes.len() {
return Err(DecodeError::Invalid("trailing data or missing frame sync"));
}
if info.total_samples != 0 && total != info.total_samples {
return Err(DecodeError::Truncated);
}
Ok(Decoded { info, channels })
}
// Validate the advertised precision, not just the storage type. Malformed
// predictors may produce an i32 that is still invalid for 4–24-bit audio.
fn checked_sample(sample: i64, bits: u32) -> Result<i32> {
if !(4..=32).contains(&bits) {
return Err(DecodeError::Invalid("invalid sample bit depth"));
}
let bound = 1i64 << (bits - 1);
if !(-bound..bound).contains(&sample) {
return Err(DecodeError::Invalid("sample outside the bit depth"));
}
i32::try_from(sample).map_err(|_| DecodeError::Invalid("sample outside the bit depth"))
}
#[cfg(test)]
mod sample_bounds_tests {
use super::checked_sample;
#[test]
fn samples_must_fit_declared_precision() {
for bits in 4..=32 {
let bound = 1i64 << (bits - 1);
assert_eq!(checked_sample(-bound, bits).map(i64::from), Ok(-bound));
assert_eq!(checked_sample(bound - 1, bits).map(i64::from), Ok(bound - 1));
assert!(checked_sample(-bound - 1, bits).is_err());
assert!(checked_sample(bound, bits).is_err());
}
assert!(checked_sample(0, 0).is_err());
assert!(checked_sample(0, 255).is_err());
}
}
+45 -15
View File
@@ -13,6 +13,8 @@
//! demuxers and [`TrackDecoder`]; everything else (FLAC, Vorbis, CAF, ADPCM, A-law, µ-law) through
//! `symphonia`, which is to be replaced by native decoders. Feature `decode-native` is the
//! same without symphonia.
//! - [`encode_ogg_vorbis`] and [`vorbis::encoder`] (feature `vorbis-encode`): Ogg Vorbis files
//! and Vorbis packets from a pure-Rust encoder, 1 to 8 channels, quality -1 to 10, with tags.
//! - [`TrackDecoder`] (any decoder feature): one track's packets from any demuxer to planar
//! `f32` in WAV channel order.
//! - [`encode`], [`FileEncoder`]: WAV/BWF/RF64 and AIFF/AIFF-C (feature `pcm-encode`, which also
@@ -27,8 +29,8 @@
//! - Codec modules for packet-level use, each with its own features: [`aac`] (`aac-decode`,
//! `aac-encode`), [`alac`] (`alac-decode`, `alac-encode`), [`opus`] (`opus-decode`, and
//! [`opus::encoder`] with `opus-encode`), [`ac3`] (`ac3-decode`), [`mpa`] (`mpa-decode`), [`mp3`]
//! (`mp3-decode`, `mp3-encode`), [`flac`] (`flac-encode`: the packet encoder), [`qoa`]
//! (`qoa-decode`, `qoa-encode`: whole QOA files).
//! (`mp3-decode`, `mp3-encode`), [`flac`] (`flac-decode`: the frame and file decoder;
//! `flac-encode`: the packet encoder), [`qoa`] (`qoa-decode`, `qoa-encode`: whole QOA files).
//!
//! The shared types below build with no features.
@@ -56,8 +58,12 @@ pub mod qoa;
mod read;
#[cfg(any(feature = "aac-decode", feature = "alac-decode", feature = "opus-decode", feature = "ac3-decode", feature = "mpa-decode"))]
mod track;
#[cfg(feature = "vorbis-encode")]
pub mod vorbis;
#[cfg(any(feature = "decode-native", feature = "pcm-encode"))]
mod wav;
/// Bounded WAV/RF64 cue, adtl and XMP marker metadata, without audio decoding.
pub mod wav_markers;
#[cfg(any(
feature = "pcm-encode",
feature = "flac-encode",
@@ -69,8 +75,18 @@ mod wav;
feature = "vorbis-encode"
))]
mod write;
/// PCM WAVE and AIFF wrappers for interchange essence.
#[cfg(feature = "pcm-encode")]
pub mod essence_pcm;
/// Streaming WAVE and AIFF headers and sample packing.
pub mod pcm_export;
/// Interleaved PCM packet decoding.
pub mod pcm_interleaved;
/// Range-backed PCM WAVE access and a 16-bit PCM writer.
pub mod wav_range;
#[cfg(feature = "vorbis-encode")]
pub use write::{OGG_BITRATES, encode_ogg};
pub use write::{OGG_BITRATES, OggVorbisOptions, encode_ogg, encode_ogg_vorbis, quality_for_bitrate};
/// The FLAC packet encoder (RFC 9639), for a caller that muxes FLAC itself (into MP4, Matroska or
/// its own `.flac` writer): planar integer samples in, frames out, and the STREAMINFO to patch in
@@ -91,10 +107,15 @@ pub use write::{OGG_BITRATES, encode_ogg};
/// assert!(file.len() < 48_000 * 2 * 2 / 2); // well under half of 16-bit PCM
/// # Ok::<(), craft_audio::flac::Error>(())
/// ```
#[cfg(feature = "flac-encode")]
#[cfg(any(feature = "flac-encode", feature = "flac-decode"))]
pub mod flac {
#[cfg(feature = "flac-decode")]
pub use crate::flac_decode::{DEFAULT_MAX_VALUES, DecodeError, Decoded, Decoder, Frame, StreamInfo, decode, decode_with_limit};
#[cfg(feature = "flac-encode")]
pub use crate::write::flac::encoder::{Encoder, EncoderConfig, Error, STREAMINFO_OFFSET};
}
#[cfg(feature = "flac-decode")]
mod flac_decode;
pub use buffer::AudioBuffer;
/// The craft-media this crate is built with, for its containers: using it rather than a separate
@@ -273,7 +294,9 @@ pub enum EncodeFormat {
Ac3,
/// An E-AC-3 stream: up to 7.1 at 32, 44.1 or 48 kHz.
Eac3,
/// Ogg Vorbis, whole-buffer encoding only.
/// Ogg Vorbis, 1–8 channels at any sample rate, whole-buffer encoding only. Variable bit
/// rate: a [`RateControl::Bitrate`] picks the quality whose nominal rate it is
/// ([`quality_for_bitrate`]), [`RateControl::Quality`] sets it directly.
Ogg,
}
@@ -374,7 +397,7 @@ impl EncodeFormat {
EncodeFormat::Mp3 => cfg!(feature = "mp3-encode"),
EncodeFormat::Ac3 => cfg!(feature = "ac3-encode"),
EncodeFormat::Eac3 => cfg!(feature = "eac3-encode"),
EncodeFormat::Ogg => cfg!(all(feature = "vorbis-encode", not(target_arch = "wasm32"))),
EncodeFormat::Ogg => cfg!(feature = "vorbis-encode"),
}
}
@@ -395,9 +418,9 @@ impl EncodeFormat {
pub fn max_channels(self) -> u16 {
match self {
EncodeFormat::Wav | EncodeFormat::Aiff => 32_767,
EncodeFormat::Flac | EncodeFormat::Aac | EncodeFormat::Alac | EncodeFormat::Eac3 => 8,
EncodeFormat::Flac | EncodeFormat::Aac | EncodeFormat::Alac | EncodeFormat::Eac3 | EncodeFormat::Ogg => 8,
EncodeFormat::Ac3 => 6,
EncodeFormat::Mp3 | EncodeFormat::Ogg => 2,
EncodeFormat::Mp3 => 2,
}
}
@@ -411,7 +434,10 @@ impl EncodeFormat {
EncodeFormat::Aac => &[&[0x4], &[0x3], &[0x7], &[0x107], &[0x607], &[0x60F], &[0x70F], &[0xFF]],
// Apple's layout tags for 1–8 channels, surrounds as back speakers.
EncodeFormat::Alac => &[&[0x4], &[0x3], &[0x7], &[0x107], &[0x37], &[0x3F], &[0x13F], &[0xFF]],
EncodeFormat::Mp3 | EncodeFormat::Ogg => &[&[0x4], &[0x3]],
EncodeFormat::Mp3 => &[&[0x4], &[0x3]],
// The Vorbis I §4.3.9 layouts for 1–8 channels (6.1 with a back centre and sides, 7.1
// with sides and backs), written in Vorbis order.
EncodeFormat::Ogg => &[&[0x4], &[0x3], &[0x7], &[0x33], &[0x37], &[0x3F], &[0x70F], &[0x63F]],
// The coding modes 1/0 to 3/2 with and without an LFE, by channel count.
EncodeFormat::Ac3 => &[&[0x4], &[0x3, 0xC], &[0x7, 0x103, 0xB], &[0x107, 0x603, 0xF, 0x10B], &[0x607, 0x10F, 0x60B], &[0x60F]],
// The same, and 7.1 with backs and sides.
@@ -453,10 +479,10 @@ impl EncodeFormat {
}
}
/// Whether the format takes [`RateControl::Quality`]: AAC. The other lossy formats are
/// written at a constant bit rate only.
/// Whether the format takes [`RateControl::Quality`]: AAC and Ogg Vorbis. The other lossy
/// formats are written at a constant bit rate only.
pub fn supports_quality(self) -> bool {
self == EncodeFormat::Aac
matches!(self, EncodeFormat::Aac | EncodeFormat::Ogg)
}
/// Whether the format loses audio: AAC, MP3, AC-3 and E-AC-3. These take
@@ -475,11 +501,12 @@ pub enum RateControl {
/// set of rates (MP3, AC-3) uses the highest one not above it; a rate outside what the
/// stream can carry is moved to the nearest one inside. AAC is written at 6 kbit/s or a
/// quarter of the sample rate per channel at least: from about half of that down its encoder
/// codes silence. Vorbis treats this as a VBR target (64–500 kbit/s), falling back to
/// a quality preset when that target is unsupported; it does not promise a constant bitrate.
/// codes silence. Vorbis takes 64–500 kbit/s and writes the quality whose nominal rate that is
/// ([`quality_for_bitrate`]); its rate varies with the audio.
Bitrate(u32),
/// A constant quality from 0.0 (smallest) to 1.0 (best), at whatever bit rate each frame
/// needs. AAC only; values outside the range are clamped, and NaN is an error.
/// needs. AAC and Vorbis (0.0 is Vorbis quality -1, 1.0 quality 10); values outside the range
/// are clamped, and NaN is an error.
Quality(f32),
}
@@ -745,3 +772,6 @@ mod tests {
assert!(matches!(e, AudioError::Encode(_)), "{e}");
}
}
/// DVD and Blu-ray LPCM packet decoding.
pub mod lpcm;
+118
View File
@@ -0,0 +1,118 @@
//! DVD-Video and Blu-ray LPCM packet grammar and sample decoding.
/// Blu-ray / AVCHD LPCM packet header (4 bytes before the samples).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct BlurayLpcm {
/// Sample rate in hertz.
pub sample_rate: u32,
/// Number of decoded channels.
pub channels: usize,
/// Significant bits per sample.
pub bits: u32,
}
/// Channels of each channel_assignment code (mono, stereo, 3/0, 2/1, 3/1, 2/2, 3/2, 3/2+LFE,
/// 3/4, 3/4+LFE).
const BD_LPCM_CHANNELS: [usize; 12] = [0, 1, 0, 2, 3, 3, 4, 4, 5, 6, 7, 8];
/// Parse and validate the four-byte Blu-ray LPCM header.
pub fn bluray_lpcm_header(p: &[u8]) -> Option<BlurayLpcm> {
if p.len() < 4 {
return None;
}
let channels = *BD_LPCM_CHANNELS.get((p[2] >> 4) as usize).filter(|&&c| c > 0)?;
let sample_rate = match p[2] & 15 {
1 => 48_000,
4 => 96_000,
5 => 192_000,
_ => return None,
};
let bits = match p[3] >> 6 {
1 => 16,
2 => 20,
3 => 24,
_ => return None,
};
Some(BlurayLpcm { sample_rate, channels, bits })
}
/// Decode one Blu-ray LPCM packet (big-endian; channels padded to an even count; 20-bit
/// samples are stored in 24 bits).
pub fn decode_bluray_lpcm(p: &[u8]) -> Option<Vec<Vec<f32>>> {
let h = bluray_lpcm_header(p)?;
let coded = (h.channels + 1) & !1;
let bps = if h.bits == 16 { 2 } else { 3 };
let data = &p[4..];
let n = data.len() / (coded * bps);
let mut out = vec![Vec::with_capacity(n); h.channels];
for k in 0..n {
for (c, o) in out.iter_mut().enumerate() {
let at = (k * coded + c) * bps;
let v = if bps == 2 {
i16::from_be_bytes([data[at], data[at + 1]]) as f32 / 32768.0
} else {
(i32::from_be_bytes([data[at], data[at + 1], data[at + 2], 0]) >> 8) as f32 / 8_388_608.0
};
o.push(v);
}
}
Some(out)
}
/// Bytes per two sample frames of DVD LPCM. The caller must bound `channels` to the DVD channel range.
pub fn dvd_lpcm_group_bytes(channels: usize, bits: u32) -> usize {
match bits {
16 => 4 * channels,
20 => 5 * channels,
_ => 6 * channels,
}
}
/// Decode DVD-Video LPCM (big-endian). 16-bit samples are interleaved; 20- and 24-bit samples
/// come in groups of two frames: the upper 16 bits of all the group's samples, then their low
/// 4 / 8 bits. The caller must bound `channels` to the DVD channel range.
pub fn decode_dvd_lpcm(data: &[u8], channels: usize, bits: u32) -> Vec<Vec<f32>> {
let ch = channels.max(1);
let group = dvd_lpcm_group_bytes(ch, bits);
let groups = data.len() / group;
let mut out = vec![Vec::with_capacity(groups * 2); ch];
for g in 0..groups {
let b = &data[g * group..(g + 1) * group];
for f in 0..2 {
for (c, o) in out.iter_mut().enumerate() {
let k = f * ch + c;
let hi = i16::from_be_bytes([b[2 * k], b[2 * k + 1]]) as i32;
let v = match bits {
16 => hi as f32 / 32768.0,
20 => {
let nib = b[4 * ch + k / 2];
let lo = if k.is_multiple_of(2) { nib >> 4 } else { nib & 15 } as i32;
((hi << 4) | lo) as f32 / 524_288.0
}
_ => ((hi << 8) | b[4 * ch + k] as i32) as f32 / 8_388_608.0,
};
o.push(v);
}
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn bluray_header_and_even_channel_padding() {
let packet = [0, 0, 0x31, 0x40, 0x40, 0, 0xc0, 0];
assert_eq!(bluray_lpcm_header(&packet), Some(BlurayLpcm { sample_rate: 48_000, channels: 2, bits: 16 }));
assert_eq!(decode_bluray_lpcm(&packet), Some(vec![vec![0.5], vec![-0.5]]));
assert_eq!(decode_bluray_lpcm(&packet[..3]), None);
}
#[test]
fn dvd_lpcm_16_20_and_24_bit_groups() {
assert_eq!(decode_dvd_lpcm(&[0x40, 0, 0xc0, 0], 1, 16), vec![vec![0.5, -0.5]]);
assert_eq!(decode_dvd_lpcm(&[0x40, 0, 0xc0, 0, 0], 1, 20), vec![vec![0.5, -0.5]]);
assert_eq!(decode_dvd_lpcm(&[0x40, 0, 0xc0, 0, 0, 0], 1, 24), vec![vec![0.5, -0.5]]);
assert_eq!(decode_dvd_lpcm(&[1, 2, 3], 1, 16), vec![Vec::<f32>::new()]);
}
}
+1 -135
View File
@@ -4,141 +4,7 @@
//! producing the N/2 "folded" time samples; the window overlap region is then unfolded against the
//! folded tail of the previous block, which the caller keeps in place in the output buffer.
#[derive(Clone, Copy, Default, Debug)]
pub struct Cpx {
pub re: f32,
pub im: f32,
}
impl Cpx {
#[inline]
fn mul(self, o: Cpx) -> Cpx {
Cpx { re: self.re * o.re - self.im * o.im, im: self.re * o.im + self.im * o.re }
}
#[inline]
fn add(self, o: Cpx) -> Cpx {
Cpx { re: self.re + o.re, im: self.im + o.im }
}
#[inline]
fn sub(self, o: Cpx) -> Cpx {
Cpx { re: self.re - o.re, im: self.im - o.im }
}
}
/// Mixed-radix (2, 3, 4, 5) forward complex FFT, unscaled. Radix 2, 3 and 4 use dedicated
/// butterflies; radix 5 uses a generic one.
pub struct Fft {
n: usize,
factors: Vec<usize>,
twiddle: Vec<Cpx>,
}
impl Fft {
pub fn new(n: usize) -> Fft {
let mut factors = Vec::new();
let mut m = n;
for p in [4usize, 2, 3, 5] {
while m.is_multiple_of(p) {
factors.push(p);
m /= p;
}
}
assert_eq!(m, 1, "unsupported FFT size {n}");
let twiddle = (0..n)
.map(|k| {
let ph = -2.0 * std::f64::consts::PI * k as f64 / n as f64;
Cpx { re: ph.cos() as f32, im: ph.sin() as f32 }
})
.collect();
Fft { n, factors, twiddle }
}
pub fn process(&self, data: &mut [Cpx], scratch: &mut Vec<Cpx>) {
scratch.clear();
scratch.extend_from_slice(&data[..self.n]);
self.rec(&mut data[..self.n], scratch, 1, self.n, 0, 1);
}
fn rec(&self, out: &mut [Cpx], inp: &[Cpx], in_stride: usize, n: usize, fi: usize, tw_stride: usize) {
let p = self.factors[fi];
let m = n / p;
if m == 1 {
for (i, o) in out.iter_mut().enumerate().take(p) {
*o = inp[i * in_stride];
}
} else {
for q in 0..p {
self.rec(&mut out[q * m..(q + 1) * m], &inp[q * in_stride..], in_stride * p, m, fi + 1, tw_stride * p);
}
}
let tw = &self.twiddle;
match p {
2 => {
for k in 0..m {
let a0 = out[k];
let a1 = out[m + k].mul(tw[k * tw_stride]);
out[k] = a0.add(a1);
out[m + k] = a0.sub(a1);
}
}
4 => {
for k in 0..m {
let t0 = out[k];
let t1 = out[m + k].mul(tw[k * tw_stride]);
let t2 = out[2 * m + k].mul(tw[2 * k * tw_stride]);
let t3 = out[3 * m + k].mul(tw[3 * k * tw_stride]);
let s0 = t0.add(t2);
let d0 = t0.sub(t2);
let s1 = t1.add(t3);
let d1 = t1.sub(t3);
out[k] = s0.add(s1);
out[2 * m + k] = s0.sub(s1);
// -j * d1 = (d1.im, -d1.re)
out[m + k] = Cpx { re: d0.re + d1.im, im: d0.im - d1.re };
out[3 * m + k] = Cpx { re: d0.re - d1.im, im: d0.im + d1.re };
}
}
3 => {
// sin(-2*pi/3) = -sqrt(3)/2
let h = tw[self.n / 3].im;
for k in 0..m {
let t0 = out[k];
let t1 = out[m + k].mul(tw[k * tw_stride]);
let t2 = out[2 * m + k].mul(tw[2 * k * tw_stride]);
let s = t1.add(t2);
let d = t1.sub(t2);
let a = Cpx { re: t0.re - 0.5 * s.re, im: t0.im - 0.5 * s.im };
out[k] = t0.add(s);
// j * h * d = (-h * d.im, h * d.re)
out[m + k] = Cpx { re: a.re - h * d.im, im: a.im + h * d.re };
out[2 * m + k] = Cpx { re: a.re + h * d.im, im: a.im - h * d.re };
}
}
_ => {
// Radix 5 (at most once per transform): generic butterfly with the five roots of
// unity looked up once.
let mut w5 = [Cpx::default(); 5];
for (q, w) in w5.iter_mut().enumerate() {
*w = tw[q * (self.n / p)];
}
let mut t = [Cpx::default(); 5];
for k in 0..m {
t[0] = out[k];
for q in 1..p {
t[q] = out[q * m + k].mul(tw[q * k * tw_stride]);
}
for q2 in 0..p {
let mut acc = t[0];
for q in 1..p {
acc = acc.add(t[q].mul(w5[(q * q2) % p]));
}
out[q2 * m + k] = acc;
}
}
}
}
}
}
use crate::opus::fft::{Cpx, Fft};
pub struct Imdct {
/// Full MDCT length (2x the number of coefficients).
+19 -327
View File
@@ -1,272 +1,45 @@
//! Pulse cache and bit allocation (RFC 6716 §4.3.3, §4.3.4.1).
use super::tables::{ALLOC, EBANDS, LOG2_FRAC_TABLE, MAX_LM, NB_EBANDS, OVERLAP};
use super::tables::{EBANDS, LOG2_FRAC_TABLE, MAX_LM, NB_EBANDS, OVERLAP};
use crate::opus::decode::range::{BITRES, RangeDecoder};
const MAX_PSEUDO: i32 = 40;
const LOG_MAX_PSEUDO: i32 = 6;
pub use crate::opus::rate_common::get_pulses;
#[cfg(test)]
use crate::opus::rate_common::log2_frac;
#[cfg(test)]
pub use crate::opus::rate_common::pvq_v;
pub const MAX_FINE_BITS: i32 = 8;
const FINE_OFFSET: i32 = 21;
pub const QTHETA_OFFSET: i32 = 4;
pub const QTHETA_OFFSET_TWOPHASE: i32 = 16;
const ALLOC_STEPS: i32 = 6;
/// Number of pulses represented by pseudo-pulse index `i`.
pub fn get_pulses(i: i32) -> i32 {
if i < 8 { i } else { (8 + (i & 7)) << ((i >> 3) - 1) }
}
/// Conservative (rounded up) log2 of `val` with `frac` fractional bits.
pub fn log2_frac(mut val: u32, mut frac: u32) -> i32 {
let mut l = crate::opus::decode::range::ilog(val);
if val & val.wrapping_sub(1) != 0 {
if l > 16 {
val = ((val - 1) >> (l - 16)) + 1;
} else {
val <<= 16 - l;
}
let mut l = (l - 1) << frac;
loop {
let b = (val >> 16) as i32;
l += b << frac;
val = (val + b as u32) >> b;
val = (val * val + 0x7FFF) >> 15;
if frac == 0 {
break;
}
frac -= 1;
}
l + (val > 0x8000) as i32
} else {
l -= 1;
l << frac
}
}
/// PVQ codebook size V(N, K) as u128 (saturating far above 2^32).
pub fn pvq_v(n: usize, k: usize) -> u128 {
// V(N,K) = V(N-1,K) + V(N,K-1) + V(N-1,K-1); computed row by row.
if k == 0 {
return 1;
}
if n == 0 {
return 0;
}
let mut row = vec![0u128; k + 1]; // V(0, k)
row[0] = 1;
for _ in 0..n {
let mut prev_diag = row[0]; // V(n-1, 0)
// V(n, 0) = 1
for kk in 1..=k {
let up = row[kk]; // V(n-1, kk)
let v = up.saturating_add(row[kk - 1]).saturating_add(prev_diag);
prev_diag = up;
row[kk] = v.min(u128::MAX >> 8);
}
row[0] = 1;
}
row[k]
}
fn fits_in32(n: usize, k: i32) -> bool {
pvq_v(n, k as usize) <= u32::MAX as u128
}
/// Precomputed per-mode data shared by all CELT decoders.
/// Decoder mode: shared allocation tables and decoder-specific inverse transforms.
pub struct Mode {
/// log2 of band width in 1/8 bits.
pub log_n: [i32; NB_EBANDS],
/// Index into `cache_bits` for `(LM+1)*NB_EBANDS + band` (-1 when unused).
pub cache_index: Vec<i16>,
/// `cache[0]` = max pseudo-pulses; `cache[k]` = bits(k) - 1 in 1/8 bits.
pub cache_bits: Vec<u8>,
/// Per-band maximum allocation: `[(2*LM + C-1)*NB_EBANDS + band]`.
pub caps: Vec<u8>,
core: crate::opus::rate_common::Mode,
pub window: [f32; OVERLAP],
pub mdct: [super::mdct::Imdct; MAX_LM + 1],
}
impl Mode {
pub fn new() -> Mode {
let mut log_n = [0i32; NB_EBANDS];
for (i, l) in log_n.iter_mut().enumerate() {
*l = log2_frac((EBANDS[i + 1] - EBANDS[i]) as u32, BITRES);
}
let (cache_index, cache_bits) = compute_pulse_cache(MAX_LM as i32);
let mut m = Mode {
log_n,
cache_index,
cache_bits,
caps: Vec::new(),
pub fn new() -> Self {
let mut mode = Self {
core: crate::opus::rate_common::Mode::new(),
window: [0.0; OVERLAP],
mdct: [super::mdct::Imdct::new(1920), super::mdct::Imdct::new(960), super::mdct::Imdct::new(480), super::mdct::Imdct::new(240)],
};
m.caps = compute_caps(&m);
for i in 0..OVERLAP {
let x = std::f64::consts::FRAC_PI_2 * (i as f64 + 0.5) / OVERLAP as f64;
let s = x.sin();
m.window[i] = (std::f64::consts::FRAC_PI_2 * s * s).sin() as f32;
mode.window[i] = (std::f64::consts::FRAC_PI_2 * s * s).sin() as f32;
}
m
}
#[inline]
pub fn cache(&self, lm: i32, band: usize) -> &[u8] {
let idx = self.cache_index[((lm + 1) as usize) * NB_EBANDS + band];
&self.cache_bits[idx.max(0) as usize..]
}
pub fn bits2pulses(&self, band: usize, lm: i32, bits: i32) -> i32 {
let cache = self.cache(lm, band);
let mut lo = 0i32;
let mut hi = cache[0] as i32;
let bits = bits - 1;
for _ in 0..LOG_MAX_PSEUDO {
let mid = (lo + hi + 1) >> 1;
if cache[mid as usize] as i32 >= bits {
hi = mid;
} else {
lo = mid;
}
}
let lo_bits = if lo == 0 { -1 } else { cache[lo as usize] as i32 };
if bits - lo_bits <= cache[hi as usize] as i32 - bits { lo } else { hi }
}
pub fn pulses2bits(&self, band: usize, lm: i32, pulses: i32) -> i32 {
if pulses == 0 { 0 } else { self.cache(lm, band)[pulses as usize] as i32 + 1 }
}
/// Per-band allocation caps for this frame (`init_caps`).
pub fn init_caps(&self, lm: usize, c: usize) -> [i32; NB_EBANDS] {
let mut cap = [0i32; NB_EBANDS];
for (i, cp) in cap.iter_mut().enumerate() {
let n = (EBANDS[i + 1] - EBANDS[i]) << lm;
*cp = ((self.caps[NB_EBANDS * (2 * lm + c - 1) + i] as i32 + 64) * c as i32 * n) >> 2;
}
cap
mode
}
}
fn compute_pulse_cache(lm_max: i32) -> (Vec<i16>, Vec<u8>) {
let nb = NB_EBANDS;
let rows = (lm_max + 2) as usize;
let mut index = vec![-1i16; nb * rows];
let mut entries: Vec<(usize, i32, usize)> = Vec::new(); // (N, K, offset)
let mut curr = 0usize;
for i in 0..rows {
for j in 0..nb {
let n = (((EBANDS[j + 1] - EBANDS[j]) << i) >> 1) as usize;
index[i * nb + j] = -1;
// Reuse the entry of an earlier band with the same size.
'search: for k in 0..=i {
for nn in 0..nb {
if k == i && nn >= j {
break;
}
if n == (((EBANDS[nn + 1] - EBANDS[nn]) << k) >> 1) as usize {
index[i * nb + j] = index[k * nb + nn];
break 'search;
}
}
}
if index[i * nb + j] == -1 && n != 0 {
let mut k = 0;
while k < MAX_PSEUDO && fits_in32(n, get_pulses(k + 1)) {
k += 1;
}
index[i * nb + j] = curr as i16;
entries.push((n, k, curr));
curr += (k + 1) as usize;
}
}
impl std::ops::Deref for Mode {
type Target = crate::opus::rate_common::Mode;
fn deref(&self) -> &Self::Target {
&self.core
}
let mut bits = vec![0u8; curr];
for &(n, k, off) in &entries {
let maxk = get_pulses(k) as usize;
let tmp = required_bits(n, maxk, BITRES);
for j in 1..=k {
bits[off + j as usize] = (tmp[get_pulses(j) as usize] - 1) as u8;
}
bits[off] = k as u8;
}
(index, bits)
}
/// Bits (1/2^frac units, rounded up) needed for each K in `0..=maxk` for dimension `n`.
fn required_bits(n: usize, maxk: usize, frac: u32) -> Vec<i32> {
let mut out = vec![0i32; maxk + 1];
if n == 1 {
for o in out.iter_mut().skip(1) {
*o = 1 << frac;
}
} else {
for (k, o) in out.iter_mut().enumerate().skip(1) {
let v = pvq_v(n, k);
*o = log2_frac(v as u32, frac);
}
}
out
}
fn compute_caps(m: &Mode) -> Vec<u8> {
let nb = NB_EBANDS;
let mut caps = vec![0u8; (MAX_LM + 1) * 2 * nb];
for i in 0..=MAX_LM as i32 {
for c in 1..=2i32 {
for j in 0..nb {
let mut n0 = EBANDS[j + 1] - EBANDS[j];
let max_bits = if n0 << i == 1 {
(c * (1 + MAX_FINE_BITS)) << BITRES
} else {
let mut lm0 = 0;
if n0 > 2 {
n0 >>= 1;
lm0 -= 1;
} else if n0 <= 1 {
lm0 = i.min(1);
n0 <<= lm0;
}
let pcache = m.cache(lm0, j);
let mut mb = pcache[pcache[0] as usize] as i32 + 1;
let mut n = n0;
for k in 0..(i - lm0) {
mb <<= 1;
let offset = ((m.log_n[j] + ((lm0 + k) << BITRES)) >> 1) - QTHETA_OFFSET;
let num = 459 * ((2 * n - 1) * offset + mb);
let den = ((2 * n - 1) << 9) - 459;
let qb = ((num + (den >> 1)) / den).min(57);
mb += qb;
n <<= 1;
}
if c == 2 {
mb <<= 1;
let offset = ((m.log_n[j] + (i << BITRES)) >> 1) - if n == 2 { QTHETA_OFFSET_TWOPHASE } else { QTHETA_OFFSET };
let ndof = 2 * n - 1 - (n == 2) as i32;
let p = if n == 2 { 512 } else { 487 };
let num = p * (mb + ndof * offset);
let den = (ndof << 9) - p;
let qb = ((num + (den >> 1)) / den).min(if n == 2 { 64 } else { 61 });
mb += qb;
}
let ndof = c * n + if c == 2 && n > 2 { 1 } else { 0 };
let mut offset = ((m.log_n[j] + (i << BITRES)) >> 1) - FINE_OFFSET;
if n == 2 {
offset += (1 << BITRES) >> 2;
}
let num = mb + ndof * offset;
let den = (ndof - 1) << BITRES;
let qb = ((num + (den >> 1)) / den).min(MAX_FINE_BITS);
mb += (c * qb) << BITRES;
mb
};
let w = c * ((EBANDS[j + 1] - EBANDS[j]) << i);
let v = (4 * max_bits / w) - 64;
caps[((i * 2 + c - 1) as usize) * nb + j] = v.clamp(0, 255) as u8;
}
}
}
caps
}
/// Result of the allocation process.
@@ -296,89 +69,8 @@ pub struct AllocationInput<'a, 'd> {
pub fn compute_allocation(input: AllocationInput<'_, '_>) -> Allocation {
let AllocationInput { mode: m, start, end, offsets, cap, trim: alloc_trim, total, channels: c, lm, decoder: dec } = input;
let ci = c as i32;
let lmi = lm as i32;
let mut total = total.max(0);
let mut skip_start = start;
let skip_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 };
total -= skip_rsv;
let mut intensity_rsv = 0;
let mut dual_stereo_rsv = 0;
if c == 2 {
intensity_rsv = LOG2_FRAC_TABLE[end - start];
if intensity_rsv > total {
intensity_rsv = 0;
} else {
total -= intensity_rsv;
dual_stereo_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 };
total -= dual_stereo_rsv;
}
}
let mut bits1 = [0i32; NB_EBANDS];
let mut bits2 = [0i32; NB_EBANDS];
let mut thresh = [0i32; NB_EBANDS];
let mut trim_offset = [0i32; NB_EBANDS];
for j in start..end {
let w = EBANDS[j + 1] - EBANDS[j];
thresh[j] = (ci << BITRES).max(((3 * w) << lmi << BITRES) >> 4);
trim_offset[j] = (ci * w * (alloc_trim - 5 - lmi) * (end as i32 - j as i32 - 1) * (1 << (lmi + BITRES as i32))) >> 6;
if w << lmi == 1 {
trim_offset[j] -= ci << BITRES;
}
}
let nb_alloc = ALLOC.len() as i32;
let mut lo = 1;
let mut hi = nb_alloc - 1;
loop {
let mut done = false;
let mut psum = 0;
let mid = (lo + hi) >> 1;
for j in (start..end).rev() {
let n = EBANDS[j + 1] - EBANDS[j];
let mut bitsj = (ci * n * (ALLOC[mid as usize][j] as i32)) << lmi >> 2;
if bitsj > 0 {
bitsj = (bitsj + trim_offset[j]).max(0);
}
bitsj += offsets[j];
if bitsj >= thresh[j] || done {
done = true;
psum += bitsj.min(cap[j]);
} else if bitsj >= ci << BITRES {
psum += ci << BITRES;
}
}
if psum > total {
hi = mid - 1;
} else {
lo = mid + 1;
}
if lo > hi {
break;
}
}
hi = lo;
lo -= 1;
for j in start..end {
let n = EBANDS[j + 1] - EBANDS[j];
let mut bits1j = (ci * n * (ALLOC[lo as usize][j] as i32)) << lmi >> 2;
let mut bits2j = if hi >= nb_alloc { cap[j] } else { (ci * n * (ALLOC[hi as usize][j] as i32)) << lmi >> 2 };
if bits1j > 0 {
bits1j = (bits1j + trim_offset[j]).max(0);
}
if bits2j > 0 {
bits2j = (bits2j + trim_offset[j]).max(0);
}
if lo > 0 {
bits1j += offsets[j];
}
bits2j += offsets[j];
if offsets[j] > 0 {
skip_start = j;
}
bits2j = (bits2j - bits1j).max(0);
bits1[j] = bits1j;
bits2[j] = bits2j;
}
let crate::opus::rate_common::Bracket { skip_start, bits1, bits2, thresh, total, skip_rsv, intensity_rsv, dual_stereo_rsv } =
crate::opus::rate_common::bracket(start, end, offsets, cap, alloc_trim, total, c, lm);
interp_bits2pulses(InterpPulsesInput {
mode: m,
start,
@@ -14,6 +14,7 @@ pub const MAX_LM: usize = 3;
pub const PREEMPH: f32 = 0.850_006_1;
/// Static allocation table (Table 57), indexed `[quality][band]`, in 1/32 bit per MDCT bin.
#[cfg(not(feature = "opus-encode"))]
pub const ALLOC: [[u8; NB_EBANDS]; 11] = transpose([
[0, 90, 110, 118, 126, 134, 144, 152, 162, 172, 200],
[0, 80, 100, 110, 119, 127, 137, 145, 155, 165, 200],
@@ -38,6 +39,7 @@ pub const ALLOC: [[u8; NB_EBANDS]; 11] = transpose([
[0, 0, 0, 0, 0, 1, 1, 1, 1, 20, 104],
]);
#[cfg(not(feature = "opus-encode"))]
const fn transpose(t: [[u8; 11]; NB_EBANDS]) -> [[u8; NB_EBANDS]; 11] {
let mut out = [[0u8; NB_EBANDS]; 11];
let mut b = 0;
+1 -19
View File
@@ -4,25 +4,7 @@
//! RFC 6716 Table 54 (and the integer delays used when no rate conversion is needed), so that the
//! SILK output lines up with the CELT layer in Hybrid mode.
/// Dot product with eight independent partial sums, so the additions pipeline and the loop
/// vectorises instead of waiting on one 4-cycle add chain. Rounds differently from a
/// sequential sum (the resampler is not normative).
#[inline]
fn dot8(a: &[f32], b: &[f32]) -> f32 {
let mut s = [0f32; 8];
let (ca, ra) = a.as_chunks::<8>();
let (cb, rb) = b.as_chunks::<8>();
for (x, y) in ca.iter().zip(cb) {
for l in 0..8 {
s[l] += x[l] * y[l];
}
}
let mut acc = ((s[0] + s[4]) + (s[1] + s[5])) + ((s[2] + s[6]) + (s[3] + s[7]));
for (x, y) in ra.iter().zip(rb) {
acc += x * y;
}
acc
}
use crate::opus::dot::dot8;
#[derive(Clone)]
pub struct Resampler {
+20
View File
@@ -0,0 +1,20 @@
//! Shared fixed-order FIR dot product.
/// Dot product with eight independent partial sums, which the compiler can vectorise (a single
/// running sum is bound by the latency of the float addition). The sums are combined pairwise.
#[inline]
pub(super) fn dot8(a: &[f32], b: &[f32]) -> f32 {
let mut s = [0f32; 8];
let (ca, ra) = a.as_chunks::<8>();
let (cb, rb) = b.as_chunks::<8>();
for (x, y) in ca.iter().zip(cb) {
for l in 0..8 {
s[l] += x[l] * y[l];
}
}
let mut acc = ((s[0] + s[4]) + (s[1] + s[5])) + ((s[2] + s[6]) + (s[3] + s[7]));
for (x, y) in ra.iter().zip(rb) {
acc += x * y;
}
acc
}
+282 -59
View File
@@ -8,7 +8,8 @@
use crate::opus::encoder::range::{BITRES, RangeEncoder, ilog};
use crate::opus::encoder::rate::{Mode, QTHETA_OFFSET, get_pulses};
use crate::opus::encoder::tables::{EBANDS, NB_EBANDS};
use crate::opus::encoder::tables::EBANDS;
use crate::opus::encoder::{CeltFrame, rate::Allocation};
pub const SPREAD_NORMAL: usize = 2;
@@ -55,7 +56,7 @@ fn exp_rotation1_inv(x: &mut [f32], len: usize, stride: usize, c: f32, s: f32) {
if len <= stride {
return;
}
if len >= 2 * stride + 1 {
if len > 2 * stride {
for i in 0..=len - 2 * stride - 1 {
let x1 = x[i];
let x2 = x[i + stride];
@@ -96,12 +97,24 @@ pub fn exp_rotation_inv(x: &mut [f32], len: usize, k: usize, spread: usize) {
}
}
/// Relative slack for the pulse-candidate filter in [`pvq_search`]: far above the f32 rounding of
/// the filter score (a few 1e-7), far below the spread of real scores.
const PVQ_SCORE_SLACK: f32 = 1e-5;
/// Greedy PVQ search: the integer vector `y` with `sum |y| = k` closest in direction to `x`.
pub fn pvq_search(x: &[f32], k: usize, y: &mut [i32]) {
let n = x.len();
y[..n].fill(0);
let abs: Vec<f32> = x.iter().map(|v| v.abs()).collect();
let sum: f32 = abs.iter().sum();
// `abs`, `dy` (2 y + 1, infinite in the padding) and the per-pass scores `score`, each padded to
// whole 8-lane chunks.
let np = n.next_multiple_of(8);
let mut buf = vec![0f32; 3 * np];
let (abs, rest) = buf.split_at_mut(np);
let (dy, score) = rest.split_at_mut(np);
for (a, v) in abs.iter_mut().zip(x) {
*a = v.abs();
}
let sum: f32 = abs[..n].iter().sum();
if !sum.is_finite() || sum <= 1e-15 {
y[0] = k as i32;
return;
@@ -113,30 +126,69 @@ pub fn pvq_search(x: &[f32], k: usize, y: &mut [i32]) {
// Pre-project onto the pyramid (rounding down) and place the rest greedily.
let r = (k as f32 - 1.0) / sum;
for j in 0..n {
let p = (abs[j] * r).floor() as i32;
// Non-negative and at most k, so the truncating cast is the floor.
let p = (abs[j] * r) as i32;
y[j] = p;
placed += p as usize;
xy += abs[j] * p as f32;
yy += (p * p) as f32;
}
}
for j in 0..np {
dy[j] = if j < n { 2.0 * y[j] as f32 + 1.0 } else { f32::INFINITY };
}
while placed < k {
// Vectorisable scores (xy + |x_j|)^2 / (yy + 2 y_j + 1) and their maximum.
for ((sc, &a), &d) in score.iter_mut().zip(abs.iter()).zip(dy.iter()) {
let t = xy + a;
*sc = t * t / (yy + d);
}
let mut lane_max = [0f32; 8];
for c in score.as_chunks::<8>().0 {
for i in 0..8 {
lane_max[i] = if c[i] > lane_max[i] { c[i] } else { lane_max[i] };
}
}
let top = lane_max.iter().fold(0f32, |m, &v| if v > m { v } else { m });
// The exact cross-multiplied test below, in index order with the first maximum winning, only
// matters for entries within rounding distance of the top score: every other entry is beaten
// by the top one whenever it is reached and never beats it afterwards. Running the test on
// those alone therefore picks the same pulse as running it on all of them.
let filter = top > 1e-30 && top.is_finite();
let lo = top * (1.0 - PVQ_SCORE_SLACK);
// Usually exactly one entry qualifies; its index falls out of a lane-wise count.
let mut count = [0u32; 8];
let mut index = [0u32; 8];
for (ci, c) in score.as_chunks::<8>().0.iter().enumerate() {
for i in 0..8 {
let hit = u32::from(c[i] >= lo);
count[i] += hit;
index[i] += hit * (ci * 8 + i) as u32;
}
}
let mut best = 0usize;
let mut best_num = -1f32;
let mut best_den = 1f32;
for j in 0..n {
let num = xy + abs[j];
let num = num * num;
let den = yy + 2.0 * y[j] as f32 + 1.0;
if num * best_den > best_num * den {
best_num = num;
best_den = den;
best = j;
if filter && count.iter().sum::<u32>() == 1 {
best = index.iter().sum::<u32>() as usize;
} else {
let mut best_num = -1f32;
let mut best_den = 1f32;
for j in 0..n {
if !filter || score[j] >= lo {
let num = xy + abs[j];
let num = num * num;
let den = yy + 2.0 * y[j] as f32 + 1.0;
if num * best_den > best_num * den {
best_num = num;
best_den = den;
best = j;
}
}
}
}
xy += abs[best];
yy += 2.0 * y[best] as f32 + 1.0;
y[best] += 1;
dy[best] = 2.0 * y[best] as f32 + 1.0;
placed += 1;
}
for j in 0..n {
@@ -146,25 +198,45 @@ pub fn pvq_search(x: &[f32], k: usize, y: &mut [i32]) {
}
}
/// Encodes the PVQ codeword `y` (`sum |y| = k`) as the index the RFC 6716 §4.3.4.2 decoder maps
/// back to `y`.
pub fn encode_pulses(y: &[i32], k: usize, enc: &mut RangeEncoder, v: &mut Vec<u64>) {
let n = y.len();
let w = k + 1;
v.clear();
v.resize((n + 1) * w, 0);
/// Largest band width and pulse count the encoder codes (176 coefficients, `get_pulses(40)` = 128).
const PVQ_MAX_N: usize = 176;
const PVQ_MAX_K: usize = 128;
/// Row-major `V(n, k)` for `n <= max_n`, `k <= max_k` with row stride `max_k + 1`, saturated at
/// `u32::MAX` (a codebook that large is never coded, see `rate::fits_in32`).
fn build_pvq_table(max_n: usize, max_k: usize) -> Vec<u32> {
let w = max_k + 1;
let mut v = vec![0u32; (max_n + 1) * w];
v[0] = 1;
for nn in 1..=n {
for nn in 1..=max_n {
v[nn * w] = 1;
for kk in 1..=k {
let a = v[(nn - 1) * w + kk];
let b = v[nn * w + kk - 1];
let c = v[(nn - 1) * w + kk - 1];
v[nn * w + kk] = a.saturating_add(b).saturating_add(c);
for kk in 1..=max_k {
let sum = v[(nn - 1) * w + kk] as u64 + v[nn * w + kk - 1] as u64 + v[(nn - 1) * w + kk - 1] as u64;
v[nn * w + kk] = sum.min(u32::MAX as u64) as u32;
}
}
let total = v[n * w + k];
let vv = |nn: usize, kx: usize| v[nn * w + kx];
v
}
fn pvq_table() -> &'static [u32] {
static TABLE: std::sync::OnceLock<Vec<u32>> = std::sync::OnceLock::new();
TABLE.get_or_init(|| build_pvq_table(PVQ_MAX_N, PVQ_MAX_K))
}
/// Encodes the PVQ codeword `y` (`sum |y| = k`) as the index the RFC 6716 §4.3.4.2 decoder maps
/// back to `y`.
pub fn encode_pulses(y: &[i32], k: usize, enc: &mut RangeEncoder) {
let n = y.len();
// Shapes outside the coded range (never produced by the band layout) get a local table.
let local;
let (v, w) = if n <= PVQ_MAX_N && k <= PVQ_MAX_K {
(pvq_table(), PVQ_MAX_K + 1)
} else {
local = build_pvq_table(n, k);
(&local[..], k + 1)
};
let row = |nn: usize| &v[nn * w..(nn + 1) * w];
let total = row(n)[k] as u64;
let mut index = 0u64;
let mut kk = k;
for j in 0..n {
@@ -173,10 +245,11 @@ pub fn encode_pulses(y: &[i32], k: usize, enc: &mut RangeEncoder, v: &mut Vec<u6
}
let rem = n - j;
let t = y[j].unsigned_abs() as usize;
let below = row(rem - 1);
// Codewords with |y_j| = t start at sum_{m < kk - t} V(rem-1, m) within their sign half.
let mut base: u64 = (0..kk - t).map(|m| vv(rem - 1, m)).sum();
let mut base: u64 = below[..kk - t].iter().map(|&x| x as u64).sum();
if y[j] < 0 {
base += (vv(rem - 1, kk) + vv(rem, kk)) / 2;
base += (below[kk] as u64 + row(rem)[kk] as u64) / 2;
}
index += base;
kk -= t;
@@ -193,7 +266,6 @@ pub struct BandEnc<'m, 'e> {
pub i: usize,
pub spread: usize,
pub remaining_bits: i32,
pvq_v: Vec<u64>,
iy: Vec<i32>,
tmp: Vec<f32>,
}
@@ -217,7 +289,7 @@ impl BandEnc<'_, '_> {
let q = qn as u32;
let half = q >> 1;
let ft = (half + 1) * (half + 1);
let (fl, fs) = if it <= half { (it * (it + 1) >> 1, it + 1) } else { (ft - ((q + 1 - it) * (q + 2 - it) >> 1), q + 1 - it) };
let (fl, fs) = if it <= half { ((it * (it + 1)) >> 1, it + 1) } else { (ft - (((q + 1 - it) * (q + 2 - it)) >> 1), q + 1 - it) };
self.enc.encode(fl, fl + fs, ft);
itheta = ((it * 16384) / q) as i32;
}
@@ -287,36 +359,29 @@ impl BandEnc<'_, '_> {
iy.clear();
iy.resize(n, 0);
pvq_search(&tmp, k, &mut iy);
encode_pulses(&iy, k, self.enc, &mut self.pvq_v);
encode_pulses(&iy, k, self.enc);
self.iy = iy;
self.tmp = tmp;
}
}
}
/// Codes all band shapes (`quant_all_bands`, encoder side, long blocks). `x` holds `c` channels
/// of `n` normalised MDCT coefficients each.
#[allow(clippy::too_many_arguments)]
pub fn quant_all_bands(
m: &Mode,
start: usize,
end: usize,
x: &[f32],
n_total: usize,
c: usize,
pulses: &[i32; NB_EBANDS],
spread: usize,
dual_stereo: bool,
intensity: usize,
total_bits: i32,
mut balance: i32,
enc: &mut RangeEncoder,
lm: usize,
coded_bands: usize,
) {
/// Shape-coding controls, independent of the selected band allocation.
pub struct BandCoding {
pub spread: usize,
pub total_bits: i32,
}
/// Codes all band shapes (`quant_all_bands`, encoder side, long blocks). `x` holds
/// `frame.channels` channels of `n_total` normalised MDCT coefficients each.
pub fn quant_all_bands(m: &Mode, frame: CeltFrame, x: &[f32], n_total: usize, allocation: &Allocation, coding: BandCoding, enc: &mut RangeEncoder) {
let CeltFrame { start, end, channels: c, lm } = frame;
let Allocation { pulses, dual_stereo, intensity, coded_bands, balance, .. } = allocation;
let (dual_stereo, intensity, coded_bands, mut balance) = (*dual_stereo, *intensity, *coded_bands, *balance);
let BandCoding { spread, total_bits } = coding;
let mm = 1usize << lm;
debug_assert!(c == 1 || dual_stereo || intensity <= start, "stereo frames are coded as dual stereo");
let mut ctx = BandEnc { m, enc, i: 0, spread, remaining_bits: 0, pvq_v: Vec::new(), iy: Vec::new(), tmp: Vec::new() };
let mut ctx = BandEnc { m, enc, i: 0, spread, remaining_bits: 0, iy: Vec::new(), tmp: Vec::new() };
for i in start..end {
ctx.i = i;
let xo = mm * EBANDS[i] as usize;
@@ -362,7 +427,7 @@ mod tests {
x[i + stride] = c * x2 + s * x1;
x[i] = c * x1 + ms * x2;
}
if len >= 2 * stride + 1 {
if len > 2 * stride {
for i in (0..=len - 2 * stride - 1).rev() {
let x1 = x[i];
let x2 = x[i + stride];
@@ -466,12 +531,11 @@ mod tests {
#[test]
fn pvq_index_roundtrip_exhaustive() {
let mut v = Vec::new();
for n in 1..6usize {
for k in 1..6usize {
for y in all_codewords(n, k) {
let mut e = RangeEncoder::new();
encode_pulses(&y, k, &mut e, &mut v);
encode_pulses(&y, k, &mut e);
let buf = e.done(16);
let mut d = RangeDecoder::new(&buf);
assert_eq!(decode_pulses(n, k, &mut d), y, "n={n} k={k}");
@@ -480,6 +544,165 @@ mod tests {
}
}
/// The per-call table encoder this module used before the shared table.
fn encode_pulses_reference(y: &[i32], k: usize, enc: &mut RangeEncoder) {
let n = y.len();
let w = k + 1;
let mut v = vec![0u64; (n + 1) * w];
v[0] = 1;
for nn in 1..=n {
v[nn * w] = 1;
for kk in 1..=k {
let a = v[(nn - 1) * w + kk];
let b = v[nn * w + kk - 1];
let c = v[(nn - 1) * w + kk - 1];
v[nn * w + kk] = a.saturating_add(b).saturating_add(c);
}
}
let total = v[n * w + k];
let vv = |nn: usize, kx: usize| v[nn * w + kx];
let mut index = 0u64;
let mut kk = k;
for j in 0..n {
if kk == 0 {
break;
}
let rem = n - j;
let t = y[j].unsigned_abs() as usize;
let mut base: u64 = (0..kk - t).map(|m| vv(rem - 1, m)).sum();
if y[j] < 0 {
base += (vv(rem - 1, kk) + vv(rem, kk)) / 2;
}
index += base;
kk -= t;
}
if total >= 2 {
enc.uint(index as u32, total as u32);
}
}
#[test]
fn pvq_table_encode_matches_per_call_table() {
let mut seed = 0x9E37_79B9_7F4A_7C15u64;
let mut next = move |m: u64| {
seed ^= seed << 13;
seed ^= seed >> 7;
seed ^= seed << 17;
seed % m
};
let mut checked = 0;
for n in (1..=PVQ_MAX_N).step_by(5).chain([PVQ_MAX_N]) {
for k in (1..=PVQ_MAX_K).step_by(3).chain([PVQ_MAX_K]) {
if crate::opus::encoder::rate::pvq_v(n, k) > u32::MAX as u128 {
continue;
}
for _ in 0..4 {
let mut y = vec![0i32; n];
for _ in 0..k {
let at = next(n as u64) as usize;
y[at] += if next(2) == 0 { 1 } else { -1 };
}
// Opposite pulses cancel; top up so that sum |y| = k again.
let mut have: usize = y.iter().map(|v| v.unsigned_abs() as usize).sum();
while have < k {
let at = next(n as u64) as usize;
y[at] += if y[at] < 0 { -1 } else { 1 };
have += 1;
}
let (mut a, mut b) = (RangeEncoder::new(), RangeEncoder::new());
encode_pulses(&y, k, &mut a);
encode_pulses_reference(&y, k, &mut b);
assert_eq!(a.done(16), b.done(16), "n={n} k={k}");
checked += 1;
}
}
}
assert!(checked > 100, "{checked}");
}
/// The scalar greedy search the vectorised one replaced.
fn pvq_search_ref(x: &[f32], k: usize, y: &mut [i32]) {
let n = x.len();
y[..n].fill(0);
let abs: Vec<f32> = x.iter().map(|v| v.abs()).collect();
let sum: f32 = abs.iter().sum();
if !sum.is_finite() || sum <= 1e-15 {
y[0] = k as i32;
return;
}
let mut placed = 0usize;
let mut xy = 0f32;
let mut yy = 0f32;
if k > n / 2 {
let r = (k as f32 - 1.0) / sum;
for j in 0..n {
let p = (abs[j] * r).floor() as i32;
y[j] = p;
placed += p as usize;
xy += abs[j] * p as f32;
yy += (p * p) as f32;
}
}
while placed < k {
let mut best = 0usize;
let mut best_num = -1f32;
let mut best_den = 1f32;
for j in 0..n {
let num = xy + abs[j];
let num = num * num;
let den = yy + 2.0 * y[j] as f32 + 1.0;
if num * best_den > best_num * den {
best_num = num;
best_den = den;
best = j;
}
}
xy += abs[best];
yy += 2.0 * y[best] as f32 + 1.0;
y[best] += 1;
placed += 1;
}
for j in 0..n {
if x[j] < 0.0 {
y[j] = -y[j];
}
}
}
#[test]
fn pvq_search_matches_scalar_reference() {
let mut state = 0x9e3779b97f4a7c15u64;
let mut rnd = move || {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
(state >> 40) as f32 / (1u64 << 24) as f32 * 2.0 - 1.0
};
let mut a = vec![0i32; 200];
let mut b = vec![0i32; 200];
let mut cases = 0;
for round in 0..6000 {
let n = 1 + (round * 37 + (rnd().abs() * 176.0) as usize) % 176;
let k = 1 + (rnd().abs() * 127.0) as usize;
let scale = [1.0f32, 1e-9, 1e6, 3e-3][round % 4];
let x: Vec<f32> = (0..n)
.map(|_| match round % 6 {
0 => rnd(),
1 => if rnd() > 0.8 { rnd() } else { 0.0 },
2 => (rnd() * 4.0).round() / 4.0,
3 => if rnd() > 0.0 { 1.0 } else { -1.0 },
4 => rnd().powi(5),
_ => if rnd() > 0.95 { 3.0 * rnd() } else { 1e-4 * rnd() },
} * scale)
.collect();
pvq_search(&x, k, &mut a);
pvq_search_ref(&x, k, &mut b);
assert_eq!(a[..n], b[..n], "n={n} k={k} round={round}");
cases += 1;
}
assert_eq!(cases, 6000);
}
#[test]
fn pvq_search_finds_good_direction() {
let x: Vec<f32> = (0..32).map(|i| ((i as f32) * 0.7).sin()).collect();
+10 -15
View File
@@ -4,8 +4,9 @@
//! The quantised energies are updated with exactly the decoder's arithmetic so the prediction
//! state never drifts between encoder and decoder.
use crate::opus::encoder::CeltFrame;
use crate::opus::encoder::range::RangeEncoder;
use crate::opus::encoder::rate::MAX_FINE_BITS;
use crate::opus::encoder::rate::{Allocation, MAX_FINE_BITS};
use crate::opus::encoder::tables::{BETA_COEF, BETA_INTRA, E_PROB_MODEL, NB_EBANDS, PRED_COEF, SMALL_ENERGY_ICDF};
const LAPLACE_MINP: u32 = 1;
@@ -56,18 +57,15 @@ pub fn laplace_encode(enc: &mut RangeEncoder, value: i32, mut fs: u32, decay: u3
/// `quant_coarse_energy` for one prediction mode. `e` holds the target log2 energies (relative
/// to `E_MEANS`), `old_e` the quantised energies, updated in place. `budget` is the frame size in
/// bits.
#[allow(clippy::too_many_arguments)]
pub fn quant_coarse_energy(
start: usize,
end: usize,
frame: CeltFrame,
e: &[f32; 2 * NB_EBANDS],
old_e: &mut [f32; 2 * NB_EBANDS],
intra: bool,
enc: &mut RangeEncoder,
c: usize,
lm: usize,
budget: i32,
) {
let CeltFrame { start, end, channels: c, lm } = frame;
let prob = &E_PROB_MODEL[lm][intra as usize];
let (coef, beta) = if intra { (0.0, BETA_INTRA) } else { (PRED_COEF[lm], BETA_COEF[lm]) };
let mut prev = [0f32; 2];
@@ -112,14 +110,13 @@ pub fn quant_coarse_energy(
/// `quant_fine_energy`: raw bits refining each band's quantised energy.
pub fn quant_fine_energy(
start: usize,
end: usize,
frame: CeltFrame,
e: &[f32; 2 * NB_EBANDS],
old_e: &mut [f32; 2 * NB_EBANDS],
fine_quant: &[i32; NB_EBANDS],
enc: &mut RangeEncoder,
c: usize,
) {
let CeltFrame { start, end, channels: c, .. } = frame;
for i in start..end {
let fq = fine_quant[i];
if fq <= 0 {
@@ -139,18 +136,16 @@ pub fn quant_fine_energy(
/// `quant_energy_finalise`: spends the bits left at the end of the frame on one more bit of
/// energy resolution per band, in priority order.
#[allow(clippy::too_many_arguments)]
pub fn quant_energy_finalise(
start: usize,
end: usize,
frame: CeltFrame,
e: &[f32; 2 * NB_EBANDS],
old_e: &mut [f32; 2 * NB_EBANDS],
fine_quant: &[i32; NB_EBANDS],
fine_priority: &[i32; NB_EBANDS],
allocation: &Allocation,
mut bits_left: i32,
enc: &mut RangeEncoder,
c: usize,
) {
let CeltFrame { start, end, channels: c, .. } = frame;
let Allocation { fine_quant, fine_priority, .. } = allocation;
for prio in 0..2 {
let mut i = start;
while i < end && bits_left >= c as i32 {
+26 -78
View File
@@ -9,84 +9,7 @@
use crate::opus::encoder::tables::OVERLAP;
#[derive(Clone, Copy, Default, Debug)]
pub struct Cpx {
pub re: f32,
pub im: f32,
}
impl Cpx {
#[inline]
fn mul(self, o: Cpx) -> Cpx {
Cpx { re: self.re * o.re - self.im * o.im, im: self.re * o.im + self.im * o.re }
}
#[inline]
fn add(self, o: Cpx) -> Cpx {
Cpx { re: self.re + o.re, im: self.im + o.im }
}
}
/// Mixed-radix (2, 3, 4, 5) forward complex FFT, unscaled (decimation in time, recursive).
pub struct Fft {
n: usize,
factors: Vec<usize>,
twiddle: Vec<Cpx>,
}
impl Fft {
pub fn new(n: usize) -> Fft {
let mut factors = Vec::new();
let mut m = n;
for p in [4usize, 2, 3, 5] {
while m.is_multiple_of(p) {
factors.push(p);
m /= p;
}
}
assert_eq!(m, 1, "unsupported FFT size {n}");
let twiddle = (0..n)
.map(|k| {
let ph = -2.0 * std::f64::consts::PI * k as f64 / n as f64;
Cpx { re: ph.cos() as f32, im: ph.sin() as f32 }
})
.collect();
Fft { n, factors, twiddle }
}
pub fn process(&self, data: &mut [Cpx], scratch: &mut Vec<Cpx>) {
scratch.clear();
scratch.extend_from_slice(&data[..self.n]);
self.rec(&mut data[..self.n], scratch, 1, self.n, 0, 1);
}
fn rec(&self, out: &mut [Cpx], inp: &[Cpx], in_stride: usize, n: usize, fi: usize, tw_stride: usize) {
let p = self.factors[fi];
let m = n / p;
if m == 1 {
for (i, o) in out.iter_mut().enumerate().take(p) {
*o = inp[i * in_stride];
}
} else {
for q in 0..p {
self.rec(&mut out[q * m..(q + 1) * m], &inp[q * in_stride..], in_stride * p, m, fi + 1, tw_stride * p);
}
}
let big = self.n;
let mut t = [Cpx::default(); 5];
for k in 0..m {
for (q, tq) in t.iter_mut().enumerate().take(p) {
*tq = out[q * m + k].mul(self.twiddle[(q * k * tw_stride) % big]);
}
for q2 in 0..p {
let mut acc = Cpx::default();
for (q, tq) in t.iter().enumerate().take(p) {
acc = acc.add(tq.mul(self.twiddle[((q * q2) % p) * (big / p)]));
}
out[q2 * m + k] = acc;
}
}
}
}
use crate::opus::fft::{Cpx, Fft};
/// Forward MDCT producing `n` coefficients from a block of `n + OVERLAP` samples.
pub struct Mdct {
@@ -199,6 +122,31 @@ mod tests {
.collect()
}
#[test]
fn fft_matches_dft() {
// Every radix combination, including the 480-point size of the 20 ms frame.
for &n in &[2usize, 3, 4, 5, 6, 8, 12, 15, 16, 20, 24, 30, 48, 60, 120, 240, 480] {
let fft = Fft::new(n);
let x: Vec<Cpx> = (0..n).map(|i| Cpx { re: ((i * 37 % 101) as f32 - 50.0) / 7.0, im: ((i * 53 % 89) as f32 - 44.0) / 9.0 }).collect();
let mut y = x.clone();
fft.process(&mut y, &mut Vec::new());
for k in 0..n {
let (mut re, mut im) = (0f64, 0f64);
for (j, v) in x.iter().enumerate() {
let ph = -2.0 * std::f64::consts::PI * ((j * k) % n) as f64 / n as f64;
re += v.re as f64 * ph.cos() - v.im as f64 * ph.sin();
im += v.re as f64 * ph.sin() + v.im as f64 * ph.cos();
}
assert!(
(y[k].re as f64 - re).abs() < 2e-3 && (y[k].im as f64 - im).abs() < 2e-3,
"n={n} k={k}: ({}, {}) vs ({re}, {im})",
y[k].re,
y[k].im
);
}
}
}
#[test]
fn matches_direct_formula() {
for &n in &[240usize, 960] {
+36 -33
View File
@@ -10,8 +10,9 @@
//! above in the same range-coded frame; constant packet size.
//! - **CELT** (fullband): MDCT transform coding; constant packet size.
//!
//! From EffectCraft's `effectcraft-opusenc` (storytold/effectcraft@c618429). Public items keep their
//! names, so EffectCraft can re-export this module.
//! From EffectCraft's `effectcraft-opusenc` (storytold/effectcraft@c618429), with its later speed-ups
//! and CELT stage refactor up to storytold/effectcraft@d6c9ad66. Public items keep their names, so
//! EffectCraft can re-export this module.
//!
//! ```
//! # #[cfg(feature = "opus-encode")] {
@@ -26,6 +27,7 @@
//! # }
//! ```
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
// The bit-allocation code mirrors the normative integer arithmetic; keep C-like shift expressions.
#![allow(clippy::precedence, clippy::int_plus_one)]
@@ -36,19 +38,28 @@ mod range;
mod rate;
mod resample;
mod silk;
mod tables;
pub(super) mod tables;
use std::sync::OnceLock;
use bands::{SPREAD_NORMAL, quant_all_bands};
use bands::{BandCoding, SPREAD_NORMAL, quant_all_bands};
use energy::{quant_coarse_energy, quant_energy_finalise, quant_fine_energy};
use mdct::Mdct;
use range::{BITRES, RangeEncoder};
use rate::{Mode as CeltMode, compute_allocation};
use rate::{AllocationInput, Mode as CeltMode, compute_allocation};
use resample::Downsampler;
use silk::SilkEncoder;
use tables::{E_MEANS, EBANDS, NB_EBANDS, OVERLAP, PREEMPH, SPREAD_ICDF, TF_SELECT_TABLE, TRIM_ICDF};
/// Band range, channel count and transform size shared by CELT coding stages.
#[derive(Clone, Copy)]
struct CeltFrame {
start: usize,
end: usize,
channels: usize,
lm: usize,
}
/// Samples per channel in one 20 ms frame at 48 kHz.
const N: usize = 960;
/// LM of a 20 ms frame (8 short blocks).
@@ -338,7 +349,7 @@ impl Encoder {
Mode::Celt => {
let (log_e, silence) = self.celt_analysis(pcm);
let len = self.frame_bytes;
packet.extend_from_slice(&self.encode_celt(RangeEncoder::new(), len, 0, NB_EBANDS, &log_e, silence));
packet.extend_from_slice(&self.encode_celt(RangeEncoder::new(), len, 0..NB_EBANDS, &log_e, silence));
}
Mode::Silk => {
let x = self.silk_input(pcm);
@@ -368,7 +379,7 @@ impl Encoder {
}
let (log_e, _) = self.celt_analysis(pcm);
let end = if self.bandwidth == Bandwidth::SuperWideband { 19 } else { NB_EBANDS };
packet.extend_from_slice(&self.encode_celt(enc, len, HYBRID_START_BAND, end, &log_e, false));
packet.extend_from_slice(&self.encode_celt(enc, len, HYBRID_START_BAND..end, &log_e, false));
}
}
Ok(packet)
@@ -430,10 +441,18 @@ impl Encoder {
/// Codes one CELT frame (RFC 6716 §4.3, in bitstream order) of `len` bytes into `enc`
/// (which already holds the SILK frame in hybrid mode), coding bands `start..end`, and
/// finalises the frame.
#[allow(clippy::too_many_arguments)]
fn encode_celt(&mut self, mut enc: RangeEncoder, len: usize, start: usize, end: usize, log_e: &[f32; 2 * NB_EBANDS], silence: bool) -> Vec<u8> {
fn encode_celt(
&mut self,
mut enc: RangeEncoder,
len: usize,
bands: std::ops::Range<usize>,
log_e: &[f32; 2 * NB_EBANDS],
silence: bool,
) -> Vec<u8> {
let m = celt_mode();
let c = self.channels;
let frame = CeltFrame { start: bands.start, end: bands.end, channels: c, lm: LM };
let CeltFrame { start, end, .. } = frame;
if c == 1 {
for i in 0..NB_EBANDS {
self.old_band_e[i] = self.old_band_e[i].max(self.old_band_e[NB_EBANDS + i]);
@@ -462,10 +481,10 @@ impl Encoder {
let mut enc_intra = enc.clone();
let mut old_intra = self.old_band_e;
enc_intra.bit_logp(true, 3);
quant_coarse_energy(start, end, log_e, &mut old_intra, true, &mut enc_intra, c, LM, total_bits_raw);
quant_coarse_energy(frame, log_e, &mut old_intra, true, &mut enc_intra, total_bits_raw);
enc.bit_logp(false, 3);
let mut old_inter = self.old_band_e;
quant_coarse_energy(start, end, log_e, &mut old_inter, false, &mut enc, c, LM, total_bits_raw);
quant_coarse_energy(frame, log_e, &mut old_inter, false, &mut enc, total_bits_raw);
if enc_intra.tell_frac() < enc.tell_frac() {
enc = enc_intra;
self.old_band_e = old_intra;
@@ -473,14 +492,14 @@ impl Encoder {
self.old_band_e = old_inter;
}
} else {
quant_coarse_energy(start, end, log_e, &mut self.old_band_e, false, &mut enc, c, LM, total_bits_raw);
quant_coarse_energy(frame, log_e, &mut self.old_band_e, false, &mut enc, total_bits_raw);
}
// TF resolution: no changes.
{
let mut budget = total_bits_raw;
let mut tell = enc.tell();
let mut logp = 4;
let tf_select_rsv = LM > 0 && tell + logp + 1 <= budget;
let tf_select_rsv = LM > 0 && tell + logp < budget;
budget -= tf_select_rsv as i32;
for _ in start..end {
if tell + logp <= budget {
@@ -512,27 +531,11 @@ impl Encoder {
enc.icdf(alloc_trim as usize, &TRIM_ICDF, 7);
}
let bits = (total_bits_raw << BITRES) - enc.tell_frac() - 1;
let alloc = compute_allocation(m, start, end, &offsets, &cap, alloc_trim, bits, c, LM, &mut enc);
quant_fine_energy(start, end, log_e, &mut self.old_band_e, &alloc.fine_quant, &mut enc, c);
quant_all_bands(
m,
start,
end,
&self.freq,
N,
c,
&alloc.pulses,
SPREAD_NORMAL,
alloc.dual_stereo,
alloc.intensity,
total_bits_raw << BITRES,
alloc.balance,
&mut enc,
LM,
alloc.coded_bands,
);
let alloc = compute_allocation(m, frame, AllocationInput { offsets: &offsets, cap: &cap, trim: alloc_trim, bits }, &mut enc);
quant_fine_energy(frame, log_e, &mut self.old_band_e, &alloc.fine_quant, &mut enc);
quant_all_bands(m, frame, &self.freq, N, &alloc, BandCoding { spread: SPREAD_NORMAL, total_bits: total_bits_raw << BITRES }, &mut enc);
let bits_left = total_bits_raw - enc.tell();
quant_energy_finalise(start, end, log_e, &mut self.old_band_e, &alloc.fine_quant, &alloc.fine_priority, bits_left, &mut enc, c);
quant_energy_finalise(frame, log_e, &mut self.old_band_e, &alloc, bits_left, &mut enc);
debug_assert!(enc.tell() <= total_bits_raw, "frame overflow: {} > {}", enc.tell(), total_bits_raw);
if c == 1 {
let (a, b) = self.old_band_e.split_at_mut(NB_EBANDS);
+45 -363
View File
@@ -4,254 +4,18 @@
//! spec-derived `filmcraft-opus` decoder (first-party, MIT OR Apache-2.0) step by step; the only
//! difference is that the encoder *writes* the band-skip, intensity and dual-stereo symbols.
use crate::opus::encoder::range::{BITRES, RangeEncoder, ilog};
use crate::opus::encoder::tables::{ALLOC, EBANDS, LOG2_FRAC_TABLE, MAX_LM, NB_EBANDS};
use crate::opus::encoder::CeltFrame;
use crate::opus::encoder::range::{BITRES, RangeEncoder};
use crate::opus::encoder::tables::{EBANDS, LOG2_FRAC_TABLE, NB_EBANDS};
const MAX_PSEUDO: i32 = 40;
const LOG_MAX_PSEUDO: i32 = 6;
#[cfg(test)]
pub use crate::opus::rate_common::pvq_v;
pub use crate::opus::rate_common::{Mode, get_pulses};
pub const MAX_FINE_BITS: i32 = 8;
const FINE_OFFSET: i32 = 21;
pub const QTHETA_OFFSET: i32 = 4;
pub const QTHETA_OFFSET_TWOPHASE: i32 = 16;
const ALLOC_STEPS: i32 = 6;
/// Number of pulses represented by pseudo-pulse index `i`.
pub fn get_pulses(i: i32) -> i32 {
if i < 8 { i } else { (8 + (i & 7)) << ((i >> 3) - 1) }
}
/// Conservative (rounded up) log2 of `val` with `frac` fractional bits.
pub fn log2_frac(mut val: u32, mut frac: u32) -> i32 {
let mut l = ilog(val);
if val & val.wrapping_sub(1) != 0 {
if l > 16 {
val = ((val - 1) >> (l - 16)) + 1;
} else {
val <<= 16 - l;
}
let mut l = (l - 1) << frac;
loop {
let b = (val >> 16) as i32;
l += b << frac;
val = (val + b as u32) >> b;
val = (val * val + 0x7FFF) >> 15;
if frac == 0 {
break;
}
frac -= 1;
}
l + (val > 0x8000) as i32
} else {
l -= 1;
l << frac
}
}
/// PVQ codebook size V(N, K) (saturating far above 2^32).
pub fn pvq_v(n: usize, k: usize) -> u128 {
if k == 0 {
return 1;
}
if n == 0 {
return 0;
}
let mut row = vec![0u128; k + 1];
row[0] = 1;
for _ in 0..n {
let mut prev_diag = row[0];
for kk in 1..=k {
let up = row[kk];
let v = up.saturating_add(row[kk - 1]).saturating_add(prev_diag);
prev_diag = up;
row[kk] = v.min(u128::MAX >> 8);
}
row[0] = 1;
}
row[k]
}
fn fits_in32(n: usize, k: i32) -> bool {
pvq_v(n, k as usize) <= u32::MAX as u128
}
/// Precomputed per-mode data.
pub struct Mode {
/// log2 of band width in 1/8 bits.
pub log_n: [i32; NB_EBANDS],
cache_index: Vec<i16>,
cache_bits: Vec<u8>,
caps: Vec<u8>,
}
impl Mode {
pub fn new() -> Mode {
let mut log_n = [0i32; NB_EBANDS];
for (i, l) in log_n.iter_mut().enumerate() {
*l = log2_frac((EBANDS[i + 1] - EBANDS[i]) as u32, BITRES);
}
let (cache_index, cache_bits) = compute_pulse_cache(MAX_LM as i32);
let mut m = Mode { log_n, cache_index, cache_bits, caps: Vec::new() };
m.caps = compute_caps(&m);
m
}
#[inline]
pub fn cache(&self, lm: i32, band: usize) -> &[u8] {
let idx = self.cache_index[((lm + 1) as usize) * NB_EBANDS + band];
&self.cache_bits[idx.max(0) as usize..]
}
pub fn bits2pulses(&self, band: usize, lm: i32, bits: i32) -> i32 {
let cache = self.cache(lm, band);
let mut lo = 0i32;
let mut hi = cache[0] as i32;
let bits = bits - 1;
for _ in 0..LOG_MAX_PSEUDO {
let mid = (lo + hi + 1) >> 1;
if cache[mid as usize] as i32 >= bits {
hi = mid;
} else {
lo = mid;
}
}
let lo_bits = if lo == 0 { -1 } else { cache[lo as usize] as i32 };
if bits - lo_bits <= cache[hi as usize] as i32 - bits { lo } else { hi }
}
pub fn pulses2bits(&self, band: usize, lm: i32, pulses: i32) -> i32 {
if pulses == 0 { 0 } else { self.cache(lm, band)[pulses as usize] as i32 + 1 }
}
/// Per-band allocation caps for this frame (`init_caps`).
pub fn init_caps(&self, lm: usize, c: usize) -> [i32; NB_EBANDS] {
let mut cap = [0i32; NB_EBANDS];
for (i, cp) in cap.iter_mut().enumerate() {
let n = (EBANDS[i + 1] - EBANDS[i]) << lm;
*cp = (self.caps[NB_EBANDS * (2 * lm + c - 1) + i] as i32 + 64) * c as i32 * n >> 2;
}
cap
}
}
fn compute_pulse_cache(lm_max: i32) -> (Vec<i16>, Vec<u8>) {
let nb = NB_EBANDS;
let rows = (lm_max + 2) as usize;
let mut index = vec![-1i16; nb * rows];
let mut entries: Vec<(usize, i32, usize)> = Vec::new();
let mut curr = 0usize;
for i in 0..rows {
for j in 0..nb {
let n = (((EBANDS[j + 1] - EBANDS[j]) << i) >> 1) as usize;
index[i * nb + j] = -1;
'search: for k in 0..=i {
for nn in 0..nb {
if k == i && nn >= j {
break;
}
if n == (((EBANDS[nn + 1] - EBANDS[nn]) << k) >> 1) as usize {
index[i * nb + j] = index[k * nb + nn];
break 'search;
}
}
}
if index[i * nb + j] == -1 && n != 0 {
let mut k = 0;
while k < MAX_PSEUDO && fits_in32(n, get_pulses(k + 1)) {
k += 1;
}
index[i * nb + j] = curr as i16;
entries.push((n, k, curr));
curr += (k + 1) as usize;
}
}
}
let mut bits = vec![0u8; curr];
for &(n, k, off) in &entries {
let maxk = get_pulses(k) as usize;
let tmp = required_bits(n, maxk, BITRES);
for j in 1..=k {
bits[off + j as usize] = (tmp[get_pulses(j) as usize] - 1) as u8;
}
bits[off] = k as u8;
}
(index, bits)
}
fn required_bits(n: usize, maxk: usize, frac: u32) -> Vec<i32> {
let mut out = vec![0i32; maxk + 1];
if n == 1 {
for o in out.iter_mut().skip(1) {
*o = 1 << frac;
}
} else {
for (k, o) in out.iter_mut().enumerate().skip(1) {
*o = log2_frac(pvq_v(n, k) as u32, frac);
}
}
out
}
fn compute_caps(m: &Mode) -> Vec<u8> {
let nb = NB_EBANDS;
let mut caps = vec![0u8; (MAX_LM + 1) * 2 * nb];
for i in 0..=MAX_LM as i32 {
for c in 1..=2i32 {
for j in 0..nb {
let mut n0 = EBANDS[j + 1] - EBANDS[j];
let max_bits = if n0 << i == 1 {
c * (1 + MAX_FINE_BITS) << BITRES
} else {
let mut lm0 = 0;
if n0 > 2 {
n0 >>= 1;
lm0 -= 1;
} else if n0 <= 1 {
lm0 = i.min(1);
n0 <<= lm0;
}
let pcache = m.cache(lm0, j);
let mut mb = pcache[pcache[0] as usize] as i32 + 1;
let mut n = n0;
for k in 0..(i - lm0) {
mb <<= 1;
let offset = ((m.log_n[j] + ((lm0 + k) << BITRES)) >> 1) - QTHETA_OFFSET;
let num = 459 * ((2 * n - 1) * offset + mb);
let den = ((2 * n - 1) << 9) - 459;
let qb = ((num + (den >> 1)) / den).min(57);
mb += qb;
n <<= 1;
}
if c == 2 {
mb <<= 1;
let offset = ((m.log_n[j] + (i << BITRES)) >> 1) - if n == 2 { QTHETA_OFFSET_TWOPHASE } else { QTHETA_OFFSET };
let ndof = 2 * n - 1 - (n == 2) as i32;
let p = if n == 2 { 512 } else { 487 };
let num = p * (mb + ndof * offset);
let den = (ndof << 9) - p;
let qb = ((num + (den >> 1)) / den).min(if n == 2 { 64 } else { 61 });
mb += qb;
}
let ndof = c * n + if c == 2 && n > 2 { 1 } else { 0 };
let mut offset = ((m.log_n[j] + (i << BITRES)) >> 1) - FINE_OFFSET;
if n == 2 {
offset += (1 << BITRES) >> 2;
}
let num = mb + ndof * offset;
let den = (ndof - 1) << BITRES;
let qb = ((num + (den >> 1)) / den).min(MAX_FINE_BITS);
mb += c * qb << BITRES;
mb
};
let w = c * ((EBANDS[j + 1] - EBANDS[j]) << i);
let v = (4 * max_bits / w) - 64;
caps[((i * 2 + c - 1) as usize) * nb + j] = v.clamp(0, 255) as u8;
}
}
}
caps
}
/// Result of the allocation process.
pub struct Allocation {
pub coded_bands: usize,
@@ -263,125 +27,43 @@ pub struct Allocation {
pub fine_priority: [i32; NB_EBANDS],
}
/// `clt_compute_allocation`, encoder side. Stereo frames are always coded as dual (L/R) stereo
/// with no intensity bands, and no band is skipped voluntarily.
#[allow(clippy::too_many_arguments)]
pub fn compute_allocation(
m: &Mode,
start: usize,
end: usize,
offsets: &[i32; NB_EBANDS],
cap: &[i32; NB_EBANDS],
alloc_trim: i32,
total: i32,
c: usize,
lm: usize,
enc: &mut RangeEncoder,
) -> Allocation {
let ci = c as i32;
let lmi = lm as i32;
let mut total = total.max(0);
let mut skip_start = start;
let skip_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 };
total -= skip_rsv;
let mut intensity_rsv = 0;
let mut dual_stereo_rsv = 0;
if c == 2 {
intensity_rsv = LOG2_FRAC_TABLE[end - start];
if intensity_rsv > total {
intensity_rsv = 0;
} else {
total -= intensity_rsv;
dual_stereo_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 };
total -= dual_stereo_rsv;
}
}
let mut bits1 = [0i32; NB_EBANDS];
let mut bits2 = [0i32; NB_EBANDS];
let mut thresh = [0i32; NB_EBANDS];
let mut trim_offset = [0i32; NB_EBANDS];
for j in start..end {
let w = EBANDS[j + 1] - EBANDS[j];
thresh[j] = (ci << BITRES).max((3 * w << lmi << BITRES) >> 4);
trim_offset[j] = ci * w * (alloc_trim - 5 - lmi) * (end as i32 - j as i32 - 1) * (1 << (lmi + BITRES as i32)) >> 6;
if w << lmi == 1 {
trim_offset[j] -= ci << BITRES;
}
}
let nb_alloc = ALLOC.len() as i32;
let mut lo = 1;
let mut hi = nb_alloc - 1;
loop {
let mut done = false;
let mut psum = 0;
let mid = (lo + hi) >> 1;
for j in (start..end).rev() {
let n = EBANDS[j + 1] - EBANDS[j];
let mut bitsj = ci * n * (ALLOC[mid as usize][j] as i32) << lmi >> 2;
if bitsj > 0 {
bitsj = (bitsj + trim_offset[j]).max(0);
}
bitsj += offsets[j];
if bitsj >= thresh[j] || done {
done = true;
psum += bitsj.min(cap[j]);
} else if bitsj >= ci << BITRES {
psum += ci << BITRES;
}
}
if psum > total {
hi = mid - 1;
} else {
lo = mid + 1;
}
if lo > hi {
break;
}
}
hi = lo;
lo -= 1;
for j in start..end {
let n = EBANDS[j + 1] - EBANDS[j];
let mut bits1j = ci * n * (ALLOC[lo as usize][j] as i32) << lmi >> 2;
let mut bits2j = if hi >= nb_alloc { cap[j] } else { ci * n * (ALLOC[hi as usize][j] as i32) << lmi >> 2 };
if bits1j > 0 {
bits1j = (bits1j + trim_offset[j]).max(0);
}
if bits2j > 0 {
bits2j = (bits2j + trim_offset[j]).max(0);
}
if lo > 0 {
bits1j += offsets[j];
}
bits2j += offsets[j];
if offsets[j] > 0 {
skip_start = j;
}
bits2j = (bits2j - bits1j).max(0);
bits1[j] = bits1j;
bits2[j] = bits2j;
}
interp_bits2pulses(m, start, end, skip_start, &bits1, &bits2, &thresh, cap, total, skip_rsv, intensity_rsv, dual_stereo_rsv, c, lm, enc)
/// Per-band limits, boosts and available budget for the allocation search.
pub struct AllocationInput<'a> {
pub offsets: &'a [i32; NB_EBANDS],
pub cap: &'a [i32; NB_EBANDS],
pub trim: i32,
pub bits: i32,
}
#[allow(clippy::too_many_arguments)]
fn interp_bits2pulses(
m: &Mode,
start: usize,
end: usize,
/// `clt_compute_allocation`, encoder side. Stereo frames are always coded as dual (L/R) stereo
/// with no intensity bands, and no band is skipped voluntarily.
pub fn compute_allocation(m: &Mode, frame: CeltFrame, input: AllocationInput<'_>, enc: &mut RangeEncoder) -> Allocation {
let CeltFrame { start, end, channels: c, lm } = frame;
let AllocationInput { offsets, cap, trim: alloc_trim, bits: total } = input;
let crate::opus::rate_common::Bracket { skip_start, bits1, bits2, thresh, total, skip_rsv, intensity_rsv, dual_stereo_rsv } =
crate::opus::rate_common::bracket(start, end, offsets, cap, alloc_trim, total, c, lm);
let interpolation =
AllocationInterpolation { skip_start, bits1: &bits1, bits2: &bits2, thresh: &thresh, cap, total, skip_rsv, intensity_rsv, dual_stereo_rsv };
interp_bits2pulses(m, frame, interpolation, enc)
}
/// Bracketing allocation vectors and signalling reserves for the interpolation stage.
struct AllocationInterpolation<'a> {
skip_start: usize,
bits1: &[i32; NB_EBANDS],
bits2: &[i32; NB_EBANDS],
thresh: &[i32; NB_EBANDS],
cap: &[i32; NB_EBANDS],
mut total: i32,
bits1: &'a [i32; NB_EBANDS],
bits2: &'a [i32; NB_EBANDS],
thresh: &'a [i32; NB_EBANDS],
cap: &'a [i32; NB_EBANDS],
total: i32,
skip_rsv: i32,
mut intensity_rsv: i32,
mut dual_stereo_rsv: i32,
c: usize,
lm: usize,
enc: &mut RangeEncoder,
) -> Allocation {
intensity_rsv: i32,
dual_stereo_rsv: i32,
}
fn interp_bits2pulses(m: &Mode, frame: CeltFrame, interpolation: AllocationInterpolation<'_>, enc: &mut RangeEncoder) -> Allocation {
let CeltFrame { start, end, channels: c, lm } = frame;
let AllocationInterpolation { skip_start, bits1, bits2, thresh, cap, mut total, skip_rsv, mut intensity_rsv, mut dual_stereo_rsv } =
interpolation;
let ci = c as i32;
let alloc_floor = ci << BITRES;
let stereo = (c > 1) as i32;
@@ -393,7 +75,7 @@ fn interp_bits2pulses(
let mut psum = 0;
let mut done = false;
for j in (start..end).rev() {
let tmp = bits1[j] + (mid * bits2[j] >> ALLOC_STEPS);
let tmp = bits1[j] + ((mid * bits2[j]) >> ALLOC_STEPS);
if tmp >= thresh[j] || done {
done = true;
psum += tmp.min(cap[j]);
@@ -411,7 +93,7 @@ fn interp_bits2pulses(
let mut psum = 0;
let mut done = false;
for j in (start..end).rev() {
let mut tmp = bits1[j] + (lo * bits2[j] >> ALLOC_STEPS);
let mut tmp = bits1[j] + ((lo * bits2[j]) >> ALLOC_STEPS);
if tmp < thresh[j] && !done {
tmp = if tmp >= alloc_floor { alloc_floor } else { 0 };
} else {
@@ -500,9 +182,9 @@ fn interp_bits2pulses(
if n == 2 {
offset += (den << BITRES) >> 2;
}
if bits[j] + offset < den * 2 << BITRES {
if bits[j] + offset < (den * 2) << BITRES {
offset += nclogn >> 2;
} else if bits[j] + offset < den * 3 << BITRES {
} else if bits[j] + offset < (den * 3) << BITRES {
offset += nclogn >> 3;
}
let mut eb = (bits[j] + offset + (den << (BITRES - 1))).max(0);
@@ -513,7 +195,7 @@ fn interp_bits2pulses(
eb = eb.min(MAX_FINE_BITS);
ebits[j] = eb;
fine_priority[j] = (eb * (den << BITRES) >= bits[j] + offset) as i32;
bits[j] -= ci * eb << BITRES;
bits[j] -= (ci * eb) << BITRES;
} else {
excess = (bit - (ci << BITRES)).max(0);
bits[j] = bit - excess;
@@ -523,7 +205,7 @@ fn interp_bits2pulses(
if excess > 0 {
let extra_fine = (excess >> (stereo + BITRES as i32)).min(MAX_FINE_BITS - ebits[j]);
ebits[j] += extra_fine;
let extra_bits = extra_fine * ci << BITRES;
let extra_bits = (extra_fine * ci) << BITRES;
fine_priority[j] = (extra_bits >= excess - balance) as i32;
excess -= extra_bits;
}
+13 -1
View File
@@ -45,6 +45,8 @@ fn design(fc: f64, beta: f64, delay: usize) -> Vec<f32> {
h.iter().map(|&v| v as f32).collect()
}
use crate::opus::dot::dot8;
/// Decimates one channel of 48 kHz audio by 3, 4 or 6.
#[derive(Clone)]
pub struct Downsampler {
@@ -77,7 +79,7 @@ impl Downsampler {
let t = taps - 1 + self.factor * j;
let win = &self.buf[t + 1 - taps..=t];
// h is symmetric, so the window can be taken in forward order.
out.push(win.iter().zip(&self.h).map(|(a, b)| a * b).sum());
out.push(dot8(win, &self.h));
}
let n = self.buf.len();
self.buf.copy_within(n - (taps - 1).., 0);
@@ -97,6 +99,16 @@ mod tests {
(x.iter().map(|&v| (v as f64).powi(2)).sum::<f64>() / x.len() as f64).sqrt()
}
#[test]
fn dot8_matches_sequential_sum() {
for len in [0usize, 1, 7, 8, 9, 63, 167, 251] {
let a: Vec<f32> = (0..len).map(|i| ((i * 37 % 101) as f32 - 50.0) / 50.0).collect();
let b: Vec<f32> = (0..len).map(|i| ((i * 53 % 89) as f32 - 44.0) / 44.0).collect();
let exact: f64 = a.iter().zip(&b).map(|(&x, &y)| x as f64 * y as f64).sum();
assert!((dot8(&a, &b) as f64 - exact).abs() < 1e-4, "len {len}");
}
}
#[test]
fn passes_band_and_rejects_aliases() {
for &(fs, pass, stop) in &[(16_000u32, 6_000.0, 8_600.0), (12_000, 4_500.0, 6_500.0), (8_000, 3_000.0, 4_500.0)] {
+104 -8
View File
@@ -69,28 +69,55 @@ pub fn levinson(r: &[f64], order: usize) -> Vec<f64> {
}
/// Evaluates the symmetric polynomial `p` (length `d + 1`, `d` even) as the cosine series
/// `e^{jωd/2} P(e^{jω})`.
fn cos_series(p: &[f64], w: f64) -> f64 {
/// `e^{jωd/2} P(e^{jω})`, with `cos_m(m)` giving `cos(m ω)`.
#[inline]
fn cos_series_with(p: &[f64], cos_m: impl Fn(usize) -> f64) -> f64 {
let h = (p.len() - 1) / 2;
let mut s = p[h];
for k in 0..h {
s += 2.0 * p[k] * ((h - k) as f64 * w).cos();
s += 2.0 * p[k] * cos_m(h - k);
}
s
}
fn cos_series(p: &[f64], w: f64) -> f64 {
cos_series_with(p, |m| (m as f64 * w).cos())
}
/// Number of grid intervals the root search scans over `[0, π]`.
const GRID: usize = 1024;
/// Largest multiple of ω in the series (half the LPC order, at most 16 / 2).
const MAX_H: usize = 8;
/// Bisection steps per root: the bracket shrinks to 3e-9 rad, far below one Q15 NLSF step (9.6e-5 rad).
const BISECT_STEPS: usize = 20;
/// `cos(m · π g / GRID)` for every grid point `g` and `m` in `0..=MAX_H`, computed with the same
/// expression as [`cos_series`], so the table values are bit-identical to calling `cos` there.
fn grid_cos() -> &'static [[f64; MAX_H + 1]] {
static TABLE: std::sync::OnceLock<Vec<[f64; MAX_H + 1]>> = std::sync::OnceLock::new();
TABLE.get_or_init(|| {
(0..=GRID)
.map(|g| {
let w = std::f64::consts::PI * g as f64 / GRID as f64;
std::array::from_fn(|m| (m as f64 * w).cos())
})
.collect()
})
}
fn roots(p: &[f64], out: &mut Vec<f64>) {
const GRID: usize = 1024;
let grid = grid_cos();
let at_grid = |g: usize| cos_series_with(p, |m| grid[g][m]);
let mut w0 = 0.0;
let mut f0 = cos_series(p, w0);
let mut f0 = at_grid(0);
for g in 1..=GRID {
let w1 = std::f64::consts::PI * g as f64 / GRID as f64;
let f1 = cos_series(p, w1);
let f1 = at_grid(g);
if f0 == 0.0 {
out.push(w0);
} else if f0 * f1 < 0.0 {
let (mut lo, mut hi, mut flo) = (w0, w1, f0);
for _ in 0..40 {
for _ in 0..BISECT_STEPS {
let mid = 0.5 * (lo + hi);
let fm = cos_series(p, mid);
if fm * flo <= 0.0 {
@@ -111,6 +138,9 @@ fn roots(p: &[f64], out: &mut Vec<f64>) {
/// when the root search fails (unstable or degenerate filter).
pub fn a2nlsf(a: &[f64]) -> Option<Vec<f64>> {
let d = a.len();
if d > 2 * MAX_H {
return None;
}
let mut c = vec![0f64; d + 2];
c[0] = 1.0;
for k in 0..d {
@@ -199,9 +229,20 @@ pub fn pitch_search(x: &[f32], start: usize, len: usize, fs_khz: usize) -> Pitch
let score = full[(lag - min_lag) as usize];
// Contour: per-subframe lag offsets around the chosen lag.
let cb: &[[i32; 4]] = if fs_khz == 8 { &CB_NB_20 } else { &CB_WB_20 };
// The contour search revisits the same (subframe, lag) pairs for neighbouring base lags and
// codebook entries, so each correlation is computed once.
let span = (18 * fs - min_lag + 1) as usize;
let memo = std::cell::RefCell::new(vec![f64::NAN; 4 * span]);
let sub_score = |k: usize, l: i32| -> f64 {
let l = l.clamp(min_lag, 18 * fs);
ncorr(x, start + k * sub..start + (k + 1) * sub, l as usize)
let slot = k * span + (l - min_lag) as usize;
let cached = memo.borrow()[slot];
if !cached.is_nan() {
return cached;
}
let v = ncorr(x, start + k * sub..start + (k + 1) * sub, l as usize);
memo.borrow_mut()[slot] = v;
v
};
let mut best = (f64::MIN, lag, 0usize);
for l0 in (lag - 2).max(min_lag)..=(lag + 2).min(max_lag) {
@@ -319,4 +360,59 @@ mod tests {
}
}
}
/// The root search as it was before the grid table: `cos` called for every evaluation.
fn roots_reference(p: &[f64], out: &mut Vec<f64>) {
let mut w0 = 0.0;
let mut f0 = cos_series(p, w0);
for g in 1..=GRID {
let w1 = std::f64::consts::PI * g as f64 / GRID as f64;
let f1 = cos_series(p, w1);
if f0 == 0.0 {
out.push(w0);
} else if f0 * f1 < 0.0 {
let (mut lo, mut hi, mut flo) = (w0, w1, f0);
for _ in 0..BISECT_STEPS {
let mid = 0.5 * (lo + hi);
let fm = cos_series(p, mid);
if fm * flo <= 0.0 {
hi = mid;
} else {
lo = mid;
flo = fm;
}
}
out.push(0.5 * (lo + hi));
}
w0 = w1;
f0 = f1;
}
}
#[test]
fn grid_table_roots_are_bit_identical_to_direct_cos() {
let mut seed = 7u64;
let mut rnd = move || {
seed ^= seed << 13;
seed ^= seed >> 7;
seed ^= seed << 17;
(seed >> 11) as f64 / (1u64 << 53) as f64 * 2.0 - 1.0
};
let mut found = 0;
for len in [11usize, 17] {
for _ in 0..300 {
let mut p = vec![0f64; len];
for k in 0..len / 2 + 1 {
p[k] = rnd();
p[len - 1 - k] = p[k];
}
let (mut a, mut b) = (Vec::new(), Vec::new());
roots(&p, &mut a);
roots_reference(&p, &mut b);
assert_eq!(a.iter().map(|v| v.to_bits()).collect::<Vec<_>>(), b.iter().map(|v| v.to_bits()).collect::<Vec<_>>());
found += a.len();
}
}
assert!(found > 1000, "{found}");
}
}
+158 -10
View File
@@ -129,6 +129,48 @@ struct Coded {
pulses: Vec<i32>,
}
/// Noise-feedback history (decoded − input), most recent first, as a double-length ring: each
/// sample is stored at `pos` and `pos + MAX_LPC_ORDER`, so `buf[pos..pos + MAX_LPC_ORDER]` is
/// always the contiguous window and a push costs two stores instead of a shift.
struct NfRing {
buf: [f64; 2 * MAX_LPC_ORDER],
/// Index of the most recent sample, always below `MAX_LPC_ORDER`.
pos: usize,
}
impl NfRing {
fn new(hist: &[f64; MAX_LPC_ORDER]) -> Self {
let mut buf = [0.0; 2 * MAX_LPC_ORDER];
buf[..MAX_LPC_ORDER].copy_from_slice(hist);
buf[MAX_LPC_ORDER..].copy_from_slice(hist);
NfRing { buf, pos: 0 }
}
/// The history, most recent first.
fn window(&self) -> &[f64] {
self.buf.get(self.pos..self.pos + MAX_LPC_ORDER).unwrap_or(&[])
}
/// Makes `v` the most recent sample and drops the oldest.
fn push(&mut self, v: f64) {
self.pos = self.pos.checked_sub(1).unwrap_or(MAX_LPC_ORDER - 1);
for i in [self.pos, self.pos + MAX_LPC_ORDER] {
if let Some(slot) = self.buf.get_mut(i) {
*slot = v;
}
}
}
/// The history as a plain array, most recent first.
fn to_array(&self) -> [f64; MAX_LPC_ORDER] {
let mut out = [0.0; MAX_LPC_ORDER];
for (o, v) in out.iter_mut().zip(self.window()) {
*o = *v;
}
out
}
}
#[derive(Clone)]
struct Channel {
fs_khz: usize,
@@ -237,7 +279,7 @@ impl Channel {
let sub = self.sub;
let order = self.order;
let st = &mut self.dec;
let nf = &mut self.nf;
let mut ring = NfRing::new(&self.nf);
let off = if p.signal_type == 0 { offset + INACTIVE_OFFSET } else { offset };
let gamma = nf_gamma(off);
let mut nf_c = [0f64; MAX_LPC_ORDER];
@@ -355,7 +397,7 @@ impl Channel {
}
let lpc_q14 = lpc_pred.saturating_mul(16);
// Target excitation (Q14) for this sample.
let fb: f64 = (0..order).map(|j| nf_c[j] * nf[j]).sum();
let fb: f64 = nf_c.iter().zip(ring.window()).take(order).map(|(c, e)| c * e).sum();
let tgt = (x[n] as f64 + fb).clamp(-32768.0, 32767.0) * inv_g - lpc_q14 as f64 - ltp_q14 as f64;
let sgn = if seed_v < 0 { -1.0 } else { 1.0 };
let u = sgn * tgt;
@@ -394,11 +436,11 @@ impl Channel {
let v = res.saturating_add(lpc_q14);
s_lpc[MAX_LPC_ORDER + i] = v;
xq[n] = sat16(rshift_round(smulww(v, gain_q10), 8)) as i16;
nf.copy_within(0..MAX_LPC_ORDER - 1, 1);
nf[0] = xq[n] as f64 - x[n] as f64;
ring.push(xq[n] as f64 - x[n] as f64);
}
s_lpc.copy_within(sub..sub + MAX_LPC_ORDER, 0);
}
self.nf = ring.to_array();
st.s_lpc_q14.copy_from_slice(&s_lpc[..MAX_LPC_ORDER]);
st.out_buf[..l].copy_from_slice(&xq);
(Coded { gain_ix, gains_q16, seed, pulses }, xq)
@@ -485,6 +527,14 @@ struct Prep {
pred_q13: [i32; 2],
}
/// One rate-loop trial: the frame coded at gain offset `offset`.
struct Trial {
offset: f64,
bits: i32,
state: SilkEncoder,
coder: RangeEncoder,
}
/// SILK encoder for one mono or stereo (mid/side) stream.
#[derive(Clone)]
pub struct SilkEncoder {
@@ -635,11 +685,20 @@ impl SilkEncoder {
pub fn encode(&mut self, input: &[Vec<f32>], target_bits: f64, max_bits: i32, enc: &mut RangeEncoder) {
let prep = self.prepare(input);
let start_bits = enc.tell();
let trial = |s: &Self, o: f64| -> i32 {
let mut t = s.clone();
let mut e = enc.clone();
t.code(&prep, o, &mut e);
e.tell() - start_bits
// Every trial codes the frame into a clone of the encoder and the range coder. The result
// is kept per offset: the hard-limit check below hits the memo for the chosen offset, and
// the final step adopts that trial instead of coding the frame once more.
let mut trials: Vec<Trial> = Vec::new();
let mut trial = |s: &Self, o: f64| -> i32 {
if let Some(t) = trials.iter().find(|t| t.offset == o) {
return t.bits;
}
let mut state = s.clone();
let mut coder = enc.clone();
state.code(&prep, o, &mut coder);
let bits = coder.tell() - start_bits;
trials.push(Trial { offset: o, bits, state, coder });
bits
};
// Bisection on the gain offset: the smallest offset (finest quantisation) within budget.
let target = target_bits.min(max_bits as f64);
@@ -701,8 +760,20 @@ impl SilkEncoder {
while trial(self, o) > max_bits && o < 12.0 {
o += 1.0;
}
match trials.iter().position(|t| t.offset == o) {
Some(i) => {
let t = trials.swap_remove(i);
*self = t.state;
*enc = t.coder;
}
None => self.code(&prep, o, enc),
}
self.end_frame(&prep, o);
}
/// Advances the input history and the frame counter after frame `prep` was coded at `o`.
fn end_frame(&mut self, prep: &Prep, o: f64) {
self.last_offset = o;
self.code(&prep, o, enc);
for ch in 0..self.channels {
self.ch[ch].push_history(&prep.x[ch]);
}
@@ -712,3 +783,80 @@ impl SilkEncoder {
self.frame += 1;
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Deterministic speech-like test input: two harmonics with a moving pitch, a syllable
/// envelope, noise and short silences, at the internal rate and 16-bit scale.
fn signal(len: usize, fs_khz: usize, seed: u32) -> Vec<f32> {
let mut s = seed;
let fs = (fs_khz * 1000) as f64;
(0..len)
.map(|i| {
s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
let noise = (s >> 8) as f64 / (1u32 << 24) as f64 - 0.5;
let t = i as f64 / fs;
let f0 = 140.0 + 40.0 * (t * 1.3).sin();
let env = ((t * 4.0).sin().max(0.0)).powf(0.7);
let voiced = (std::f64::consts::TAU * f0 * t).sin() + 0.5 * (std::f64::consts::TAU * 2.0 * f0 * t).sin();
(8000.0 * env * voiced + 600.0 * noise) as f32
})
.collect()
}
#[test]
fn nf_ring_matches_a_shifted_history() {
let mut hist = [0f64; MAX_LPC_ORDER];
for (i, h) in hist.iter_mut().enumerate() {
*h = i as f64 * 0.5 - 3.0;
}
let mut ring = NfRing::new(&hist);
assert_eq!(ring.window(), &hist[..]);
for n in 0..100 {
let v = (n as f64 * 0.37).sin() * 100.0;
hist.copy_within(0..MAX_LPC_ORDER - 1, 1);
hist[0] = v;
ring.push(v);
assert_eq!(ring.window(), &hist[..], "after push {n}");
}
assert_eq!(ring.to_array(), hist);
}
/// The rate loop adopts the memoised trial for the chosen offset. A second encoder that codes
/// every frame afresh at that offset (what the loop did before the memo) must produce the
/// same bytes, frame after frame, so the adopted encoder and range-coder states are exact.
#[test]
fn adopting_the_memoised_trial_equals_coding_the_frame_again() {
for (fs_khz, channels, target_bits, max_bits) in
[(8, 1, 200.0, 1000), (12, 1, 320.0, 1000), (16, 1, 480.0, 1000), (16, 2, 640.0, 1000), (16, 1, 900.0, 400), (8, 2, 120.0, 160)]
{
let mut memo = SilkEncoder::new(fs_khz, channels);
let mut fresh = memo.clone();
let l = memo.frame_len();
let frames = 40;
let left = signal(l * frames, fs_khz, 7);
let right: Vec<f32> = signal(l * frames + 37, fs_khz, 11)[37..].iter().map(|v| v * 0.6).collect();
for f in 0..frames {
let mut input = vec![left[f * l..(f + 1) * l].to_vec()];
if channels == 2 {
input.push(right[f * l..(f + 1) * l].to_vec());
}
let mut enc = RangeEncoder::new();
memo.encode(&input, target_bits, max_bits, &mut enc);
let o = memo.last_offset;
let prep = fresh.prepare(&input);
let mut enc_fresh = RangeEncoder::new();
fresh.code(&prep, o, &mut enc_fresh);
fresh.end_frame(&prep, o);
assert_eq!(enc.tell(), enc_fresh.tell(), "{fs_khz} kHz {channels} ch frame {f}");
assert_eq!(enc.done(1275), enc_fresh.done(1275), "{fs_khz} kHz {channels} ch frame {f}");
for c in 0..channels {
assert_eq!(memo.ch[c].nf, fresh.ch[c].nf, "noise feedback, channel {c}, frame {f}");
assert_eq!(memo.ch[c].dec.s_lpc_q14, fresh.ch[c].dec.s_lpc_q14, "LPC state, channel {c}, frame {f}");
}
}
}
}
}
+130
View File
@@ -0,0 +1,130 @@
//! Shared CELT forward complex FFT.
#[derive(Clone, Copy, Default, Debug)]
pub struct Cpx {
pub re: f32,
pub im: f32,
}
impl Cpx {
#[inline]
pub(crate) fn mul(self, o: Cpx) -> Cpx {
Cpx { re: self.re * o.re - self.im * o.im, im: self.re * o.im + self.im * o.re }
}
#[inline]
pub(crate) fn add(self, o: Cpx) -> Cpx {
Cpx { re: self.re + o.re, im: self.im + o.im }
}
#[inline]
pub(crate) fn sub(self, o: Cpx) -> Cpx {
Cpx { re: self.re - o.re, im: self.im - o.im }
}
}
/// Mixed-radix (2, 3, 4, 5) forward complex FFT, unscaled (decimation in time, recursive).
pub struct Fft {
pub(crate) n: usize,
pub(crate) factors: Vec<usize>,
pub(crate) twiddle: Vec<Cpx>,
}
impl Fft {
pub fn new(n: usize) -> Fft {
let mut factors = Vec::new();
let mut m = n;
for p in [4usize, 2, 3, 5] {
while m.is_multiple_of(p) {
factors.push(p);
m /= p;
}
}
assert_eq!(m, 1, "unsupported FFT size {n}");
let twiddle = (0..n)
.map(|k| {
let ph = -2.0 * std::f64::consts::PI * k as f64 / n as f64;
Cpx { re: ph.cos() as f32, im: ph.sin() as f32 }
})
.collect();
Fft { n, factors, twiddle }
}
pub fn process(&self, data: &mut [Cpx], scratch: &mut Vec<Cpx>) {
scratch.clear();
scratch.extend_from_slice(&data[..self.n]);
self.rec(&mut data[..self.n], scratch, 1, self.n, 0, 1);
}
fn rec(&self, out: &mut [Cpx], inp: &[Cpx], in_stride: usize, n: usize, fi: usize, tw_stride: usize) {
let p = self.factors[fi];
let m = n / p;
if m == 1 {
for (i, o) in out.iter_mut().enumerate().take(p) {
*o = inp[i * in_stride];
}
} else {
for q in 0..p {
self.rec(&mut out[q * m..(q + 1) * m], &inp[q * in_stride..], in_stride * p, m, fi + 1, tw_stride * p);
}
}
// The twiddle index q * k * tw_stride is always below n, so no modulo is needed.
let tw = &self.twiddle;
match p {
2 => {
for k in 0..m {
let a0 = out[k];
let a1 = out[m + k].mul(tw[k * tw_stride]);
out[k] = a0.add(a1);
out[m + k] = a0.sub(a1);
}
}
4 => {
for k in 0..m {
let t0 = out[k];
let t1 = out[m + k].mul(tw[k * tw_stride]);
let t2 = out[2 * m + k].mul(tw[2 * k * tw_stride]);
let t3 = out[3 * m + k].mul(tw[3 * k * tw_stride]);
let (s0, d0) = (t0.add(t2), t0.sub(t2));
let (s1, d1) = (t1.add(t3), t1.sub(t3));
out[k] = s0.add(s1);
out[2 * m + k] = s0.sub(s1);
// The odd outputs combine d0 with -j * d1 = (d1.im, -d1.re) and its negative.
out[m + k] = Cpx { re: d0.re + d1.im, im: d0.im - d1.re };
out[3 * m + k] = Cpx { re: d0.re - d1.im, im: d0.im + d1.re };
}
}
3 => {
// Imaginary part of exp(-2 pi i / 3) = -sqrt(3) / 2.
let h = tw[self.n / 3].im;
for k in 0..m {
let t0 = out[k];
let t1 = out[m + k].mul(tw[k * tw_stride]);
let t2 = out[2 * m + k].mul(tw[2 * k * tw_stride]);
let (s, d) = (t1.add(t2), t1.sub(t2));
let a = Cpx { re: t0.re - 0.5 * s.re, im: t0.im - 0.5 * s.im };
out[k] = t0.add(s);
// j * h * d = (-h * d.im, h * d.re)
out[m + k] = Cpx { re: a.re - h * d.im, im: a.im + h * d.re };
out[2 * m + k] = Cpx { re: a.re + h * d.im, im: a.im - h * d.re };
}
}
_ => {
// Generic DFT butterfly (radix 5, once per transform).
let roots: [Cpx; 5] = std::array::from_fn(|q| if q < p { tw[q * (self.n / p)] } else { Cpx::default() });
let mut t = [Cpx::default(); 5];
for k in 0..m {
t[0] = out[k];
for q in 1..p {
t[q] = out[q * m + k].mul(tw[q * k * tw_stride]);
}
for q2 in 0..p {
let mut acc = t[0];
for q in 1..p {
acc = acc.add(t[q].mul(roots[(q * q2) % p]));
}
out[q2 * m + k] = acc;
}
}
}
}
}
}
+6 -2
View File
@@ -7,14 +7,18 @@
//! 20 ms, mono or stereo). From EffectCraft's `effectcraft-opusenc` (storytold/effectcraft@c618429).
//!
//! Public items keep their names, so FilmCraft and EffectCraft can re-export this module. The two
//! halves come from separate crates and do not share code yet (each has its own range coder, MDCT
//! and band tables). [`Bandwidth`] is one type for both (`encoder::Bandwidth` is the same type);
//! halves share the FFT, resampling dot product and CELT allocation preparation; range coders and
//! mode-specific MDCT processing remain separate. [`Bandwidth`] is one type for both (`encoder::Bandwidth` is the same type);
//! the two `Mode` enums keep their apps' variant names and convert into each other.
// Index loops and long argument lists follow the specification's pseudo-code (FilmCraft and
// EffectCraft allow these lints workspace-wide).
#![allow(clippy::needless_range_loop, clippy::too_many_arguments)]
mod dot;
mod fft;
mod rate_common;
#[cfg(feature = "opus-decode")]
mod decode;
#[cfg(feature = "opus-encode")]
+362
View File
@@ -0,0 +1,362 @@
//! CELT pulse cache and allocation caps shared by encoder and decoder.
#[cfg(not(feature = "opus-encode"))]
use super::decode::celt::tables::{ALLOC, EBANDS, LOG2_FRAC_TABLE, MAX_LM, NB_EBANDS};
#[cfg(feature = "opus-encode")]
use super::encoder::tables::{ALLOC, EBANDS, LOG2_FRAC_TABLE, MAX_LM, NB_EBANDS};
const BITRES: u32 = 3;
const MAX_PSEUDO: i32 = 40;
const LOG_MAX_PSEUDO: i32 = 6;
const MAX_FINE_BITS: i32 = 8;
const FINE_OFFSET: i32 = 21;
const QTHETA_OFFSET: i32 = 4;
const QTHETA_OFFSET_TWOPHASE: i32 = 16;
fn ilog(x: u32) -> i32 {
(32 - x.leading_zeros()) as i32
}
/// Number of pulses represented by pseudo-pulse index `i`.
pub fn get_pulses(i: i32) -> i32 {
if i < 8 { i } else { (8 + (i & 7)) << ((i >> 3) - 1) }
}
/// Conservative (rounded up) log2 of `val` with `frac` fractional bits.
pub fn log2_frac(mut val: u32, mut frac: u32) -> i32 {
let mut l = ilog(val);
if val & val.wrapping_sub(1) != 0 {
if l > 16 {
val = ((val - 1) >> (l - 16)) + 1;
} else {
val <<= 16 - l;
}
let mut l = (l - 1) << frac;
loop {
let b = (val >> 16) as i32;
l += b << frac;
val = (val + b as u32) >> b;
val = (val * val + 0x7FFF) >> 15;
if frac == 0 {
break;
}
frac -= 1;
}
l + (val > 0x8000) as i32
} else {
l -= 1;
l << frac
}
}
/// PVQ codebook size V(N, K) (saturating far above 2^32).
pub fn pvq_v(n: usize, k: usize) -> u128 {
if k == 0 {
return 1;
}
if n == 0 {
return 0;
}
let mut row = vec![0u128; k + 1];
row[0] = 1;
for _ in 0..n {
let mut prev_diag = row[0];
for kk in 1..=k {
let up = row[kk];
let v = up.saturating_add(row[kk - 1]).saturating_add(prev_diag);
prev_diag = up;
row[kk] = v.min(u128::MAX >> 8);
}
row[0] = 1;
}
row[k]
}
fn fits_in32(n: usize, k: i32) -> bool {
pvq_v(n, k as usize) <= u32::MAX as u128
}
/// Precomputed per-mode data.
pub struct Mode {
/// log2 of band width in 1/8 bits.
pub log_n: [i32; NB_EBANDS],
pub(crate) cache_index: Vec<i16>,
pub(crate) cache_bits: Vec<u8>,
pub(crate) caps: Vec<u8>,
}
impl Mode {
pub fn new() -> Mode {
let mut log_n = [0i32; NB_EBANDS];
for (i, l) in log_n.iter_mut().enumerate() {
*l = log2_frac((EBANDS[i + 1] - EBANDS[i]) as u32, BITRES);
}
let (cache_index, cache_bits) = compute_pulse_cache(MAX_LM as i32);
let mut m = Mode { log_n, cache_index, cache_bits, caps: Vec::new() };
m.caps = compute_caps(&m);
m
}
#[inline]
pub fn cache(&self, lm: i32, band: usize) -> &[u8] {
let idx = self.cache_index[((lm + 1) as usize) * NB_EBANDS + band];
&self.cache_bits[idx.max(0) as usize..]
}
pub fn bits2pulses(&self, band: usize, lm: i32, bits: i32) -> i32 {
let cache = self.cache(lm, band);
let mut lo = 0i32;
let mut hi = cache[0] as i32;
let bits = bits - 1;
for _ in 0..LOG_MAX_PSEUDO {
let mid = (lo + hi + 1) >> 1;
if cache[mid as usize] as i32 >= bits {
hi = mid;
} else {
lo = mid;
}
}
let lo_bits = if lo == 0 { -1 } else { cache[lo as usize] as i32 };
if bits - lo_bits <= cache[hi as usize] as i32 - bits { lo } else { hi }
}
pub fn pulses2bits(&self, band: usize, lm: i32, pulses: i32) -> i32 {
if pulses == 0 { 0 } else { self.cache(lm, band)[pulses as usize] as i32 + 1 }
}
/// Per-band allocation caps for this frame (`init_caps`).
pub fn init_caps(&self, lm: usize, c: usize) -> [i32; NB_EBANDS] {
let mut cap = [0i32; NB_EBANDS];
for (i, cp) in cap.iter_mut().enumerate() {
let n = (EBANDS[i + 1] - EBANDS[i]) << lm;
*cp = ((self.caps[NB_EBANDS * (2 * lm + c - 1) + i] as i32 + 64) * c as i32 * n) >> 2;
}
cap
}
}
fn compute_pulse_cache(lm_max: i32) -> (Vec<i16>, Vec<u8>) {
let nb = NB_EBANDS;
let rows = (lm_max + 2) as usize;
let mut index = vec![-1i16; nb * rows];
let mut entries: Vec<(usize, i32, usize)> = Vec::new();
let mut curr = 0usize;
for i in 0..rows {
for j in 0..nb {
let n = (((EBANDS[j + 1] - EBANDS[j]) << i) >> 1) as usize;
index[i * nb + j] = -1;
'search: for k in 0..=i {
for nn in 0..nb {
if k == i && nn >= j {
break;
}
if n == (((EBANDS[nn + 1] - EBANDS[nn]) << k) >> 1) as usize {
index[i * nb + j] = index[k * nb + nn];
break 'search;
}
}
}
if index[i * nb + j] == -1 && n != 0 {
let mut k = 0;
while k < MAX_PSEUDO && fits_in32(n, get_pulses(k + 1)) {
k += 1;
}
index[i * nb + j] = curr as i16;
entries.push((n, k, curr));
curr += (k + 1) as usize;
}
}
}
let mut bits = vec![0u8; curr];
for &(n, k, off) in &entries {
let maxk = get_pulses(k) as usize;
let tmp = required_bits(n, maxk, BITRES);
for j in 1..=k {
bits[off + j as usize] = (tmp[get_pulses(j) as usize] - 1) as u8;
}
bits[off] = k as u8;
}
(index, bits)
}
fn required_bits(n: usize, maxk: usize, frac: u32) -> Vec<i32> {
let mut out = vec![0i32; maxk + 1];
if n == 1 {
for o in out.iter_mut().skip(1) {
*o = 1 << frac;
}
} else {
for (k, o) in out.iter_mut().enumerate().skip(1) {
*o = log2_frac(pvq_v(n, k) as u32, frac);
}
}
out
}
fn compute_caps(m: &Mode) -> Vec<u8> {
let nb = NB_EBANDS;
let mut caps = vec![0u8; (MAX_LM + 1) * 2 * nb];
for i in 0..=MAX_LM as i32 {
for c in 1..=2i32 {
for j in 0..nb {
let mut n0 = EBANDS[j + 1] - EBANDS[j];
let max_bits = if n0 << i == 1 {
(c * (1 + MAX_FINE_BITS)) << BITRES
} else {
let mut lm0 = 0;
if n0 > 2 {
n0 >>= 1;
lm0 -= 1;
} else if n0 <= 1 {
lm0 = i.min(1);
n0 <<= lm0;
}
let pcache = m.cache(lm0, j);
let mut mb = pcache[pcache[0] as usize] as i32 + 1;
let mut n = n0;
for k in 0..(i - lm0) {
mb <<= 1;
let offset = ((m.log_n[j] + ((lm0 + k) << BITRES)) >> 1) - QTHETA_OFFSET;
let num = 459 * ((2 * n - 1) * offset + mb);
let den = ((2 * n - 1) << 9) - 459;
let qb = ((num + (den >> 1)) / den).min(57);
mb += qb;
n <<= 1;
}
if c == 2 {
mb <<= 1;
let offset = ((m.log_n[j] + (i << BITRES)) >> 1) - if n == 2 { QTHETA_OFFSET_TWOPHASE } else { QTHETA_OFFSET };
let ndof = 2 * n - 1 - (n == 2) as i32;
let p = if n == 2 { 512 } else { 487 };
let num = p * (mb + ndof * offset);
let den = (ndof << 9) - p;
let qb = ((num + (den >> 1)) / den).min(if n == 2 { 64 } else { 61 });
mb += qb;
}
let ndof = c * n + if c == 2 && n > 2 { 1 } else { 0 };
let mut offset = ((m.log_n[j] + (i << BITRES)) >> 1) - FINE_OFFSET;
if n == 2 {
offset += (1 << BITRES) >> 2;
}
let num = mb + ndof * offset;
let den = (ndof - 1) << BITRES;
let qb = ((num + (den >> 1)) / den).min(MAX_FINE_BITS);
mb += (c * qb) << BITRES;
mb
};
let w = c * ((EBANDS[j + 1] - EBANDS[j]) << i);
let v = (4 * max_bits / w) - 64;
caps[((i * 2 + c - 1) as usize) * nb + j] = v.clamp(0, 255) as u8;
}
}
}
caps
}
/// Shared allocation search before encoder/decoder signalling diverges.
pub struct Bracket {
pub skip_start: usize,
pub bits1: [i32; NB_EBANDS],
pub bits2: [i32; NB_EBANDS],
pub thresh: [i32; NB_EBANDS],
pub total: i32,
pub skip_rsv: i32,
pub intensity_rsv: i32,
pub dual_stereo_rsv: i32,
}
pub fn bracket(
start: usize,
end: usize,
offsets: &[i32; NB_EBANDS],
cap: &[i32; NB_EBANDS],
alloc_trim: i32,
total: i32,
c: usize,
lm: usize,
) -> Bracket {
let ci = c as i32;
let lmi = lm as i32;
let mut total = total.max(0);
let mut skip_start = start;
let skip_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 };
total -= skip_rsv;
let mut intensity_rsv = 0;
let mut dual_stereo_rsv = 0;
if c == 2 {
intensity_rsv = LOG2_FRAC_TABLE[end - start];
if intensity_rsv > total {
intensity_rsv = 0;
} else {
total -= intensity_rsv;
dual_stereo_rsv = if total >= 1 << BITRES { 1 << BITRES } else { 0 };
total -= dual_stereo_rsv;
}
}
let mut bits1 = [0i32; NB_EBANDS];
let mut bits2 = [0i32; NB_EBANDS];
let mut thresh = [0i32; NB_EBANDS];
let mut trim_offset = [0i32; NB_EBANDS];
for j in start..end {
let w = EBANDS[j + 1] - EBANDS[j];
thresh[j] = (ci << BITRES).max(((3 * w) << lmi << BITRES) >> 4);
trim_offset[j] = (ci * w * (alloc_trim - 5 - lmi) * (end as i32 - j as i32 - 1) * (1 << (lmi + BITRES as i32))) >> 6;
if w << lmi == 1 {
trim_offset[j] -= ci << BITRES;
}
}
let nb_alloc = ALLOC.len() as i32;
let mut lo = 1;
let mut hi = nb_alloc - 1;
loop {
let mut done = false;
let mut psum = 0;
let mid = (lo + hi) >> 1;
for j in (start..end).rev() {
let n = EBANDS[j + 1] - EBANDS[j];
let mut bitsj = (ci * n * (ALLOC[mid as usize][j] as i32)) << lmi >> 2;
if bitsj > 0 {
bitsj = (bitsj + trim_offset[j]).max(0);
}
bitsj += offsets[j];
if bitsj >= thresh[j] || done {
done = true;
psum += bitsj.min(cap[j]);
} else if bitsj >= ci << BITRES {
psum += ci << BITRES;
}
}
if psum > total {
hi = mid - 1;
} else {
lo = mid + 1;
}
if lo > hi {
break;
}
}
hi = lo;
lo -= 1;
for j in start..end {
let n = EBANDS[j + 1] - EBANDS[j];
let mut bits1j = (ci * n * (ALLOC[lo as usize][j] as i32)) << lmi >> 2;
let mut bits2j = if hi >= nb_alloc { cap[j] } else { (ci * n * (ALLOC[hi as usize][j] as i32)) << lmi >> 2 };
if bits1j > 0 {
bits1j = (bits1j + trim_offset[j]).max(0);
}
if bits2j > 0 {
bits2j = (bits2j + trim_offset[j]).max(0);
}
if lo > 0 {
bits1j += offsets[j];
}
bits2j += offsets[j];
if offsets[j] > 0 {
skip_start = j;
}
bits2j = (bits2j - bits1j).max(0);
bits1[j] = bits1j;
bits2[j] = bits2j;
}
Bracket { skip_start, bits1, bits2, thresh, total, skip_rsv, intensity_rsv, dual_stereo_rsv }
}
+287
View File
@@ -0,0 +1,287 @@
//! Audio-only files: WAV (RIFF, little-endian PCM; RF64 past 4 GiB) and AIFF (big-endian PCM),
//! 16, 24 or 32-bit integer or 32-bit float ([`SampleFormat`]), for streaming export.
//! Float WAV is `WAVE_FORMAT_IEEE_FLOAT` with a `fact` chunk; float AIFF is AIFF-C (`fl32`).
//!
//! The header is written first from the known length (`header`), then the samples in chunks
//! (`encode`) and the pad byte (`pad`), so an export streams to disk instead of building the
//! file in memory.
/// PCM representation in an exported file.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SampleFormat {
/// 16-bit signed integer.
Int16,
/// 24-bit signed integer.
Int24,
/// 32-bit signed integer.
Int32,
/// 32-bit IEEE float.
Float32,
}
impl SampleFormat {
/// Bits per sample.
pub fn bits(self) -> u16 {
match self {
Self::Int16 => 16,
Self::Int24 => 24,
Self::Int32 | Self::Float32 => 32,
}
}
/// Bytes per sample.
pub fn bytes(self) -> usize {
usize::from(self.bits() / 8)
}
/// Whether this is a floating point format.
pub fn is_float(self) -> bool {
self == Self::Float32
}
/// Clip and quantize one sample with symmetric integer full scale; NaN becomes silence.
pub fn quantise(self, s: f32) -> i32 {
let s = s.clamp(-1.0, 1.0);
match self {
Self::Int16 => (s * 32_767.0).round() as i32,
Self::Int24 => (s * 8_388_607.0).round() as i32,
Self::Int32 => (f64::from(s) * 2_147_483_647.0).round() as i32,
Self::Float32 => 0,
}
}
/// Sanitize a floating sample: NaN becomes silence, infinity becomes the largest finite value.
pub fn float(s: f32) -> f32 {
if s.is_nan() { 0.0 } else { s.clamp(f32::MIN, f32::MAX) }
}
}
/// The container of an audio-only export.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum PcmContainer {
/// RIFF/WAVE or RF64.
Wav,
/// AIFF or AIFF-C.
Aiff,
}
/// Bytes of sample data for `frames` frames, or None when that does not fit 64 bits.
fn data_len(frames: u64, channels: u16, format: SampleFormat) -> Option<u64> {
frames.checked_mul(u64::from(channels))?.checked_mul(format.bytes() as u64)
}
/// The pad byte after an odd-sized data chunk (RIFF and AIFF chunks are word-aligned).
pub fn pad(frames: u64, channels: u16, format: SampleFormat) -> &'static [u8] {
if data_len(frames, channels, format).is_some_and(|n| n & 1 == 1) { &[0] } else { &[] }
}
/// `KSDATAFORMAT_SUBTYPE_PCM` / `_IEEE_FLOAT` after their first two bytes (the format tag).
const SUBTYPE_TAIL: [u8; 14] = [0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71];
/// AIFF-C format version 1 (`FVER`).
const AIFC_VERSION_1: u32 = 0xA280_5140;
/// The AIFF-C compression name of `fl32`, as a Pascal string of even length.
const FL32_NAME: &[u8] = b"\x1532-bit floating point";
/// Everything before the samples of a file of `frames` frames. A WAV whose RIFF size would not
/// fit 32 bits (4 GiB: 24-bit 5.1 at 96 kHz after about 41 minutes) is written as RF64 (EBU Tech
/// 3306), with the 64-bit sizes in a `ds64` chunk. AIFF has no 64-bit form, so a longer AIFF is
/// an error naming WAV instead.
pub fn header(container: PcmContainer, channels: u16, sample_rate: u32, format: SampleFormat, frames: u64) -> Result<Vec<u8>, String> {
let too_long = || "the audio is too long to write".to_string();
if channels == 0 {
return Err("audio needs at least one channel".into());
}
if sample_rate == 0 {
return Err("audio needs a sample rate".into());
}
let bits = format.bits();
let bps = bits / 8;
let float = format.is_float();
let data = data_len(frames, channels, format).ok_or_else(too_long)?;
let block_align = channels.checked_mul(bps).ok_or("too many audio channels")?;
match container {
PcmContainer::Wav => {
let byte_rate =
u32::try_from(u64::from(sample_rate) * u64::from(block_align)).map_err(|_| "the audio sample rate is too high".to_string())?;
let ext = channels > 2;
// plain PCM has no cbSize; IEEE float has an empty one
let fmt_len: u32 = if ext {
40
} else if float {
18
} else {
16
};
// every format but integer PCM has a `fact` chunk (the frame count; RF64: in `ds64`)
let fact_len: u64 = if float { 12 } else { 0 };
let padded = data.checked_add(data & 1).ok_or_else(too_long)?;
// "WAVE" + fmt chunk + fact chunk + data chunk header + samples
let riff = (4 + 8 + u64::from(fmt_len) + fact_len + 8).checked_add(padded).ok_or_else(too_long)?;
let rf64 = riff > u64::from(u32::MAX);
let mut v = Vec::with_capacity(100);
if rf64 {
const DS64: u64 = 8 + 28;
v.extend_from_slice(b"RF64");
v.extend_from_slice(&u32::MAX.to_le_bytes());
v.extend_from_slice(b"WAVEds64");
v.extend_from_slice(&28u32.to_le_bytes());
v.extend_from_slice(&riff.checked_add(DS64).ok_or_else(too_long)?.to_le_bytes());
v.extend_from_slice(&data.to_le_bytes());
v.extend_from_slice(&frames.to_le_bytes());
v.extend_from_slice(&0u32.to_le_bytes()); // no table entries
} else {
v.extend_from_slice(b"RIFF");
v.extend_from_slice(&(riff as u32).to_le_bytes());
v.extend_from_slice(b"WAVE");
}
let tag: u16 = if float { 3 } else { 1 };
v.extend_from_slice(b"fmt ");
v.extend_from_slice(&fmt_len.to_le_bytes());
v.extend_from_slice(&(if ext { 0xFFFE } else { tag }).to_le_bytes());
v.extend_from_slice(&channels.to_le_bytes());
v.extend_from_slice(&sample_rate.to_le_bytes());
v.extend_from_slice(&byte_rate.to_le_bytes());
v.extend_from_slice(&block_align.to_le_bytes());
v.extend_from_slice(&bits.to_le_bytes());
if ext {
// cbSize, valid bits, channel mask, KSDATAFORMAT_SUBTYPE_PCM / _IEEE_FLOAT
v.extend_from_slice(&22u16.to_le_bytes());
v.extend_from_slice(&bits.to_le_bytes());
let mask: u32 = if channels == 6 { 0x3F } else { 0 };
v.extend_from_slice(&mask.to_le_bytes());
v.extend_from_slice(&tag.to_le_bytes());
v.extend_from_slice(&SUBTYPE_TAIL);
} else if float {
v.extend_from_slice(&0u16.to_le_bytes());
}
if float {
v.extend_from_slice(b"fact");
v.extend_from_slice(&4u32.to_le_bytes());
v.extend_from_slice(&(if rf64 { u32::MAX } else { u32::try_from(frames).unwrap_or(u32::MAX) }).to_le_bytes());
}
v.extend_from_slice(b"data");
v.extend_from_slice(&(if rf64 { u32::MAX } else { data as u32 }).to_le_bytes());
Ok(v)
}
PcmContainer::Aiff => {
let aiff_too_long = || "AIFF holds at most 4 GB of audio: export WAV, which has no such limit".to_string();
let frames32 = u32::try_from(frames).map_err(|_| aiff_too_long())?;
// SSND: offset + block size + samples
let ssnd = u32::try_from(data.checked_add(8).ok_or_else(aiff_too_long)?).map_err(|_| aiff_too_long())?;
// floats need AIFF-C: a version chunk, and the compression type and name in COMM
let comm: u32 = if float { 18 + 4 + FL32_NAME.len() as u32 } else { 18 };
let fver: u32 = if float { 8 + 4 } else { 0 };
let form = (4u32 + fver + (8 + comm))
.checked_add(8)
.and_then(|n| n.checked_add(ssnd))
.and_then(|n| n.checked_add(ssnd & 1))
.ok_or_else(aiff_too_long)?;
let mut v = Vec::with_capacity(90);
v.extend_from_slice(b"FORM");
v.extend_from_slice(&form.to_be_bytes());
if float {
v.extend_from_slice(b"AIFCFVER");
v.extend_from_slice(&4u32.to_be_bytes());
v.extend_from_slice(&AIFC_VERSION_1.to_be_bytes());
} else {
v.extend_from_slice(b"AIFF");
}
v.extend_from_slice(b"COMM");
v.extend_from_slice(&comm.to_be_bytes());
v.extend_from_slice(&channels.to_be_bytes());
v.extend_from_slice(&frames32.to_be_bytes());
v.extend_from_slice(&bits.to_be_bytes());
v.extend_from_slice(&extended(sample_rate));
if float {
v.extend_from_slice(b"fl32");
v.extend_from_slice(FL32_NAME);
}
v.extend_from_slice(b"SSND");
v.extend_from_slice(&ssnd.to_be_bytes());
v.extend_from_slice(&0u32.to_be_bytes()); // offset
v.extend_from_slice(&0u32.to_be_bytes()); // block size
Ok(v)
}
}
}
/// Append interleaved samples in `format` (little-endian for WAV, big-endian for AIFF).
pub fn encode(container: PcmContainer, interleaved: &[f32], format: SampleFormat, out: &mut Vec<u8>) {
let bps = format.bytes();
out.reserve(interleaved.len().saturating_mul(bps));
for &s in interleaved {
let q = if format.is_float() { SampleFormat::float(s).to_bits() as i32 } else { format.quantise(s) };
match container {
PcmContainer::Wav => out.extend_from_slice(q.to_le_bytes().get(..bps).unwrap_or_default()),
PcmContainer::Aiff => out.extend_from_slice(q.to_be_bytes().get(4 - bps..).unwrap_or_default()),
}
}
}
/// A whole file in memory (short audio; exports stream with [`header`] and [`encode`]).
fn write_whole(container: PcmContainer, interleaved: &[f32], channels: u16, sample_rate: u32, format: SampleFormat) -> Result<Vec<u8>, String> {
let per_frame = usize::from(channels.max(1));
if !interleaved.len().is_multiple_of(per_frame) {
return Err("the samples do not make whole frames".into());
}
let frames = (interleaved.len() / per_frame) as u64;
let mut v = header(container, channels, sample_rate, format, frames)?;
encode(container, interleaved, format, &mut v);
v.extend_from_slice(pad(frames, channels, format));
Ok(v)
}
/// A RIFF/WAVE file of interleaved samples. More than two channels are written as
/// `WAVE_FORMAT_EXTENSIBLE` with the speaker mask of the layout (6 channels: L, R, C, LFE, Ls, Rs
/// = `0x3F`).
pub fn write_wav(interleaved: &[f32], channels: u16, sample_rate: u32, format: SampleFormat) -> Result<Vec<u8>, String> {
write_whole(PcmContainer::Wav, interleaved, channels, sample_rate, format)
}
/// IEEE 754 80-bit extended encoding of a sample rate (AIFF `COMM`).
pub fn extended(rate: u32) -> [u8; 10] {
let mut out = [0u8; 10];
if rate == 0 {
return out;
}
let shift = rate.leading_zeros();
let mant = (rate as u64) << (32 + shift);
let exp = 16383 + 31 - shift as u16;
out[..2].copy_from_slice(&exp.to_be_bytes());
out[2..].copy_from_slice(&mant.to_be_bytes());
out
}
/// An AIFF file (`FORM`/`AIFF`, `COMM` + `SSND`; AIFF-C for floats) of interleaved samples.
pub fn write_aiff(interleaved: &[f32], channels: u16, sample_rate: u32, format: SampleFormat) -> Result<Vec<u8>, String> {
write_whole(PcmContainer::Aiff, interleaved, channels, sample_rate, format)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn streaming_formats_preserve_samples_and_bounds() {
let samples = [1.0, -1.0, f32::NAN];
let wav = write_wav(&samples, 1, 48_000, SampleFormat::Int24).unwrap();
assert_eq!(&wav[..4], b"RIFF");
assert_eq!(&wav[44..53], &[255, 255, 127, 1, 0, 128, 0, 0, 0]);
let h = header(PcmContainer::Wav, 6, 96_000, SampleFormat::Int24, 96_000 * 3600).unwrap();
assert_eq!(&h[..4], b"RF64");
assert!(header(PcmContainer::Aiff, 2, 48_000, SampleFormat::Int24, u64::MAX).is_err());
let aiff = write_aiff(&[0.25, -3.0], 1, 44_100, SampleFormat::Float32).unwrap();
assert_eq!(&aiff[8..12], b"AIFC");
assert_eq!(&aiff[aiff.len() - 8..], [0.25f32.to_be_bytes(), (-3.0f32).to_be_bytes()].concat());
}
#[test]
fn zero_channels_rates_and_partial_frames_are_refused() {
for container in [PcmContainer::Wav, PcmContainer::Aiff] {
assert!(header(container, 0, 48_000, SampleFormat::Int16, 1).is_err());
assert!(header(container, 1, 0, SampleFormat::Int16, 1).is_err());
assert!(header(container, 2, 48_000, SampleFormat::Int16, 0).is_ok(), "zero frames stay valid");
}
assert!(write_wav(&[0.5], 0, 48_000, SampleFormat::Int16).is_err());
assert!(write_wav(&[0.1, 0.2, 0.3], 2, 48_000, SampleFormat::Int16).is_err());
assert!(write_aiff(&[0.1, 0.2, 0.3], 2, 48_000, SampleFormat::Int16).is_err());
assert!(write_aiff(&[0.1, 0.2], 2, 48_000, SampleFormat::Int16).is_ok());
}
}
+96
View File
@@ -0,0 +1,96 @@
//! Interleaved PCM packet decoding into caller-owned planar channels.
/// The sample representation in an interleaved PCM packet.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Format {
/// Bits per sample.
pub bits: u16,
/// IEEE floating-point samples.
pub float: bool,
/// Big-endian sample bytes.
pub big_endian: bool,
/// Signed 8-bit integers (wider integer PCM is signed).
pub signed: bool,
}
/// Append complete interleaved frames to caller-owned planar channels. Empty channel slices and
/// zero-bit formats append nothing. Incomplete frames are ignored; unsupported widths produce
/// silent samples, matching the container adapters that use this API.
pub fn decode_into(data: &[u8], cfg: Format, out: &mut [Vec<f32>]) {
let ch = out.len();
if cfg.bits == 0 || ch == 0 {
return;
}
let bps = usize::from(cfg.bits).div_ceil(8);
let Some(frame_bytes) = bps.checked_mul(ch).filter(|&n| n > 0) else { return };
let n = data.len() / frame_bytes;
for channel in out.iter_mut() {
channel.reserve(n);
}
// `n` is whole frames within `data`, so every sample slice below is in bounds.
for i in 0..n {
for (c, o) in out.iter_mut().enumerate() {
let off = (i * ch + c) * bps;
let b = &data[off..off + bps];
let v = match (cfg.bits, cfg.float, cfg.big_endian) {
(32, true, false) => f32::from_le_bytes([b[0], b[1], b[2], b[3]]),
(32, true, true) => f32::from_be_bytes([b[0], b[1], b[2], b[3]]),
(64, true, false) => f64::from_le_bytes(b.try_into().unwrap_or([0; 8])) as f32,
(64, true, true) => f64::from_be_bytes(b.try_into().unwrap_or([0; 8])) as f32,
(8, _, _) => {
if cfg.signed {
b[0] as i8 as f32 / 128.0
} else {
(b[0] as f32 - 128.0) / 128.0
}
}
(16, _, false) => i16::from_le_bytes([b[0], b[1]]) as f32 / 32768.0,
(16, _, true) => i16::from_be_bytes([b[0], b[1]]) as f32 / 32768.0,
(24, _, false) => (i32::from_le_bytes([0, b[0], b[1], b[2]]) >> 8) as f32 / 8_388_608.0,
(24, _, true) => (i32::from_be_bytes([b[0], b[1], b[2], 0]) >> 8) as f32 / 8_388_608.0,
(32, false, false) => i32::from_le_bytes([b[0], b[1], b[2], b[3]]) as f32 / 2_147_483_648.0,
(32, false, true) => i32::from_be_bytes([b[0], b[1], b[2], b[3]]) as f32 / 2_147_483_648.0,
_ => 0.0,
};
o.push(v);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn decodes_endian_and_interleaving_without_touching_partial_frames() {
let mut channels = vec![Vec::new(), Vec::new()];
let fmt = Format { bits: 16, float: false, big_endian: true, signed: true };
decode_into(&[0x40, 0, 0xc0, 0, 0xff], fmt, &mut channels);
assert_eq!(channels, [vec![0.5], vec![-0.5]]);
decode_into(&[0; 8], Format { bits: 0, ..fmt }, &mut channels);
assert_eq!(channels, [vec![0.5], vec![-0.5]]);
}
#[test]
fn decodes_sample_representations_and_ignores_invalid_inputs() {
let base = Format { bits: 8, float: false, big_endian: false, signed: false };
let cases: &[(Format, &[u8], f32)] = &[
(base, &[192], 0.5),
(Format { signed: true, ..base }, &[192], -0.5),
(Format { bits: 16, ..base }, &[0, 64], 0.5),
(Format { bits: 24, ..base }, &[0, 0, 64], 0.5),
(Format { bits: 24, big_endian: true, ..base }, &[0xc0, 0, 0], -0.5),
(Format { bits: 32, ..base }, &[0, 0, 0, 64], 0.5),
(Format { bits: 32, big_endian: true, ..base }, &[0xc0, 0, 0, 0], -0.5),
(Format { bits: 32, float: true, ..base }, &0.5_f32.to_le_bytes(), 0.5),
(Format { bits: 32, float: true, big_endian: true, ..base }, &(-0.5_f32).to_be_bytes(), -0.5),
(Format { bits: 64, float: true, ..base }, &0.5_f64.to_le_bytes(), 0.5),
(Format { bits: 64, float: true, big_endian: true, ..base }, &(-0.5_f64).to_be_bytes(), -0.5),
(Format { bits: 12, ..base }, &[0xff, 0xff], 0.0),
];
for &(format, bytes, expected) in cases {
let mut out = [vec![1.0]];
decode_into(bytes, format, &mut out);
assert_eq!(out[0], [1.0, expected], "{format:?}");
}
decode_into(&[1], base, &mut []);
}
}
+287
View File
@@ -0,0 +1,287 @@
//! Bit packing (Vorbis I §2) and codebooks (§3): Huffman lengths, the codeword assignment the
//! decoder derives from them, VQ lattices (lookup type 1) and the header encoding.
use super::{Error, Result};
/// LSB-first bit packer (Vorbis I §2.1.4): bits fill each byte from its least significant end.
#[derive(Clone, Debug, Default)]
pub struct BitWriter {
buf: Vec<u8>,
bits: u64,
}
impl BitWriter {
pub fn new() -> BitWriter {
BitWriter::default()
}
/// Writes the low `n` bits of `v` (`n` ≤ 32).
pub fn write(&mut self, v: u32, n: u32) {
let n = n.min(32);
for i in 0..n {
let bit = (v >> i) & 1;
let at = (self.bits / 8) as usize;
if at == self.buf.len() {
self.buf.push(0);
}
if bit != 0
&& let Some(b) = self.buf.get_mut(at)
{
*b |= 1 << (self.bits % 8);
}
self.bits += 1;
}
}
pub fn flag(&mut self, b: bool) {
self.write(b as u32, 1);
}
/// Writes a codeword MSB first: the decoder walks the tree from its first bit.
fn codeword(&mut self, word: u32, len: u32) {
for i in (0..len).rev() {
self.write((word >> i) & 1, 1);
}
}
pub fn bytes(&mut self, data: &[u8]) {
for &b in data {
self.write(u32::from(b), 8);
}
}
pub fn finish(self) -> Vec<u8> {
self.buf
}
}
/// `ilog` of the specification (§9.2.1): the position of the highest set bit, 0 for 0.
pub fn ilog(v: u32) -> u32 {
32 - v.leading_zeros()
}
/// The 32-bit float format of codebook headers (§9.2.2), for values it represents exactly.
pub fn float32_pack(v: f64) -> u32 {
if v == 0.0 || !v.is_finite() {
return 0;
}
let sign = if v < 0.0 { 0x8000_0000u32 } else { 0 };
let mut m = v.abs();
let mut exp: i32 = 0;
// a 21-bit mantissa: scale into [2^20, 2^21)
while m >= (1 << 21) as f64 {
m /= 2.0;
exp += 1;
}
while m < (1 << 20) as f64 {
m *= 2.0;
exp -= 1;
}
let mant = (m.round() as u32).min((1 << 21) - 1);
let biased = (exp + 788).clamp(0, 1023) as u32;
sign | (biased << 21) | mant
}
/// Huffman code lengths (1..=`max_len`) for `weights`, every entry coded.
pub fn huffman_lengths(weights: &[f64], max_len: u32) -> Vec<u32> {
let n = weights.len();
if n <= 1 {
return vec![1; n];
}
let total: f64 = weights.iter().map(|w| if w.is_finite() { w.max(0.0) } else { 0.0 }).sum::<f64>().max(1e-300);
let mut floor = 1e-9;
loop {
let w: Vec<f64> = weights.iter().map(|&x| (if x.is_finite() { x.max(0.0) } else { 0.0 }) / total + floor).collect();
let lens = huffman(&w);
if lens.iter().all(|&l| l <= max_len) || floor > 1.0 {
return lens.into_iter().map(|l| l.clamp(1, max_len)).collect();
}
floor *= 4.0;
}
}
fn huffman(w: &[f64]) -> Vec<u32> {
// nodes: (weight, children) with leaves first
let n = w.len();
let mut parent = vec![usize::MAX; 2 * n];
let mut weight: Vec<f64> = w.to_vec();
let mut alive: Vec<usize> = (0..n).collect();
while alive.len() > 1 {
alive.sort_by(|&a, &b| weight[b].partial_cmp(&weight[a]).unwrap_or(std::cmp::Ordering::Equal).then(b.cmp(&a)));
let (Some(a), Some(b)) = (alive.pop(), alive.pop()) else { break };
let id = weight.len();
weight.push(weight[a] + weight[b]);
if parent.len() <= id {
parent.resize(id + 1, usize::MAX);
}
parent[a] = id;
parent[b] = id;
alive.push(id);
}
(0..n)
.map(|i| {
let mut d = 0;
let mut x = i;
while let Some(&p) = parent.get(x) {
if p == usize::MAX {
break;
}
d += 1;
x = p;
}
d
})
.collect()
}
/// The codewords the decoder assigns to `lengths` (§3.2.1): in entry order, each entry gets the
/// lowest available codeword of its length (no prefix of an assigned codeword, and no assigned
/// codeword a prefix of it). An error when the lengths do not fit a prefix code.
pub fn assign_codewords(lengths: &[u32]) -> Result<Vec<u32>> {
// A trie of the assigned codewords: `children[node]` (0 = none), `leaf[node]`.
let mut children: Vec<[usize; 2]> = vec![[0, 0]];
let mut leaf = vec![false];
let mut words = Vec::with_capacity(lengths.len());
for &len in lengths {
if len == 0 || len > 32 {
return Err(Error::Internal("codeword length"));
}
let bits = lowest_free(&children, &leaf, 0, len).ok_or(Error::Internal("over-subscribed codebook"))?;
let mut node = 0usize;
let mut word = 0u32;
for &bit in &bits {
word = (word << 1) | bit as u32;
node = match children[node][bit] {
0 => {
children.push([0, 0]);
leaf.push(false);
let id = children.len() - 1;
children[node][bit] = id;
id
}
c => c,
};
}
leaf[node] = true;
words.push(word);
}
Ok(words)
}
/// The leftmost free path of `depth` bits below `node` (depth-first, 0 before 1).
fn lowest_free(children: &[[usize; 2]], leaf: &[bool], node: usize, depth: u32) -> Option<Vec<usize>> {
if leaf[node] || depth == 0 {
return None;
}
for bit in 0..2 {
let c = children[node][bit];
if c == 0 {
let mut p = vec![bit];
p.resize(depth as usize, 0);
return Some(p);
}
if let Some(mut rest) = lowest_free(children, leaf, c, depth - 1) {
rest.insert(0, bit);
return Some(rest);
}
}
None
}
/// A codebook: entry lengths and codewords, and for VQ books a lattice (lookup type 1).
#[derive(Clone, Debug)]
pub struct Codebook {
pub dimensions: u32,
pub lengths: Vec<u32>,
pub words: Vec<u32>,
/// Lattice values per dimension (lookup type 1: `values^dimensions == entries`).
pub lattice: Option<Lattice>,
}
#[derive(Clone, Copy, Debug)]
pub struct Lattice {
pub minimum: f64,
pub delta: f64,
/// Values per dimension.
pub values: u32,
}
impl Codebook {
/// A scalar book (no lookup) of `weights.len()` entries.
pub fn scalar(dimensions: u32, weights: &[f64]) -> Result<Codebook> {
let lengths = huffman_lengths(weights, 20);
let words = assign_codewords(&lengths)?;
Ok(Codebook { dimensions, lengths, words, lattice: None })
}
/// A lattice VQ book: `values` integers per dimension from `minimum` in steps of `delta`;
/// `weight(v)` is the relative probability of one component `v`.
pub fn lattice(dimensions: u32, values: u32, minimum: i32, delta: i32, weight: impl Fn(i32) -> f64) -> Result<Codebook> {
let entries = values.checked_pow(dimensions).ok_or(Error::Internal("codebook size"))?;
let w: Vec<f64> = (0..entries)
.map(|e| {
let mut p = 1.0;
let mut rest = e;
for _ in 0..dimensions {
let t = (rest % values) as i32;
rest /= values;
p *= weight(minimum + t * delta);
}
p
})
.collect();
let lengths = huffman_lengths(&w, 20);
let words = assign_codewords(&lengths)?;
Ok(Codebook { dimensions, lengths, words, lattice: Some(Lattice { minimum: f64::from(minimum), delta: f64::from(delta), values }) })
}
pub fn entries(&self) -> usize {
self.lengths.len()
}
/// Writes entry `e`.
pub fn put(&self, w: &mut BitWriter, e: usize) {
if let (Some(&word), Some(&len)) = (self.words.get(e), self.lengths.get(e)) {
w.codeword(word, len);
}
}
/// The entry of the lattice vector `v` (component 0 is the least significant digit, §3.2.1).
pub fn entry_of(&self, v: &[i32]) -> usize {
let Some(l) = self.lattice else { return 0 };
let mut e = 0usize;
let mut scale = 1usize;
for &x in v.iter().take(self.dimensions as usize) {
let t = ((f64::from(x) - l.minimum) / l.delta).round().clamp(0.0, f64::from(l.values - 1)) as usize;
e += t * scale;
scale *= l.values as usize;
}
e
}
/// The header encoding (§3.2.1).
pub fn write_header(&self, w: &mut BitWriter) {
w.write(0x56_4342, 24);
w.write(self.dimensions, 16);
w.write(self.entries() as u32, 24);
w.flag(false); // not ordered
w.flag(false); // not sparse
for &l in &self.lengths {
w.write(l - 1, 5);
}
match self.lattice {
None => w.write(0, 4),
Some(l) => {
w.write(1, 4);
w.write(float32_pack(l.minimum), 32);
w.write(float32_pack(l.delta), 32);
let bits = ilog(l.values - 1).max(1);
w.write(bits - 1, 4);
w.flag(false); // sequence_p
for m in 0..l.values {
w.write(m, bits);
}
}
}
}
}
+207
View File
@@ -0,0 +1,207 @@
//! Floor type 1 (Vorbis I §7): a piecewise-linear spectral envelope in the decoder's dB scale.
//!
//! The encoder picks a target value at each of the floor's X positions, codes it against the
//! decoder's prediction from its neighbours (step 2 of §7.2.4, inverted), and renders the curve
//! exactly as the decoder will (`render_line`, integer arithmetic), so the residue is computed
//! against the very floor the decoder multiplies it by.
use super::book::{BitWriter, Codebook, ilog};
/// `floor1_multiplier`: 2, so values run 0..128 and step two entries of the dB table.
pub const MULTIPLIER: u32 = 2;
/// The value range for [`MULTIPLIER`] 2.
pub const RANGE: i32 = 128;
const RANGE_BITS: u32 = 10;
/// Points per partition, and partitions.
pub const CLASS_DIM: usize = 4;
pub const PARTITIONS: usize = 8;
/// Step (in dB) between entries of `floor1_inverse_dB_table`: it runs from 1.0649863e-07 to 1
/// over 256 entries.
fn db_step() -> f64 {
(1.0f64 / 1.064_986_3e-7).ln() / 255.0
}
/// `floor1_inverse_dB_table[i]`.
pub fn inverse_db(i: i32) -> f32 {
(f64::from(i.clamp(0, 255) - 255) * db_step()).exp() as f32
}
/// The table index whose amplitude is nearest (in dB) to `a`.
pub fn db_index(a: f32) -> i32 {
if !a.is_finite() || a <= 0.0 {
return 0;
}
((f64::from(a)).ln() / db_step() + 255.0).round().clamp(0.0, 255.0) as i32
}
/// A floor 1 configuration for blocks of `n/2 = 2^RANGE_BITS` bins.
#[derive(Clone, Debug)]
pub struct Floor1 {
/// X positions: 0, 1024, then the partitions' points.
pub xs: Vec<i32>,
/// Index of the book coding the point values.
pub book: usize,
/// For each point (in coding order) its low and high neighbours.
neighbours: Vec<(usize, usize)>,
/// Point indices sorted by X.
sorted: Vec<usize>,
}
impl Floor1 {
/// Points spaced about evenly in log frequency over 1..1024.
pub fn new(book: usize) -> Floor1 {
let count = CLASS_DIM * PARTITIONS;
let mut xs = vec![0, 1 << RANGE_BITS];
let mut last = 0;
for i in 1..=count {
// geometric from 2 to ~1000 bins, at least one bin apart
let x = (2.0f64 * (500.0f64).powf((i - 1) as f64 / (count - 1) as f64)).round() as i32;
let x = x.max(last + 1).min((1 << RANGE_BITS) - 1);
xs.push(x);
last = x;
}
let neighbours = (0..xs.len())
.map(|i| {
if i < 2 {
return (0, 0);
}
let (mut lo, mut hi) = (0usize, 1usize);
for j in 0..i {
if xs[j] < xs[i] && xs[j] > xs[lo] {
lo = j;
}
if xs[j] > xs[i] && xs[j] < xs[hi] {
hi = j;
}
}
(lo, hi)
})
.collect();
let mut sorted: Vec<usize> = (0..xs.len()).collect();
sorted.sort_by_key(|&i| xs[i]);
Floor1 { xs, book, neighbours, sorted }
}
/// The setup-header encoding (§7.2.2): one class of `CLASS_DIM` points, no subclasses.
pub fn write_header(&self, w: &mut BitWriter) {
w.write(PARTITIONS as u32, 5);
for _ in 0..PARTITIONS {
w.write(0, 4);
}
w.write(CLASS_DIM as u32 - 1, 3);
w.write(0, 2); // no subclasses
w.write(self.book as u32 + 1, 8);
w.write(MULTIPLIER - 1, 2);
w.write(RANGE_BITS, 4);
for &x in &self.xs[2..] {
w.write(x as u32, RANGE_BITS);
}
}
/// Codes the target values `want` (one per point, 0..RANGE) into `w`; returns the floor curve
/// the decoder renders (`n2` amplitudes).
pub fn encode(&self, w: &mut BitWriter, book: &Codebook, want: &[i32], n2: usize) -> Vec<f32> {
let n = self.xs.len();
let want = |i: usize| want.get(i).copied().unwrap_or(0).clamp(0, RANGE - 1);
let mut fin = vec![0i32; n];
let mut used = vec![false; n];
let mut coded = vec![0u32; n];
fin[0] = want(0);
fin[1] = want(1);
used[0] = true;
used[1] = true;
for i in 2..n {
let (lo, hi) = self.neighbours[i];
let pred = render_point(self.xs[lo], fin[lo], self.xs[hi], fin[hi], self.xs[i]);
let f = want(i);
let (highroom, lowroom) = (RANGE - pred, pred);
let room = 2 * highroom.min(lowroom);
let val = if f == pred {
0
} else {
let d = f - pred;
let inside = if d > 0 { 2 * d } else { -2 * d - 1 };
if inside < room {
inside
} else if highroom > lowroom {
f - pred + lowroom
} else {
pred - f + highroom - 1
}
};
coded[i] = val.clamp(0, RANGE - 1) as u32;
if val != 0 {
used[lo] = true;
used[hi] = true;
used[i] = true;
fin[i] = f;
} else {
fin[i] = pred;
}
}
let bits = ilog(RANGE as u32 - 1);
w.write(fin[0] as u32, bits);
w.write(fin[1] as u32, bits);
for &c in &coded[2..] {
book.put(w, c as usize);
}
self.render(&fin, &used, n2)
}
/// The decoder's curve (§7.2.4 step 2, `render_line`) as amplitudes.
fn render(&self, fin: &[i32], used: &[bool], n2: usize) -> Vec<f32> {
let mut v = vec![0i32; n2];
let m = MULTIPLIER as i32;
let (mut lx, mut ly) = (0i32, fin[self.sorted[0]] * m);
let (mut hx, mut hy) = (0i32, ly);
for &i in &self.sorted[1..] {
if used[i] {
hy = fin[i] * m;
hx = self.xs[i];
render_line(lx, ly, hx, hy, &mut v);
lx = hx;
ly = hy;
}
}
if (hx as usize) < n2 {
render_line(hx, hy, n2 as i32, hy, &mut v);
}
v.iter().map(|&i| inverse_db(i)).collect()
}
}
fn render_point(x0: i32, y0: i32, x1: i32, y1: i32, x: i32) -> i32 {
let dy = y1 - y0;
let adx = (x1 - x0).max(1);
let err = dy.abs() * (x - x0);
let off = err / adx;
if dy < 0 { y0 - off } else { y0 + off }
}
fn render_line(x0: i32, y0: i32, x1: i32, y1: i32, v: &mut [i32]) {
let dy = y1 - y0;
let adx = x1 - x0;
if adx <= 0 {
return;
}
let base = dy / adx;
let sy = if dy < 0 { base - 1 } else { base + 1 };
let ady = dy.abs() - base.abs() * adx;
let (mut y, mut err) = (y0, 0);
if let Some(s) = v.get_mut(x0 as usize) {
*s = y;
}
for x in x0 + 1..x1 {
err += ady;
if err >= adx {
err -= adx;
y += sy;
} else {
y += base;
}
if let Some(s) = v.get_mut(x as usize) {
*s = y;
}
}
}
+134
View File
@@ -0,0 +1,134 @@
//! Forward MDCT of a windowed block of `n` samples to `n/2` coefficients:
//! `X[k] = Σ x[j] cos(2π/n · (j + 1/2 + n/4)(k + 1/2))`, the transform the Vorbis decoder inverts
//! (Vorbis I §1.3.2, §4.3.9). Computed as a DCT-IV of the folded block through an `n/8`-point
//! complex FFT.
use std::f64::consts::PI;
#[derive(Clone, Copy, Default, Debug)]
struct Cpx {
re: f64,
im: f64,
}
impl Cpx {
fn mul(self, o: Cpx) -> Cpx {
Cpx { re: self.re * o.re - self.im * o.im, im: self.re * o.im + self.im * o.re }
}
}
/// A forward MDCT for one block size (a power of two, at least 16).
#[derive(Clone, Debug)]
pub struct Mdct {
n: usize,
twiddle: Vec<Cpx>,
post: Vec<Cpx>,
fft_tw: Vec<Cpx>,
rev: Vec<usize>,
}
impl Mdct {
pub fn new(n: usize) -> Mdct {
let m = n / 2; // DCT-IV length
let h = m / 2; // FFT length
let twiddle = (0..h).map(|j| cis(-PI * (4.0 * j as f64 + 1.0) / (4.0 * m as f64))).collect();
let post = (0..h).map(|k| cis(-PI * k as f64 / m as f64)).collect();
let fft_tw = (0..h / 2).map(|j| cis(-2.0 * PI * j as f64 / h as f64)).collect();
let bits = h.trailing_zeros();
let rev = (0..h).map(|i| if bits == 0 { 0 } else { i.reverse_bits() >> (usize::BITS - bits) }).collect();
Mdct { n, twiddle, post, fft_tw, rev }
}
/// `x`: `n` windowed samples; `out`: `n/2` coefficients (scaled by `scale`).
pub fn forward(&self, x: &[f32], out: &mut [f32], scale: f64) {
let n = self.n;
let m = n / 2;
let q = n / 4;
let at = |i: usize| f64::from(x.get(i).copied().unwrap_or(0.0));
// fold (a, b, c, d) to the DCT-IV input (-c_r - d, a - b_r)
let mut v = vec![0f64; m];
for (i, s) in v.iter_mut().enumerate() {
*s = if i < q { -at(3 * q - 1 - i) - at(3 * q + i) } else { at(i - q) - at(3 * q - 1 - i) };
}
// DCT-IV through a half-length complex FFT
let h = m / 2;
let mut z: Vec<Cpx> = (0..h).map(|j| Cpx { re: v[2 * j], im: v[m - 1 - 2 * j] }.mul(self.twiddle[j])).collect();
self.fft(&mut z);
for (k, c) in z.iter().enumerate() {
let d = c.mul(self.post[k]);
if let Some(o) = out.get_mut(2 * k) {
*o = (d.re * scale) as f32;
}
if let Some(o) = out.get_mut(m - 1 - 2 * k) {
*o = (-d.im * scale) as f32;
}
}
}
/// In-place radix-2 FFT (`e^{-2πi jk/h}`).
fn fft(&self, a: &mut [Cpx]) {
let h = a.len();
for i in 0..h {
let j = self.rev[i];
if i < j {
a.swap(i, j);
}
}
let mut len = 2;
while len <= h {
let step = h / len;
for start in (0..h).step_by(len) {
for k in 0..len / 2 {
let w = self.fft_tw[k * step];
let u = a[start + k];
let t = a[start + k + len / 2].mul(w);
a[start + k] = Cpx { re: u.re + t.re, im: u.im + t.im };
a[start + k + len / 2] = Cpx { re: u.re - t.re, im: u.im - t.im };
}
}
len *= 2;
}
}
}
fn cis(t: f64) -> Cpx {
Cpx { re: t.cos(), im: t.sin() }
}
/// The Vorbis window (§4.3.1) for a block of `n` between blocks of the same size.
pub fn window(n: usize) -> Vec<f32> {
(0..n)
.map(|i| {
let s = (PI * (i as f64 + 0.5) / n as f64).sin();
(PI / 2.0 * s * s).sin() as f32
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn matches_the_direct_formula() {
for n in [16usize, 64, 256, 2048] {
let x: Vec<f32> = (0..n).map(|i| ((i * 7919 % 211) as f32 - 105.0) / 50.0).collect();
let mut fast = vec![0f32; n / 2];
Mdct::new(n).forward(&x, &mut fast, 1.0);
for k in [0usize, 1, 2, n / 4, n / 2 - 1] {
let direct: f64 =
(0..n).map(|j| f64::from(x[j]) * (2.0 * PI / n as f64 * (j as f64 + 0.5 + n as f64 / 4.0) * (k as f64 + 0.5)).cos()).sum();
assert!((direct - f64::from(fast[k])).abs() < 1e-3 * (1.0 + direct.abs()), "n={n} k={k}: {direct} vs {}", fast[k]);
}
}
}
#[test]
fn the_window_is_power_complementary() {
let w = window(2048);
for i in 0..1024 {
let s = w[i] * w[i] + w[i + 1024] * w[i + 1024];
assert!((s - 1.0).abs() < 1e-5, "{i}: {s}");
}
}
}
+494
View File
@@ -0,0 +1,494 @@
//! Pure-Rust Vorbis I encoder, written from the Vorbis I specification (Xiph.Org, 2020-07-04).
//!
//! No `unsafe` and no C code; builds for `wasm32-unknown-unknown`. ffmpeg is used only as an
//! external test oracle.
//!
//! The stream: one mode of 2048-sample blocks; per channel a floor 1 envelope of 34 points; one
//! residue type 2 over the interleaved channels, partitions of 32 values (a multiple of the
//! channel count for 3, 5, 6 and 7 channels) in seven classes (silent,
//! three single-pass lattice books, and two- and three-pass cascades for large values); no channel
//! coupling. Quality (-1 to 10) sets how far the floor sits from the spectral envelope (below it
//! for tonal regions, above it for noise-like ones, from the spectral flatness), and with it the
//! residue's precision: variable bit rate, like other Vorbis encoders.
//!
//! From FilmCraft's `filmcraft-vorbis` by @JordyUSA (storytold/filmcraft#4481).
//!
//! The packets go into Ogg with [`craft_media::ogg::PageWriter`] and a comment header from
//! [`craft_media::ogg::vorbis_comment`]; [`crate::encode_ogg_vorbis`] does all of it for a
//! whole buffer.
//!
//! ```
//! use craft_audio::vorbis::encoder::{Encoder, EncoderConfig};
//!
//! let mut enc = Encoder::new(EncoderConfig::new(48_000, 2, 5.0))?;
//! let tone: Vec<f32> = (0..4_800).map(|i| (i as f32 * 0.05).sin() * 0.5).collect();
//! let mut packets = enc.encode(&[&tone, &tone])?;
//! packets.extend(enc.finish()?);
//! // the last granule position is the input's length, so a decoder trims the padding
//! assert_eq!(packets.last().map(|p| p.1), Some(4_800));
//! assert_eq!(&enc.identification()[1..7], b"vorbis");
//! # Ok::<(), craft_audio::vorbis::encoder::Error>(())
//! ```
// Index loops follow the specification, as in FilmCraft (which allows this lint workspace-wide).
#![allow(clippy::needless_range_loop)]
mod book;
mod floor;
mod mdct;
use book::{BitWriter, Codebook};
use floor::Floor1;
use mdct::Mdct;
/// Why an encoder could not be created or input could not be encoded.
#[non_exhaustive]
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum Error {
/// A setting or the input is outside what the encoder takes.
#[error("invalid argument: {0}")]
InvalidArgument(&'static str),
/// An encoder invariant failed (a bug, reported instead of panicking).
#[error("internal error: {0}")]
Internal(&'static str),
}
/// The encoder's result type.
pub type Result<T> = std::result::Result<T, Error>;
/// Block size (samples); the short block size of the header is never used.
pub const BLOCK: usize = 2048;
const HOP: usize = BLOCK / 2;
/// Residue values per partition for `channels` channels: 32, or for 3, 5, 6 and 7 channels a
/// multiple of the channel count (48, 40, 48, 56). libvorbis decodes each partition of an
/// interleaved (type 2) residue as if it started on channel 0, so a partition must hold whole
/// sample frames; the size also divides `channels × 1024` and is a multiple of every book's
/// dimension (4).
fn partition_size(channels: usize) -> usize {
match channels {
3 | 6 => 48,
5 => 40,
7 => 56,
_ => 32,
}
}
const CLASSES: usize = 8;
/// Largest residue magnitude the cascade codes: 7·225 + 7·15 + 7.
const MAX_RESIDUE: i32 = 1687;
/// The quality a NaN or infinite setting reads as.
pub const DEFAULT_QUALITY: f32 = 6.0;
/// Encoder settings.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct EncoderConfig {
/// Samples per second, 1 to 768 000.
pub sample_rate: u32,
/// 1 to 8, in Vorbis channel order (§4.3.9).
pub channels: usize,
/// -1 (smallest) to 10 (best); clamped.
pub quality: f32,
}
impl EncoderConfig {
/// Settings for `channels` channels (in Vorbis order) at `sample_rate` and `quality`.
pub fn new(sample_rate: u32, channels: usize, quality: f32) -> EncoderConfig {
EncoderConfig { sample_rate, channels, quality }
}
}
/// Book indices in the setup header.
const BOOK_CLASS: usize = 0;
const BOOK_FLOOR: usize = 1;
const BOOK_A1_SPARSE: usize = 2;
const BOOK_A1: usize = 3;
const BOOK_A2: usize = 4;
const BOOK_A4: usize = 5;
const BOOK_F: usize = 6;
const BOOK_M: usize = 7;
const BOOK_C: usize = 8;
/// Books per class and pass (`None`: nothing coded in that pass).
const CASCADE: [[Option<usize>; 3]; CLASSES] = [
[None, None, None],
[Some(BOOK_A1_SPARSE), None, None],
[Some(BOOK_A1), None, None],
[Some(BOOK_A2), None, None],
[Some(BOOK_A4), None, None],
[Some(BOOK_F), None, None],
[Some(BOOK_M), Some(BOOK_F), None],
[Some(BOOK_C), Some(BOOK_M), Some(BOOK_F)],
];
/// A Vorbis encoder for one stream: planar `f32` in, Vorbis packets with their granule
/// positions out.
pub struct Encoder {
cfg: EncoderConfig,
books: Vec<Codebook>,
floor: Floor1,
mdct: Mdct,
window: Vec<f32>,
/// Input not yet past a block, per channel; the next block starts at `start`.
pending: Vec<Vec<f32>>,
/// Read cursor into `pending`: consumed hops are dropped once per call, not per block.
start: usize,
samples_in: u64,
blocks: u64,
finished: bool,
/// Floor offset below the envelope (dB) for tonal and for noise-like regions.
tonal_db: f32,
noise_db: f32,
/// Residue magnitudes below this quantise to 0.
dead_zone: f32,
}
impl Encoder {
/// An encoder for `cfg`: 1 to 8 channels, a sample rate of 1 to 768 000; the quality is
/// clamped to -1..=10, and NaN or infinite reads as [`DEFAULT_QUALITY`].
pub fn new(cfg: EncoderConfig) -> Result<Encoder> {
if !(1..=8).contains(&cfg.channels) {
return Err(Error::InvalidArgument("Vorbis encoder: 1 to 8 channels"));
}
if !(1..=768_000).contains(&cfg.sample_rate) {
return Err(Error::InvalidArgument("Vorbis encoder: sample rate"));
}
let quality = if cfg.quality.is_finite() { cfg.quality.clamp(-1.0, 10.0) } else { DEFAULT_QUALITY };
let cfg = EncoderConfig { quality, ..cfg };
let books = vec![
Codebook::scalar(2, &(0..CLASSES * CLASSES).map(|e| class_weight(e / CLASSES) * class_weight(e % CLASSES)).collect::<Vec<_>>())?,
Codebook::scalar(1, &(0..floor::RANGE).map(|v| 1.0 / (1.0 + f64::from(v)).powf(1.6)).collect::<Vec<_>>())?,
Codebook::lattice(4, 3, -1, 1, |v| if v == 0 { 0.92 } else { 0.04 })?,
Codebook::lattice(4, 3, -1, 1, |v| if v == 0 { 0.5 } else { 0.25 })?,
Codebook::lattice(2, 5, -2, 1, |v| 0.6f64.powi(v.abs()))?,
Codebook::lattice(2, 9, -4, 1, |v| 0.7f64.powi(v.abs()))?,
Codebook::lattice(2, 15, -7, 1, |v| 0.8f64.powi(v.abs()))?,
Codebook::lattice(2, 15, -105, 15, |v| 0.55f64.powi((v / 15).abs()))?,
Codebook::lattice(2, 15, -1575, 225, |v| 0.4f64.powi((v / 225).abs()))?,
];
Ok(Encoder {
cfg,
books,
floor: Floor1::new(BOOK_FLOOR),
mdct: Mdct::new(BLOCK),
window: mdct::window(BLOCK),
// the first block is centred on sample 0: the decoder's first output is input sample 0
pending: vec![vec![0.0; HOP]; cfg.channels],
start: 0,
samples_in: 0,
blocks: 0,
finished: false,
// tones 4 dB (-1) to 30.4 dB (10) under their envelope; noise 14 dB above it (-1)
// to 4.1 dB (10)
tonal_db: 4.0 + 2.4 * (quality + 1.0),
noise_db: -14.0 + 0.9 * (quality + 1.0),
dead_zone: 0.8 - 0.025 * (quality + 1.0),
})
}
/// Channels per sample frame.
pub fn channels(&self) -> usize {
self.cfg.channels
}
/// The quality in use, after clamping.
pub fn quality(&self) -> f32 {
self.cfg.quality
}
/// Input samples per channel taken so far.
pub fn samples_in(&self) -> u64 {
self.samples_in
}
/// The identification header packet (§4.2.2): the first packet of the stream.
pub fn identification(&self) -> Vec<u8> {
let mut w = BitWriter::new();
w.write(1, 8);
w.bytes(b"vorbis");
w.write(0, 32);
w.write(self.cfg.channels as u32, 8);
w.write(self.cfg.sample_rate, 32);
w.write(0, 32); // bit rates: unset (variable)
w.write(0, 32);
w.write(0, 32);
w.write(8, 4); // 256-sample short blocks (unused)
w.write(11, 4); // 2048-sample long blocks
w.flag(true);
w.finish()
}
/// The setup header packet (§4.2.4): codebooks, floor, residue, mapping and mode. It follows
/// the comment header.
pub fn setup(&self) -> Vec<u8> {
let mut w = BitWriter::new();
w.write(5, 8);
w.bytes(b"vorbis");
w.write(self.books.len() as u32 - 1, 8);
for b in &self.books {
b.write_header(&mut w);
}
w.write(0, 6); // one time-domain transform placeholder
w.write(0, 16);
w.write(0, 6); // one floor
w.write(1, 16);
self.floor.write_header(&mut w);
w.write(0, 6); // one residue, type 2
w.write(2, 16);
let end = (self.cfg.channels * HOP) as u32;
w.write(0, 24);
w.write(end, 24);
w.write(partition_size(self.cfg.channels) as u32 - 1, 24);
w.write(CLASSES as u32 - 1, 6);
w.write(BOOK_CLASS as u32, 8);
for c in &CASCADE {
let bits: u32 = c.iter().enumerate().map(|(j, b)| u32::from(b.is_some()) << j).sum();
w.write(bits & 7, 3);
w.flag(bits > 7);
if bits > 7 {
w.write(bits >> 3, 5);
}
}
for c in &CASCADE {
for b in c.iter().flatten() {
w.write(*b as u32, 8);
}
}
w.write(0, 6); // one mapping, type 0
w.write(0, 16);
w.flag(false); // one submap
w.flag(false); // no coupling
w.write(0, 2);
w.write(0, 8); // submap 0: time, floor 0, residue 0
w.write(0, 8);
w.write(0, 8);
w.write(0, 6); // one mode: long blocks, mapping 0
w.flag(true);
w.write(0, 16);
w.write(0, 16);
w.write(0, 8);
w.flag(true);
w.finish()
}
/// Encodes planar samples (one slice per channel, all the same length, nominal range ±1);
/// returns the packets completed with their granule positions. NaN or infinite samples, or a
/// channel count or length mismatch, are an error.
pub fn encode(&mut self, planar: &[&[f32]]) -> Result<Vec<(Vec<u8>, i64)>> {
if self.finished {
return Err(Error::InvalidArgument("Vorbis encoder: encode after finish"));
}
if planar.len() != self.cfg.channels {
return Err(Error::InvalidArgument("Vorbis encoder: one slice per channel"));
}
let len = planar.first().map_or(0, |p| p.len());
if planar.iter().any(|p| p.len() != len) {
return Err(Error::InvalidArgument("Vorbis encoder: channels of different lengths"));
}
if planar.iter().any(|p| p.iter().any(|v| !v.is_finite())) {
return Err(Error::InvalidArgument("Vorbis encoder: NaN or infinite sample"));
}
for (b, p) in self.pending.iter_mut().zip(planar) {
b.extend_from_slice(p);
}
self.samples_in += len as u64;
self.drain()
}
/// Encodes what remains, padded with silence; the last packet's granule position is the
/// input's length, so the decoder trims the padding (§A.2).
pub fn finish(&mut self) -> Result<Vec<(Vec<u8>, i64)>> {
if self.finished {
return Ok(Vec::new());
}
self.finished = true;
// block k ends output at k·HOP: blocks 0..=ceil(len / HOP)
let blocks = self.samples_in.div_ceil(HOP as u64) + 1;
let missing = blocks.saturating_sub(self.blocks) as usize;
self.compact();
let need = (missing + 1) * HOP;
for b in self.pending.iter_mut() {
if b.len() < need {
b.resize(need, 0.0);
}
}
let mut out = Vec::new();
for _ in 0..missing {
out.push(self.block()?);
}
Ok(out)
}
fn drain(&mut self) -> Result<Vec<(Vec<u8>, i64)>> {
let mut out = Vec::new();
while self.pending.first().is_some_and(|b| b.len().saturating_sub(self.start) >= BLOCK) {
out.push(self.block()?);
}
self.compact();
Ok(out)
}
/// Drops the hops already encoded from the front of every channel buffer.
fn compact(&mut self) {
for b in self.pending.iter_mut() {
b.drain(..self.start.min(b.len()));
}
self.start = 0;
}
/// Encodes the block at `start` in `pending` and advances by a hop.
fn block(&mut self) -> Result<(Vec<u8>, i64)> {
let ch = self.cfg.channels;
let mut w = BitWriter::new();
w.write(0, 1); // audio packet; one mode, so no mode number
w.flag(true); // previous and next windows long
w.flag(true);
let mut residue = vec![0i32; ch * HOP];
let mut any = false;
let mut x = vec![0f32; BLOCK];
let mut coeffs = vec![0f32; HOP];
for c in 0..ch {
let src = self.pending.get(c).and_then(|b| b.get(self.start..)).ok_or(Error::Internal("channel buffer"))?;
for (i, v) in x.iter_mut().enumerate() {
*v = src.get(i).copied().unwrap_or(0.0).clamp(-8.0, 8.0) * self.window[i];
}
self.mdct.forward(&x, &mut coeffs, 2.0 / HOP as f64);
let peak = coeffs.iter().fold(0f32, |m, v| m.max(v.abs()));
// (a NaN peak cannot happen: samples are finite and clamped)
if peak <= 1e-7 {
w.flag(false); // floor unused: the channel is silent
continue;
}
any = true;
w.flag(true);
let want = self.floor_targets(&coeffs);
let curve = self.floor.encode(&mut w, &self.books[BOOK_FLOOR], &want, HOP);
for (k, (&v, &f)) in coeffs.iter().zip(&curve).enumerate() {
let r = if f > 0.0 { v / f } else { 0.0 };
// a dead zone: values just past half a step are noise peaks, not worth a ±1
let q = if r.abs() < self.dead_zone { 0.0 } else { r.round() };
residue[k * ch + c] = if q.is_finite() { (q as i32).clamp(-MAX_RESIDUE, MAX_RESIDUE) } else { 0 };
}
}
if any {
self.write_residue(&mut w, &residue);
}
let granule = if self.blocks == 0 {
0
} else {
let natural = self.blocks.saturating_mul(HOP as u64);
if self.finished { natural.min(self.samples_in) } else { natural }
};
self.start += HOP;
self.blocks += 1;
Ok((w.finish(), i64::try_from(granule).unwrap_or(i64::MAX)))
}
/// Floor values at the floor's points: the spectral envelope (power smoothed over a quarter of
/// the frequency, at least 8 bins, on each side) less an offset that grows with the region's tonality, from its
/// spectral flatness: noise needs a few dB of precision, tones much more.
fn floor_targets(&self, coeffs: &[f32]) -> Vec<i32> {
let n = coeffs.len();
let mut cum = vec![0f64; n + 1];
let mut cum_ln = vec![0f64; n + 1];
for (i, &v) in coeffs.iter().enumerate() {
let p = f64::from(v) * f64::from(v);
cum[i + 1] = cum[i] + p;
cum_ln[i + 1] = cum_ln[i] + (p + 1e-30).ln();
}
self.floor
.xs
.iter()
.map(|&x| {
let k = (x as usize).min(n - 1);
let half = (k / 4).max(8);
let (lo, hi) = (k.saturating_sub(half), (k + half + 1).min(n));
let len = (hi - lo) as f64;
let am = (cum[hi] - cum[lo]) / len;
let gm = ((cum_ln[hi] - cum_ln[lo]) / len).exp();
// spectral flatness: about -2.5 dB for noise, far below for a tone
let sfm_db = 10.0 * (gm / am.max(1e-30)).max(1e-30).log10();
let tonality = ((-sfm_db - 2.5) / 20.0).clamp(0.0, 1.0);
let offset = tonality * f64::from(self.tonal_db) + (1.0 - tonality) * f64::from(self.noise_db);
let amp = (am.sqrt() * 10f64.powf(-offset / 20.0)).max(1e-7) as f32;
(floor::db_index(amp) + 1) / floor::MULTIPLIER as i32
})
.collect()
}
/// Residue type 2 (§8.6.2 with the interleaved vector): classes per partition, then the
/// cascade's passes in the decoder's order.
fn write_residue(&self, w: &mut BitWriter, r: &[i32]) {
let partition = partition_size(self.cfg.channels);
let parts = r.len() / partition;
let classes: Vec<usize> = r.chunks(partition).map(class_of).collect();
let per_word = self.books[BOOK_CLASS].dimensions as usize;
for pass in 0..3 {
let mut pc = 0;
while pc < parts {
if pass == 0 {
let mut e = 0usize;
for i in 0..per_word {
e = e * CLASSES + classes.get(pc + i).copied().unwrap_or(0);
}
self.books[BOOK_CLASS].put(w, e);
}
for _ in 0..per_word {
if pc >= parts {
break;
}
let class = classes[pc];
if let Some(b) = CASCADE[class][pass] {
let book = &self.books[b];
let dims = book.dimensions as usize;
let part = &r[pc * partition..(pc + 1) * partition];
for chunk in part.chunks(dims) {
let v: Vec<i32> = chunk.iter().map(|&x| pass_value(x, class, pass)).collect();
book.put(w, book.entry_of(&v));
}
}
pc += 1;
}
}
}
}
}
/// Relative frequency of each residue class (for the class book's code lengths).
fn class_weight(c: usize) -> f64 {
[0.3, 0.2, 0.12, 0.12, 0.1, 0.08, 0.06, 0.02].get(c).copied().unwrap_or(0.01)
}
/// The part of residue value `v` coded in `pass` for `class`: the passes add up to `v`.
fn pass_value(v: i32, class: usize, pass: usize) -> i32 {
let split = |v: i32, step: i32| -> (i32, i32) {
let a = ((f64::from(v) / f64::from(step)).round() as i32).clamp(-7, 7);
(a * step, v - a * step)
};
match (class, pass) {
(6, 0) => split(v, 15).0,
(6, _) => split(v, 15).1,
(7, 0) => split(v, 225).0,
(7, 1) => split(split(v, 225).1, 15).0,
(7, _) => split(split(v, 225).1, 15).1,
_ => v,
}
}
/// The residue class of a partition: silent; ±1 with at most four nonzero values, or more; then
/// by the largest magnitude.
fn class_of(p: &[i32]) -> usize {
let m = p.iter().map(|v| v.abs()).max().unwrap_or(0);
match m {
0 => 0,
1 if p.iter().filter(|&&v| v != 0).count() <= 4 => 1,
1 => 2,
2 => 3,
3..=4 => 4,
5..=7 => 5,
8..=112 => 6,
_ => 7,
}
}
#[cfg(test)]
mod tests;
+285
View File
@@ -0,0 +1,285 @@
//! Unit tests, and ffmpeg (as an external oracle only) decoding the encoder's streams.
#![allow(clippy::unwrap_used)]
use std::path::Path;
use std::process::Command;
use super::book::{BitWriter, assign_codewords, float32_pack};
use super::{BLOCK, Encoder, EncoderConfig, HOP, partition_size};
#[test]
fn codewords_follow_the_specification_example() {
// Vorbis I §3.2.1: lengths 2 4 4 4 4 2 3 3
let words = assign_codewords(&[2, 4, 4, 4, 4, 2, 3, 3]).unwrap();
assert_eq!(words, [0b00, 0b0100, 0b0101, 0b0110, 0b0111, 0b10, 0b110, 0b111]);
assert!(assign_codewords(&[1, 1, 1]).is_err(), "over-subscribed");
}
#[test]
fn every_book_is_a_complete_prefix_code() {
let e = Encoder::new(EncoderConfig::new(48_000, 2, 5.0)).unwrap();
for (i, b) in e.books.iter().enumerate() {
let kraft: f64 = b.lengths.iter().map(|&l| 0.5f64.powi(l as i32)).sum();
assert!((kraft - 1.0).abs() < 1e-9, "book {i}: Kraft sum {kraft}");
assert!(b.lengths.iter().all(|&l| (1..=20).contains(&l)));
for (a, (&wa, &la)) in b.words.iter().zip(&b.lengths).enumerate() {
for (c, (&wc, &lc)) in b.words.iter().zip(&b.lengths).enumerate() {
if a != c && la <= lc {
assert_ne!(wc >> (lc - la), wa, "book {i}: codeword {a} is a prefix of {c}");
}
}
}
}
}
#[test]
fn float32_pack_is_exact_for_lattice_values() {
let unpack = |x: u32| {
let mant = f64::from(x & 0x1F_FFFF);
let exp = ((x >> 21) & 0x3FF) as i32 - 788;
let v = mant * 2f64.powi(exp);
if x & 0x8000_0000 != 0 { -v } else { v }
};
for v in [1.0, -1.0, -2.0, -4.0, -7.0, -105.0, 15.0, -1575.0, 225.0, 0.5, 3.0] {
assert_eq!(unpack(float32_pack(v)), v);
}
assert_eq!(float32_pack(0.0), 0);
}
#[test]
fn bits_are_packed_lsb_first() {
let mut w = BitWriter::new();
w.write(0b1, 1);
w.write(0b10, 2);
w.write(0x1F, 5);
w.write(0xABC, 12);
assert_eq!(w.finish(), [0b1111_1101, 0xBC, 0x0A]);
}
#[test]
fn hostile_input_is_an_error_not_a_panic() {
assert!(Encoder::new(EncoderConfig::new(48_000, 0, 3.0)).is_err());
assert!(Encoder::new(EncoderConfig::new(48_000, 9, 3.0)).is_err());
assert!(Encoder::new(EncoderConfig::new(0, 2, 3.0)).is_err());
assert_eq!(Encoder::new(EncoderConfig::new(48_000, 1, f32::NAN)).unwrap().cfg.quality, 6.0, "NaN quality uses the default, as the export does");
let mut e = Encoder::new(EncoderConfig::new(44_100, 2, 3.0)).unwrap();
assert!(e.encode(&[&[0.0; 4]]).is_err());
assert!(e.encode(&[&[0.0; 4], &[0.0; 3]]).is_err());
assert!(e.encode(&[&[f32::NAN; 4], &[0.0; 4]]).is_err());
let loud = vec![1e30f32; 5000];
assert!(!e.encode(&[&loud, &loud]).unwrap().is_empty());
assert!(!e.finish().unwrap().is_empty());
assert!(e.finish().unwrap().is_empty());
assert!(e.encode(&[&[0.0; 4], &[0.0; 4]]).is_err());
}
#[test]
fn granules_count_the_samples_and_trim_the_end() {
for len in [0usize, 1, 1023, 1024, 1025, 48_000] {
let mut e = Encoder::new(EncoderConfig::new(48_000, 1, 3.0)).unwrap();
let x = vec![0.1f32; len];
let mut packets = e.encode(&[&x]).unwrap();
packets.extend(e.finish().unwrap());
assert_eq!(packets.len(), len.div_ceil(HOP) + 1, "len {len}");
// packet k ends output at k·1024 samples; the last one at the input's length
let want: Vec<i64> = (0..packets.len()).map(|k| ((k * HOP) as i64).min(len as i64)).collect();
let got: Vec<i64> = packets.iter().map(|p| p.1).collect();
assert_eq!(got, want, "len {len}");
}
}
// --- ffmpeg oracle ---
/// An Ogg Vorbis file of the three headers and the audio packets, through craft-media's writer.
fn ogg(packets: &[(Vec<u8>, i64)]) -> Vec<u8> {
let mut w = craft_media::ogg::PageWriter::new(7);
let mut out = Vec::new();
for (i, (p, g)) in packets.iter().enumerate() {
out.extend(if i < 3 { w.header(p) } else { w.packet(p, *g) });
}
out.extend(w.finish(None));
out
}
fn comment() -> Vec<u8> {
craft_media::ogg::vorbis_comment("test", &[])
}
/// Encodes `input` (planar) and decodes it with ffmpeg's `decoder`; returns the planar output.
/// ffmpeg's decoders to check: its own `vorbis` always, and `libvorbis` when ffmpeg has it.
fn decoders(ff: &Path) -> Vec<&'static str> {
let list =
Command::new(ff).args(["-hide_banner", "-decoders"]).output().map(|o| String::from_utf8_lossy(&o.stdout).into_owned()).unwrap_or_default();
if list.lines().any(|l| l.split_whitespace().nth(1) == Some("libvorbis")) {
vec!["vorbis", "libvorbis"]
} else {
eprintln!("SKIPPED (libvorbis decoder): this ffmpeg has no libvorbis; checking its own decoder only");
vec!["vorbis"]
}
}
fn round_trip(input: &[Vec<f32>], rate: u32, quality: f32, decoder: &str, name: &str) -> Option<(Vec<Vec<f32>>, usize)> {
let ff = craft_media_testkit::ffmpeg_or_skip("vorbis oracle")?;
let mut e = Encoder::new(EncoderConfig::new(rate, input.len(), quality)).unwrap();
let mut packets = vec![(e.identification(), 0), (comment(), 0), (e.setup(), 0)];
let n = input[0].len();
let mut at = 0;
while at < n {
let end = (at + 4410).min(n);
let s: Vec<&[f32]> = input.iter().map(|c| &c[at..end]).collect();
packets.extend(e.encode(&s).unwrap());
at = end;
}
packets.extend(e.finish().unwrap());
let bytes: usize = packets[3..].iter().map(|p| p.0.len()).sum();
let dir = craft_media_testkit::fixtures_dir("audio/vorbis-encoder");
let path = dir.join(format!("{name}.ogg"));
std::fs::write(&path, ogg(&packets)).unwrap();
let out = Command::new(ff)
.args(["-v", "error", "-xerror", "-c:a", decoder, "-i"])
.arg(&path)
.args(["-f", "f32le", "-c:a", "pcm_f32le", "-"])
.output()
.unwrap();
assert!(out.status.success() && out.stderr.is_empty(), "{name} {decoder}: {}", String::from_utf8_lossy(&out.stderr));
let pcm: Vec<f32> = out.stdout.as_chunks::<4>().0.iter().map(|&b| f32::from_le_bytes(b)).collect();
let ch = input.len();
Some(((0..ch).map(|c| pcm.iter().skip(c).step_by(ch).copied().collect()).collect(), bytes))
}
fn snr_db(a: &[f32], b: &[f32]) -> f64 {
let (mut s, mut e) = (0f64, 0f64);
for (&x, &y) in a.iter().zip(b) {
s += f64::from(x) * f64::from(x);
e += (f64::from(x) - f64::from(y)).powi(2);
}
10.0 * (s.max(1e-20) / e.max(1e-20)).log10()
}
fn music(n: usize, seed: u32) -> Vec<f32> {
let mut r = seed;
(0..n)
.map(|i| {
let t = i as f32 / 48_000.0;
r = r.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
let noise = (r >> 8) as f32 / 8_388_608.0 - 1.0;
let chord: f32 = [220.0f32, 277.18, 329.63, 440.0]
.iter()
.enumerate()
.map(|(k, f)| (0.2 / (k + 1) as f32) * (2.0 * std::f32::consts::PI * f * t).sin())
.sum();
chord + 0.01 * noise
})
.collect()
}
/// Harmonic chords changing every half second, with decaying noise bursts (music-like).
fn chords(n: usize, seed: u32) -> Vec<f32> {
let mut r = seed;
let notes = [220.0f32, 277.2, 329.6, 440.0, 349.2, 523.3, 392.0, 659.3];
(0..n)
.map(|i| {
let t = i as f32 / 48_000.0;
let chord = (t * 2.0) as usize;
let mut s = 0.0;
for k in 0..3 {
let f = notes[(chord * 3 + k * 2) % notes.len()];
for h in 1..6 {
s += (2.0 * std::f32::consts::PI * f * h as f32 * t).sin() * 0.08 / h as f32;
}
}
r = r.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
let env = (-(t * 2.0).fract() * 3.0).exp();
s * (0.5 + 0.5 * env) + 0.02 * ((r >> 8) as f32 / 8_388_608.0 - 1.0) * env
})
.collect()
}
/// ffmpeg's own Vorbis decoder and libvorbis read every stream exactly as long as the input,
/// with SNR floors about 6 dB (chords 4 dB) under what the encoder measured when written, and bit
/// rates near libvorbis's for the same quality.
/// A signal, and per quality the SNR floor and the bit-rate range (kbit/s).
type Case = (&'static str, Vec<Vec<f32>>, [(f32, f64, std::ops::Range<usize>); 5]);
#[test]
fn ffmpeg_decodes_every_quality() {
let Some(ff) = craft_media_testkit::ffmpeg_or_skip("vorbis oracle") else { return };
let n = 48_000 * 3;
let cases: [Case; 2] = [
(
"music",
vec![music(n, 1), music(n, 2).iter().map(|v| v * 0.7).collect()],
[(-1.0, 3.0, 15..90), (0.0, 6.5, 20..110), (3.0, 12.0, 60..180), (6.0, 18.5, 120..280), (10.0, 25.5, 180..400)],
),
(
"chords",
vec![chords(n, 1), chords(n, 2).iter().map(|v| v * 0.8).collect()],
[(-1.0, 4.5, 20..90), (0.0, 6.0, 25..110), (3.0, 10.0, 70..200), (6.0, 14.0, 130..300), (10.0, 19.0, 180..420)],
),
];
for (name, input, levels) in &cases {
for (q, floor, kbps) in levels {
for decoder in decoders(&ff) {
let Some((out, bytes)) = round_trip(input, 48_000, *q, decoder, &format!("{name}{q}")) else { return };
assert!(out.iter().all(|c| c.len() == n), "{name} q{q} {decoder}: length {}", out[0].len());
let snr = snr_db(&input[0][4096..], &out[0][4096..]);
assert!(snr >= *floor, "{name} q{q} {decoder}: {snr:.2} dB < {floor}");
let rate = bytes * 8 / 3 / 1000;
assert!(kbps.contains(&rate), "{name} q{q}: {rate} kbit/s outside {kbps:?}");
}
}
}
}
/// Mono, 44.1 kHz, 3.0, 5.1 and 6.1 streams decode too, with libvorbis as well (which reads each
/// residue partition from channel 0); a silent channel costs one bit per block.
#[test]
fn channel_counts_and_rates() {
let Some(ff) = craft_media_testkit::ffmpeg_or_skip("vorbis oracle") else { return };
let n = 44_100;
for (rate, ch) in [(44_100u32, 1usize), (22_050, 2), (48_000, 3), (48_000, 6), (48_000, 7)] {
let mut input: Vec<Vec<f32>> = (0..ch).map(|c| chords(n, c as u32 + 1).iter().map(|v| v * (1.0 - 0.1 * c as f32)).collect()).collect();
if ch == 6 {
input[3] = vec![0.0; n];
}
for decoder in decoders(&ff) {
let Some((mut out, _)) = round_trip(&input, rate, 5.0, decoder, &format!("ch{ch}_{rate}_{decoder}")) else { return };
if decoder == "vorbis" {
// ffmpeg's decoder gives WAV order (5.1: FL FR FC LFE BL BR); the stream is in Vorbis
// order (L C R Ls Rs LFE), which libvorbis passes through
let wav_to_vorbis: &[usize] = match ch {
3 => &[0, 2, 1],
6 => &[0, 2, 1, 4, 5, 3],
7 => &[0, 2, 1, 5, 6, 4, 3],
_ => &[0, 1],
};
if ch > 2 {
out = wav_to_vorbis.iter().map(|&i| out[i].clone()).collect();
}
}
assert_eq!(out.len(), ch);
for (c, (a, b)) in input.iter().zip(&out).enumerate() {
assert_eq!(b.len(), n, "{ch} ch {rate} Hz {decoder}: channel {c} length");
if a.iter().all(|&v| v == 0.0) {
assert!(b.iter().all(|v| v.abs() < 1e-6), "silent channel {c}");
} else {
let snr = snr_db(&a[4096..], &b[4096..]);
assert!(snr > 8.0, "{ch} ch {rate} Hz {decoder} channel {c}: {snr:.1} dB");
}
}
}
}
}
/// Every partition holds whole sample frames (libvorbis decodes each one from channel 0), splits
/// the interleaved residue evenly and fits every book's dimension.
#[test]
fn partitions_hold_whole_frames() {
for ch in 1..=8 {
let p = partition_size(ch);
assert_eq!(p % ch, 0, "{ch} channels");
assert_eq!((ch * BLOCK / 2) % p, 0, "{ch} channels");
assert_eq!(p % 4, 0, "{ch} channels");
}
}
+4
View File
@@ -0,0 +1,4 @@
//! Vorbis I (feature `vorbis-encode`): the pure-Rust [`encoder`]. Decoding Vorbis goes through
//! [`crate::decode`].
pub mod encoder;
+721
View File
@@ -0,0 +1,721 @@
//! Markers a WAV file carries, read from its RIFF chunks.
//!
//! Two encodings are read into separate lists for the caller to map and merge:
//! - **Cue points** (the standard): a `cue ` chunk with one point per marker, named by `labl` (or
//! `note`) sub-chunks of a `LIST` `adtl` chunk, and given a length by `ltxt`.
//! - **XMP** (Adobe's tools): an `_PMX` chunk holding an XMP packet whose `xmpDM:markers` list has
//! one entry per marker (`xmpDM:startTime`, `xmpDM:duration`, `xmpDM:name`), counted in frames
//! of `xmpDM:frameRate` (`f44100` for samples at 44.1 kHz), or with an explicit
//! FrameCount rate suffix (`15f24`, `15f30000s1001`). Durations with a distinct rate
//! are converted to the marker start rate without app-specific time quantization.
//!
//! RF64 files (WAVs over 4 GB) carry a `0xFFFFFFFF` size for a chunk too big for 32 bits and give its real size in a
//! `ds64` chunk; those sizes are resolved so the chunks after the audio are still found.
//!
//! The file is untrusted input: every read is bounds-checked, sizes use checked arithmetic, and
//! the number of markers, of `adtl` sub-chunks looked at and the size of the XMP packet are capped.
use std::collections::{HashMap, HashSet};
/// Marker coordinates before app-specific time quantization. Explicit XMP duration
/// rates are normalized to the start coordinate rate; bare counts retain the legacy
/// track/file-rate interpretation.
#[derive(Clone, Debug, PartialEq)]
pub struct Marker {
/// Start in frames at `rate`.
pub start: f64,
/// Duration in frames at `rate`.
pub duration: f64,
/// Frames per second; consumers decide how invalid or absent rates map to their timeline.
pub rate: f64,
/// Label text.
pub name: String,
}
/// Separate source lists: rounding, deduplication, ordering and labels belong to the caller.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Markers {
/// RIFF cue/adtl markers in file order.
pub cue: Vec<Marker>,
/// XMP markers in packet order.
pub xmp: Vec<Marker>,
/// A chunk-inventory or cue-label retention limit omitted metadata. Marker coordinates
/// remain available when only their label was omitted. Existing marker/XMP prefix limits
/// are separate and do not set this flag.
pub limited: bool,
}
/// Most markers read from one file.
pub const MAX_MARKERS: usize = 10_000;
/// Largest XMP packet read (bytes).
const MAX_XMP: usize = 4 << 20;
/// Most `adtl` sub-chunks (`labl`, `note`, `ltxt`) looked at in one file: a few per cue point is plenty.
const MAX_ADTL_RECORDS: usize = 4 * MAX_MARKERS;
/// Most entries read from an RF64 file's `ds64` size table.
const MAX_DS64_TABLE: usize = 1_024;
/// Maximum retained RIFF chunk descriptors (including empty chunks).
pub const MAX_CHUNKS: usize = 100_000;
/// Maximum UTF-8 bytes in one retained cue label.
pub const MAX_LABEL_BYTES: usize = 64 * 1024;
/// Aggregate cue label bytes, enforced separately for the name map and emitted markers.
pub const MAX_LABEL_TOTAL_BYTES: usize = 4 * 1024 * 1024;
fn u32_at(b: &[u8], at: usize) -> Option<u32> {
let s = b.get(at..at.checked_add(4)?)?;
Some(u32::from_le_bytes(s.try_into().ok()?))
}
fn u64_at(b: &[u8], at: usize) -> Option<u64> {
let s = b.get(at..at.checked_add(8)?)?;
Some(u64::from_le_bytes(s.try_into().ok()?))
}
/// The text of a NUL-terminated label (the rest of the chunk), lossily decoded.
fn label(b: &[u8]) -> Option<String> {
// Look only as far as one label can be retained; never copy an arbitrary chunk body.
let end = b.iter().take(MAX_LABEL_BYTES + 1).position(|&c| c == 0).unwrap_or(b.len());
if end > MAX_LABEL_BYTES {
return None;
}
let text = String::from_utf8_lossy(b.get(..end)?);
let text = text.trim();
(text.len() <= MAX_LABEL_BYTES).then(|| text.to_string())
}
/// Whether `bytes` start a RIFF/WAVE file.
pub fn is_wav(bytes: &[u8]) -> bool {
matches!(bytes.get(0..4), Some(b"RIFF" | b"RF64")) && bytes.get(8..12) == Some(b"WAVE")
}
/// The sizes an RF64 file's `ds64` chunk gives for chunks too big for the 32-bit size field.
struct Ds64 {
/// The `data` chunk's size.
data: u64,
/// Other chunks: `(id, size)`.
table: Vec<([u8; 4], u64)>,
}
impl Ds64 {
/// The real size of chunk `id`, when the file says.
fn size_of(&self, id: &[u8; 4]) -> Option<u64> {
if id == b"data" { Some(self.data) } else { self.table.iter().find(|(t, _)| t == id).map(|(_, size)| *size) }
}
}
/// The `ds64` chunk of an RF64 file (`RF64`, size, `WAVE`, then `ds64`); `None` for any other file or one whose
/// `ds64` doesn't fit.
fn ds64(bytes: &[u8]) -> Option<Ds64> {
if bytes.get(0..4) != Some(b"RF64") || bytes.get(12..16) != Some(b"ds64") {
return None;
}
let size = usize::try_from(u32_at(bytes, 16)?).ok()?;
let body = bytes.get(20..20usize.checked_add(size)?)?;
// riff size (8), data size (8), sample count (8), table length (4), then `(id, size)` entries of 12 bytes
let data = u64_at(body, 8)?;
let entries = usize::try_from(u32_at(body, 24)?).ok()?;
let mut table = Vec::new();
for i in 0..entries.min(MAX_DS64_TABLE) {
let at = 28usize.checked_add(i.checked_mul(12)?)?;
let id = <[u8; 4]>::try_from(body.get(at..at.checked_add(4)?)?).ok()?;
table.push((id, u64_at(body, at + 4)?));
}
Some(Ds64 { data, table })
}
/// The chunks of a WAVE file as `(id, body)`, stopping at the first one that doesn't fit.
fn chunks<'a>(bytes: &'a [u8], limited: &mut bool) -> Vec<([u8; 4], &'a [u8])> {
let big = ds64(bytes);
let mut out = Vec::new();
let mut at = 12usize;
while let Some(head) = bytes.get(at..at.saturating_add(8)) {
if out.len() == MAX_CHUNKS {
*limited = true;
break;
}
let (Ok(id), Some(field)) = (<[u8; 4]>::try_from(&head[..4]), u32_at(head, 4)) else { break };
// In an RF64 file a size field of 0xFFFFFFFF stands for the size `ds64` gives.
let size = match &big {
Some(d) if field == u32::MAX => d.size_of(&id).unwrap_or(u64::from(field)),
_ => u64::from(field),
};
let start = at.saturating_add(8);
let Some(end) = usize::try_from(size).ok().and_then(|n| start.checked_add(n)) else { break };
let Some(body) = bytes.get(start..end) else { break };
out.push((id, body));
// Chunks are padded to an even size.
at = end.saturating_add(end & 1);
}
out
}
/// Read bounded cue/adtl and XMP metadata without interpreting application timeline policy.
pub fn read(bytes: &[u8]) -> Markers {
if !is_wav(bytes) {
return Markers::default();
}
let mut limited = false;
let chunks = chunks(bytes, &mut limited);
let rate = chunks.iter().find(|(id, _)| id == b"fmt ").and_then(|(_, b)| u32_at(b, 4)).map_or(0.0, f64::from);
let cue = cue_markers(&chunks, rate, &mut limited);
let xmp = chunks.iter().find(|(id, _)| id == b"_PMX").map_or_else(Vec::new, |(_, xmp)| xmp_markers(xmp, rate));
Markers { cue, xmp, limited }
}
/// Cue points named and sized by the `adtl` list.
fn cue_markers(chunks: &[([u8; 4], &[u8])], rate: f64, limited: &mut bool) -> Vec<Marker> {
let Some((_, cue)) = chunks.iter().find(|(id, _)| id == b"cue ") else { return Vec::new() };
let count = (u32_at(cue, 0).unwrap_or(0) as usize).min(MAX_MARKERS);
// The cue points first: `(id, offset in samples)`.
let mut points = Vec::new();
for i in 0..count {
let Some(point) = cue.get(4usize.saturating_add(i.saturating_mul(24))..) else { break };
let (Some(id), Some(offset)) = (u32_at(point, 0), u32_at(point, 20)) else { break };
points.push((id, offset));
}
// Only the names and lengths of those ids are kept, so a hostile `adtl` list cannot grow them past the cue count,
// and each record is a lookup, not a scan.
let wanted: HashSet<u32> = points.iter().map(|(id, _)| *id).collect();
let mut names: HashMap<u32, String> = HashMap::new();
let mut lengths: HashMap<u32, u32> = HashMap::new();
let mut name_bytes = 0usize;
let mut records = 0usize;
'lists: for (_, list) in chunks.iter().filter(|(id, b)| id == b"LIST" && b.get(0..4) == Some(b"adtl")) {
let mut at = 4usize;
while let Some(head) = list.get(at..at.saturating_add(8)) {
let (Ok(id), Some(size)) = (<[u8; 4]>::try_from(&head[..4]), u32_at(head, 4)) else { break };
let start = at.saturating_add(8);
let Some(end) = start.checked_add(size as usize) else { break };
let Some(body) = list.get(start..end) else { break };
records += 1;
if records > MAX_ADTL_RECORDS {
break 'lists;
}
if let Some(cue_id) = u32_at(body, 0).filter(|c| wanted.contains(c)) {
match &id {
// A `labl` is the name; a `note` only when there is none.
b"labl" | b"note" if id == *b"labl" || !names.contains_key(&cue_id) => {
let without_old = name_bytes.saturating_sub(names.get(&cue_id).map_or(0, String::len));
let text = label(body.get(4..).unwrap_or_default())
.filter(|text| text.len() <= MAX_LABEL_TOTAL_BYTES.saturating_sub(without_old))
.unwrap_or_else(|| {
*limited = true;
String::new()
});
name_bytes = without_old + text.len();
names.insert(cue_id, text);
}
// The first `ltxt` of a cue point gives its length.
b"ltxt" => {
lengths.entry(cue_id).or_insert_with(|| u32_at(body, 4).unwrap_or(0));
}
_ => {}
}
}
at = end.saturating_add(end & 1);
}
}
let mut emitted_bytes = 0usize;
points
.into_iter()
.map(|(id, offset)| {
let comment = match names.get(&id) {
Some(text) if text.len() <= MAX_LABEL_TOTAL_BYTES.saturating_sub(emitted_bytes) => {
emitted_bytes += text.len();
text.clone()
}
Some(_) => {
*limited = true;
String::new()
}
None => String::new(),
};
let length = lengths.get(&id).copied().unwrap_or(0);
Marker { start: f64::from(offset), duration: f64::from(length), rate, name: comment }
})
.collect()
}
/// The text of `name` in an XMP entry, as an attribute (`name="v"`) or an element
/// (`<name>v</name>`).
fn xmp_value(entry: &str, name: &str) -> Option<String> {
// Each entry starts just after `<rdf:li`. Scan only tag attributes, skipping
// quoted values so a label containing an attribute-like string cannot spoof it.
let bytes = entry.as_bytes();
let mut at = 0;
let mut in_tag = true;
while at < bytes.len() {
match bytes[at] {
b'<' => in_tag = true,
b'>' => in_tag = false,
b'\'' | b'"' if in_tag => {
let quote = bytes[at];
at += 1;
while at < bytes.len() && bytes[at] != quote {
at += 1;
}
}
_ if in_tag && (at == 0 || xml_space(bytes[at - 1])) && bytes[at..].starts_with(name.as_bytes()) => {
let mut value = at + name.len();
while value < bytes.len() && xml_space(bytes[value]) {
value += 1;
}
if bytes.get(value) == Some(&b'=') {
value += 1;
while value < bytes.len() && xml_space(bytes[value]) {
value += 1;
}
if let Some(&quote @ (b'\'' | b'"')) = bytes.get(value) {
let start = value + 1;
let end = bytes.get(start..)?.iter().position(|&b| b == quote)? + start;
return Some(unescape(entry.get(start..end)?));
}
}
}
_ => {}
}
at += 1;
}
let open = format!("<{name}>");
let i = entry.find(&open)?;
let rest = entry.get(i + open.len()..)?;
Some(unescape(rest.get(..rest.find('<')?)?))
}
fn xml_space(b: u8) -> bool {
matches!(b, b' ' | b'\t' | b'\r' | b'\n')
}
fn unescape(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut rest = s;
while let Some(i) = rest.find('&') {
out.push_str(rest.get(..i).unwrap_or_default());
let tail = rest.get(i..).unwrap_or_default();
let Some(semi) = tail.as_bytes().iter().take(11).position(|b| *b == b';') else {
out.push('&');
rest = tail.get(1..).unwrap_or_default();
continue;
};
let ent = tail.get(1..semi).unwrap_or_default();
match ent {
"amp" => out.push('&'),
"lt" => out.push('<'),
"gt" => out.push('>'),
"quot" => out.push('"'),
"apos" => out.push('\''),
_ => {
let code = ent
.strip_prefix("#x")
.and_then(|h| u32::from_str_radix(h, 16).ok())
.or_else(|| ent.strip_prefix('#').and_then(|d| d.parse().ok()));
match code.and_then(char::from_u32) {
Some(c) => out.push(c),
None => out.push_str(tail.get(..=semi).unwrap_or_default()),
}
}
}
rest = tail.get(semi + 1..).unwrap_or_default();
}
out.push_str(rest);
out.trim().to_string()
}
/// A frame rate as XMP writes it: `f44100`, or `f30000s1001` (rate over scale).
fn xmp_rate(text: &str) -> Option<f64> {
let t = text.trim().strip_prefix('f')?;
let (num, den) = t.split_once('s').unwrap_or((t, "1"));
let (n, d): (f64, f64) = (num.parse().ok()?, den.parse().ok()?);
(n.is_finite() && d.is_finite() && n > 0.0 && d > 0.0 && (n / d).is_finite() && n / d > 0.0).then_some(n / d)
}
/// Adobe XMP FrameCount: a count optionally followed by `fRATE[sSCALE]`.
/// Bare counts retain the caller's legacy track/file-rate interpretation.
fn xmp_count(text: &str, default_rate: f64) -> Option<(f64, f64)> {
let text = text.trim();
let (count, rate) = match text.find('f') {
Some(at) => (text.get(..at)?, xmp_rate(text.get(at..)?)?),
None => (text, default_rate),
};
let count: f64 = count.parse().ok()?;
(count.is_finite() && rate.is_finite() && rate > 0.0).then_some((count, rate))
}
/// The markers of an XMP packet's `xmpDM:markers` entries.
fn xmp_markers(packet: &[u8], file_rate: f64) -> Vec<Marker> {
let text = String::from_utf8_lossy(packet.get(..packet.len().min(MAX_XMP)).unwrap_or_default());
let mut out = Vec::new();
// The frame rate belongs to the track an entry follows.
let mut rate = file_rate;
for entry in text.split("<rdf:li").skip(1) {
if let Some(r) = xmp_value(entry, "xmpDM:frameRate").as_deref().and_then(xmp_rate) {
rate = r;
}
let Some((start, marker_rate)) = xmp_value(entry, "xmpDM:startTime").and_then(|v| xmp_count(&v, rate)) else { continue };
let length = xmp_value(entry, "xmpDM:duration")
.and_then(|v| xmp_count(&v, rate))
.map(|(count, duration_rate)| if duration_rate == marker_rate { count } else { count / duration_rate * marker_rate })
.unwrap_or(0.0);
if !length.is_finite() {
continue;
}
let comment = xmp_value(entry, "xmpDM:name").unwrap_or_default();
out.push(Marker { start, duration: length, rate: marker_rate, name: comment });
if out.len() >= MAX_MARKERS {
break;
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
struct TimedMarker {
time: f64,
duration: f64,
comment: String,
}
fn read(bytes: &[u8]) -> Vec<TimedMarker> {
let metadata = super::read(bytes);
let time = |frames: f64, rate: f64| if rate > 0.0 && frames.is_finite() { (frames / rate).max(0.0) } else { 0.0 };
let map = |m: Marker| TimedMarker { time: time(m.start, m.rate), duration: time(m.duration, m.rate), comment: m.name };
let mut out: Vec<_> = metadata.cue.into_iter().map(map).collect();
let mut seen: HashSet<_> = out.iter().map(|m| (m.comment.clone(), m.time.to_bits())).collect();
for m in metadata.xmp.into_iter().map(map) {
if seen.insert((m.comment.clone(), m.time.to_bits())) {
out.push(m);
}
}
out.truncate(MAX_MARKERS);
out.sort_by(|a, b| a.time.total_cmp(&b.time));
out
}
fn chunk(id: &[u8; 4], body: &[u8]) -> Vec<u8> {
let mut v = id.to_vec();
v.extend((body.len() as u32).to_le_bytes());
v.extend(body);
if body.len() % 2 == 1 {
v.push(0);
}
v
}
fn fmt(rate: u32) -> Vec<u8> {
let mut b = Vec::new();
b.extend(1u16.to_le_bytes()); // PCM
b.extend(1u16.to_le_bytes()); // mono
b.extend(rate.to_le_bytes());
b.extend((rate * 2).to_le_bytes());
b.extend(2u16.to_le_bytes());
b.extend(16u16.to_le_bytes());
chunk(b"fmt ", &b)
}
fn wav(chunks: &[Vec<u8>]) -> Vec<u8> {
let body: Vec<u8> = chunks.iter().flatten().copied().collect();
let mut v = b"RIFF".to_vec();
v.extend((body.len() as u32 + 4).to_le_bytes());
v.extend(b"WAVE");
v.extend(body);
v
}
fn cue(points: &[(u32, u32)]) -> Vec<u8> {
let mut b = (points.len() as u32).to_le_bytes().to_vec();
for (id, offset) in points {
b.extend(id.to_le_bytes());
b.extend(0u32.to_le_bytes());
b.extend(b"data");
b.extend(0u32.to_le_bytes());
b.extend(0u32.to_le_bytes());
b.extend(offset.to_le_bytes());
}
chunk(b"cue ", &b)
}
fn labl(id: &[u8; 4], cue_id: u32, text: &str) -> Vec<u8> {
let mut b = cue_id.to_le_bytes().to_vec();
b.extend(text.as_bytes());
b.push(0);
chunk(id, &b)
}
fn adtl(subs: &[Vec<u8>]) -> Vec<u8> {
let mut b = b"adtl".to_vec();
b.extend(subs.iter().flatten());
chunk(b"LIST", &b)
}
#[test]
fn cue_points_are_named_by_the_adtl_list_and_timed_in_samples() {
let file = wav(&[
fmt(44100),
chunk(b"data", &[0; 8]),
cue(&[(2, 88200), (1, 22050)]),
adtl(&[labl(b"labl", 1, "SEQ01_Prologue"), labl(b"labl", 2, "SEQ02_Vallee_P1_Agriculture")]),
]);
let m = read(&file);
assert_eq!(m.len(), 2);
assert_eq!((m[0].comment.as_str(), m[0].time), ("SEQ01_Prologue", 0.5));
assert_eq!((m[1].comment.as_str(), m[1].time), ("SEQ02_Vallee_P1_Agriculture", 2.0));
}
#[test]
fn a_note_names_a_cue_only_when_it_has_no_label_and_ltxt_gives_a_length() {
let mut ltxt = 1u32.to_le_bytes().to_vec();
ltxt.extend(44100u32.to_le_bytes());
ltxt.extend([0; 16]);
let file = wav(&[
fmt(44100),
cue(&[(1, 0), (2, 44100)]),
adtl(&[labl(b"note", 1, "from note"), chunk(b"ltxt", &ltxt), labl(b"labl", 2, "named"), labl(b"note", 2, "ignored")]),
]);
let m = read(&file);
assert_eq!((m[0].comment.as_str(), m[0].duration), ("from note", 1.0));
assert_eq!(m[1].comment, "named");
}
const XMP_ELEMENTS: &str = r#"<x:xmpmeta><rdf:RDF><rdf:Description><xmpDM:Tracks><rdf:Bag><rdf:li rdf:parseType="Resource"><xmpDM:trackName>Markers</xmpDM:trackName><xmpDM:trackType>Cue</xmpDM:trackType><xmpDM:frameRate>f44100</xmpDM:frameRate><xmpDM:markers><rdf:Seq><rdf:li rdf:parseType="Resource"><xmpDM:startTime>44100</xmpDM:startTime><xmpDM:duration>0</xmpDM:duration><xmpDM:name>One &amp; Two</xmpDM:name></rdf:li><rdf:li rdf:parseType="Resource"><xmpDM:startTime>132300</xmpDM:startTime><xmpDM:duration>22050</xmpDM:duration><xmpDM:name>Three</xmpDM:name></rdf:li></rdf:Seq></xmpDM:markers></rdf:li></rdf:Bag></xmpDM:Tracks></rdf:Description></rdf:RDF></x:xmpmeta>"#;
#[test]
fn xmp_markers_are_read_as_elements_in_frames_of_the_track_rate() {
let file = wav(&[fmt(48000), chunk(b"_PMX", XMP_ELEMENTS.as_bytes())]);
let m = read(&file);
assert_eq!(m.len(), 2);
// 44100 frames at f44100 is one second, whatever the file's own rate.
assert_eq!((m[0].comment.as_str(), m[0].time), ("One & Two", 1.0));
assert_eq!((m[1].comment.as_str(), m[1].time, m[1].duration), ("Three", 3.0, 0.5));
}
#[test]
fn xmp_markers_are_read_as_attributes_and_use_the_file_rate_without_a_track_rate() {
let xmp = r#"<rdf:Seq><rdf:li xmpDM:startTime="24000" xmpDM:duration="0" xmpDM:name="Attr &#233;"/></rdf:Seq>"#;
let m = read(&wav(&[fmt(48000), chunk(b"_PMX", xmp.as_bytes())]));
assert_eq!((m[0].comment.as_str(), m[0].time), ("Attr é", 0.5));
}
#[test]
fn a_marker_in_both_encodings_is_listed_once() {
let xmp = r#"<rdf:li xmpDM:startTime="44100" xmpDM:name="Same"/><rdf:li xmpDM:startTime="88200" xmpDM:name="Only XMP"/>"#;
let file = wav(&[fmt(44100), cue(&[(1, 44100)]), adtl(&[labl(b"labl", 1, "Same")]), chunk(b"_PMX", xmp.as_bytes())]);
let names: Vec<String> = read(&file).into_iter().map(|m| m.comment).collect();
assert_eq!(names, ["Same", "Only XMP"]);
}
#[test]
fn damaged_or_hostile_files_read_as_far_as_they_go_without_panicking() {
assert!(read(b"").is_empty());
assert!(read(b"RIFF").is_empty());
assert!(read(b"not a wav file at all").is_empty());
let good = wav(&[fmt(44100), cue(&[(1, 44100)]), adtl(&[labl(b"labl", 1, "Kept")])]);
// Every truncation is safe.
for n in 0..good.len() {
let _ = read(&good[..n]);
}
// A cue count far beyond the chunk, and chunk sizes past the end of the file.
let mut huge = 0xFFFF_FFFFu32.to_le_bytes().to_vec();
huge.extend([0; 24]);
let m = read(&wav(&[fmt(44100), chunk(b"cue ", &huge)]));
assert!(m.len() <= 1);
let mut lying = good.clone();
lying.extend(b"LIST\xff\xff\xff\xffadtl");
assert_eq!(read(&lying).len(), 1);
// No sample rate: times are zero, not a division by zero.
let m = read(&wav(&[cue(&[(1, 44100)])]));
assert_eq!(m[0].time, 0.0);
}
/// A chunk whose size field says `field` (an RF64 file's `0xFFFFFFFF`) while its body is `body`.
fn chunk_with_size_field(id: &[u8; 4], field: u32, body: &[u8]) -> Vec<u8> {
let mut v = id.to_vec();
v.extend(field.to_le_bytes());
v.extend(body);
if body.len() % 2 == 1 {
v.push(0);
}
v
}
/// An RF64 `ds64` chunk: the `data` chunk's size and a table of other chunks' sizes.
fn ds64_chunk(data_size: u64, table: &[([u8; 4], u64)]) -> Vec<u8> {
let mut b = 0u64.to_le_bytes().to_vec(); // riff size (not used)
b.extend(data_size.to_le_bytes());
b.extend(0u64.to_le_bytes()); // sample count
b.extend((table.len() as u32).to_le_bytes());
for (id, size) in table {
b.extend(id);
b.extend(size.to_le_bytes());
}
chunk(b"ds64", &b)
}
fn rf64(chunks: &[Vec<u8>]) -> Vec<u8> {
let body: Vec<u8> = chunks.iter().flatten().copied().collect();
let mut v = b"RF64".to_vec();
v.extend(u32::MAX.to_le_bytes());
v.extend(b"WAVE");
v.extend(body);
v
}
#[test]
fn rf64_markers_after_a_data_chunk_of_sentinel_size_are_read() {
let audio = [0u8; 8];
let file = rf64(&[
ds64_chunk(audio.len() as u64, &[]),
fmt(44100),
chunk_with_size_field(b"data", u32::MAX, &audio),
cue(&[(1, 44100)]),
adtl(&[labl(b"labl", 1, "After the audio")]),
]);
let m = read(&file);
assert_eq!(m.len(), 1);
assert_eq!((m[0].comment.as_str(), m[0].time), ("After the audio", 1.0));
}
#[test]
fn rf64_sizes_of_other_chunks_come_from_the_ds64_table() {
let junk = [7u8; 6];
let tail = || [cue(&[(1, 8000)]), adtl(&[labl(b"labl", 1, "Found")])].concat();
let with_entry = rf64(&[ds64_chunk(0, &[(*b"junk", junk.len() as u64)]), fmt(8000), chunk_with_size_field(b"junk", u32::MAX, &junk), tail()]);
assert_eq!(read(&with_entry)[0].comment, "Found");
// Without the table entry the sentinel is all there is, and what follows is not reached.
let without = rf64(&[ds64_chunk(0, &[]), fmt(8000), chunk_with_size_field(b"junk", u32::MAX, &junk), tail()]);
assert!(read(&without).is_empty());
}
#[test]
fn hostile_rf64_size_tables_read_as_far_as_they_go_without_panicking() {
// sizes that overflow when added to an offset
for data in [u64::MAX, u64::MAX - 7, 1 << 63] {
let file =
rf64(&[ds64_chunk(data, &[(*b"junk", u64::MAX)]), fmt(8000), chunk_with_size_field(b"data", u32::MAX, &[0; 4]), cue(&[(1, 0)])]);
assert!(read(&file).is_empty());
}
// a table that claims far more entries than its chunk holds
let mut liar = ds64_chunk(0, &[]);
liar[8 + 24..8 + 28].copy_from_slice(&u32::MAX.to_le_bytes());
let _ = read(&rf64(&[liar, fmt(8000), cue(&[(1, 0)])]));
// every truncation of a good RF64 file is safe
let good = rf64(&[
ds64_chunk(4, &[]),
fmt(44100),
chunk_with_size_field(b"data", u32::MAX, &[0; 4]),
cue(&[(1, 44100)]),
adtl(&[labl(b"labl", 1, "Kept")]),
]);
assert_eq!(read(&good).len(), 1);
for n in 0..good.len() {
let _ = read(&good[..n]);
}
}
#[test]
fn a_huge_adtl_list_is_read_in_linear_time_and_only_for_the_cue_points_that_exist() {
// Records for ids that have no cue point are skipped, not stored, and the number looked at is capped.
let junk = |n: u32| (1000..1000 + n).map(|id| labl(b"labl", id, "n")).collect::<Vec<_>>();
let started = std::time::Instant::now();
// the real label comes first: read
let mut subs = vec![labl(b"labl", 1, "Real")];
subs.extend(junk(100_000));
let m = read(&wav(&[fmt(44100), cue(&[(1, 44100)]), adtl(&subs)]));
assert_eq!(m[0].comment, "Real");
// it comes after more records than are looked at: not read, and the marker is still there
let mut subs = junk(MAX_ADTL_RECORDS as u32 + 10);
subs.push(labl(b"labl", 1, "Too late"));
let m = read(&wav(&[fmt(44100), cue(&[(1, 44100)]), adtl(&subs)]));
assert_eq!((m.len(), m[0].comment.as_str()), (1, ""));
assert!(started.elapsed() < std::time::Duration::from_secs(5), "took {:?}", started.elapsed());
}
#[test]
fn metadata_keeps_original_coordinates_and_separate_sources() {
let xmp = br#"<rdf:li xmpDM:startTime="1" xmpDM:duration="2" xmpDM:name="Same"/>"#;
let file = wav(&[fmt(44100), cue(&[(1, 1)]), adtl(&[labl(b"labl", 1, "Same")]), chunk(b"_PMX", xmp)]);
let data = super::read(&file);
assert_eq!(data.cue[0].start, 1.0);
assert_eq!(data.cue[0].rate, 44100.0);
assert_eq!(data.xmp[0].start, 1.0);
assert_eq!(data.xmp[0].duration, 2.0);
assert_eq!(data.cue.len(), 1);
assert_eq!(data.xmp.len(), 1);
}
#[test]
fn repeated_unterminated_entities_preserve_text_with_bounded_lookahead() {
let text = "&".repeat(100_000);
assert_eq!(unescape(&text), text);
assert_eq!(unescape("&#0000065; &#00000065; &amp; &unknown;"), "A A & &unknown;");
}
#[test]
fn repeated_cue_ids_cannot_multiply_label_storage() {
let text = "x".repeat(MAX_LABEL_BYTES);
let points: Vec<_> = (0..MAX_MARKERS).map(|i| (1, i as u32)).collect();
let file = wav(&[fmt(44100), cue(&points), adtl(&[labl(b"labl", 1, &text)])]);
let metadata = super::read(&file);
assert_eq!(metadata.cue.len(), MAX_MARKERS);
assert_eq!(metadata.cue.iter().map(|m| m.name.len()).sum::<usize>(), MAX_LABEL_TOTAL_BYTES);
assert!(metadata.limited);
assert_eq!(metadata.cue[0].name, text);
assert!(metadata.cue[MAX_MARKERS - 1].name.is_empty());
assert_eq!(metadata.cue[MAX_MARKERS - 1].start, (MAX_MARKERS - 1) as f64);
}
#[test]
fn cue_name_map_and_single_label_are_bounded() {
let count = MAX_LABEL_TOTAL_BYTES / MAX_LABEL_BYTES + 2;
let text = "x".repeat(MAX_LABEL_BYTES);
let points: Vec<_> = (0..count as u32).map(|i| (i, i)).collect();
let labels: Vec<_> = (0..count as u32).map(|i| labl(b"labl", i, &text)).collect();
let metadata = super::read(&wav(&[fmt(44100), cue(&points), adtl(&labels)]));
assert!(metadata.limited);
assert_eq!(metadata.cue.iter().map(|m| m.name.len()).sum::<usize>(), MAX_LABEL_TOTAL_BYTES);
let oversized = "x".repeat(MAX_LABEL_BYTES + 1);
let metadata = super::read(&wav(&[fmt(44100), cue(&[(1, 3)]), adtl(&[labl(b"labl", 1, &oversized)])]));
assert!(metadata.limited);
assert_eq!(metadata.cue[0].start, 3.0);
assert!(metadata.cue[0].name.is_empty());
}
#[test]
fn empty_chunk_inventory_is_bounded() {
let empty = chunk(b"junk", &[]);
let many = empty.repeat(MAX_CHUNKS + 1);
let metadata = super::read(&wav(&[many, fmt(44100), cue(&[(1, 1)])]));
assert!(metadata.limited);
assert!(metadata.cue.is_empty());
}
#[test]
fn entity_lookahead_keeps_previous_semicolon_boundary() {
assert_eq!(unescape("&#00000065;"), "A");
assert_eq!(unescape("&#000000065;"), "&#000000065;");
}
#[test]
fn xmp_quotes_spacing_and_attribute_boundaries() {
let packet = br#"<rdf:li xmpDM:startTime = '24000' xmpDM:duration =
'12000' xmpDM:name='single &amp; quoted'/>"#;
let markers = xmp_markers(packet, 48000.0);
assert_eq!(markers.len(), 1);
assert_eq!((markers[0].start, markers[0].duration, markers[0].rate), (24000.0, 12000.0, 48000.0));
assert_eq!(markers[0].name, "single & quoted");
let spoof = r#" label=" xmpDM:startTime='7'" otherxmpDM:startTime='8' xmpDM:startTime = '9'/>"#;
assert_eq!(xmp_value(spoof, "xmpDM:startTime").as_deref(), Some("9"));
assert!(xmp_value(" xmpDM:startTime='unterminated", "xmpDM:startTime").is_none());
}
#[test]
fn xmp_frame_counts_preserve_explicit_and_mixed_rates() {
let packet = br#"<rdf:li xmpDM:frameRate='f48000'><rdf:li xmpDM:startTime='15f24' xmpDM:duration='15f30000s1001'/><rdf:li xmpDM:startTime='48000' xmpDM:duration='12f24'/><rdf:li xmpDM:startTime='15f24' xmpDM:duration='24000'/></rdf:li>"#;
let markers = xmp_markers(packet, 44100.0);
assert_eq!(markers.len(), 3);
assert_eq!((markers[0].start, markers[0].rate), (15.0, 24.0));
assert!((markers[0].duration / markers[0].rate - 0.5005).abs() < 1e-12);
assert_eq!((markers[1].start, markers[1].duration, markers[1].rate), (48000.0, 24000.0, 48000.0));
assert_eq!((markers[2].start, markers[2].duration, markers[2].rate), (15.0, 12.0, 24.0));
for invalid in ["15f0", "15f24s0", "15fNaN", "15finf", "NaN", "1e999", "15f24junk"] {
assert!(xmp_count(invalid, 48000.0).is_none(), "{invalid}");
}
}
}
+223
View File
@@ -0,0 +1,223 @@
//! Range-backed PCM WAVE reader and canonical 16-bit writer.
//!
//! Reads chunk headers and only the requested sample range; it never loads a whole file.
use crate::{AudioError, Result};
use craft_media::ByteSource;
/// Whether a prefix identifies a RIFF or RF64 WAVE file.
pub fn sniff(b: &[u8]) -> bool {
b.len() >= 12 && (&b[0..4] == b"RIFF" || &b[0..4] == b"RF64") && &b[8..12] == b"WAVE"
}
const MAX_CHUNKS: usize = 1 << 16;
const IXML_MAX: u64 = 1 << 20;
/// Maximum sample data returned by a single range read.
pub const MAX_RANGE_BYTES: u64 = 4 << 20;
fn le16(b: &[u8], o: usize) -> u16 {
b.get(o..o.saturating_add(2)).and_then(|s| s.try_into().ok()).map_or(0, u16::from_le_bytes)
}
fn le32(b: &[u8], o: usize) -> u32 {
b.get(o..o.saturating_add(4)).and_then(|s| s.try_into().ok()).map_or(0, u32::from_le_bytes)
}
fn read_range(source: &impl ByteSource, off: u64, len: usize) -> Result<Vec<u8>> {
let mut bytes = vec![0; len];
source.read_at(off, &mut bytes)?;
Ok(bytes)
}
/// A PCM WAVE file held by an arbitrary random-access source.
pub struct WavRange<S> {
source: S,
/// Number of complete sample frames.
pub frames: u64,
/// Channel count.
pub channels: u16,
/// Sample rate in Hz.
pub rate: u32,
/// Container bits per sample.
pub bits: u16,
/// Whether samples are floating point.
pub float: bool,
/// BWF time reference in samples, from bext or iXML.
pub time_reference: Option<u64>,
data_off: u64,
}
impl<S: ByteSource> WavRange<S> {
/// Open with caller-selected bounds on channel count and sample rate.
pub fn open(source: S, max_channels: u32, max_rate: u32) -> Result<Self> {
let len = source.len();
let mut pos: u64 = 12;
let mut fmt = None;
let mut data = None;
let mut bext_tr = None;
let mut ixml_tr = None;
let mut ds64_data = None;
for _ in 0..MAX_CHUNKS {
let Some(body) = pos.checked_add(8).filter(|&b| b <= len) else { break };
let mut hdr = [0u8; 8];
source.read_at(pos, &mut hdr)?;
let id = &hdr[0..4];
let mut clen = u64::from(le32(&hdr, 4));
let rest = len - body;
if id == b"data" {
if clen == 0xFFFF_FFFF {
clen = ds64_data.unwrap_or(rest);
}
clen = clen.min(rest);
}
if clen > rest {
break;
}
match id {
b"fmt " if clen >= 16 => {
let b = read_range(&source, body, clen.min(40) as usize)?;
let mut tag = le16(&b, 0);
if tag == 0xFFFE && clen >= 40 {
tag = le16(&b, 24);
}
fmt = Some((tag, le16(&b, 2), le32(&b, 4), le16(&b, 14)));
}
b"ds64" if clen >= 16 => {
let b = read_range(&source, body + 8, 8)?;
ds64_data = Some(u64::from(le32(&b, 0)) | u64::from(le32(&b, 4)) << 32);
}
b"data" => data = Some((body, clen)),
b"bext" if clen >= 346 => {
let b = read_range(&source, body + 338, 8)?;
bext_tr = Some(u64::from(le32(&b, 0)) | u64::from(le32(&b, 4)) << 32);
}
b"iXML" => {
let b = read_range(&source, body, clen.min(IXML_MAX) as usize)?;
ixml_tr = ixml_time_reference(&b);
}
_ => {}
}
pos = body.saturating_add(clen).saturating_add(clen & 1);
}
let (tag, channels, rate, bits) = fmt.ok_or_else(|| AudioError::Malformed("missing fmt chunk".into()))?;
let (data_off, dlen) = data.ok_or_else(|| AudioError::Malformed("missing data chunk".into()))?;
let float = tag == 3;
if !(tag == 1 || float)
|| channels == 0
|| rate == 0
|| u32::from(channels) > max_channels
|| rate > max_rate
|| !matches!(bits, 8 | 16 | 24 | 32 | 64)
{
return Err(AudioError::Unsupported(format!("WAV format tag {tag}, {bits} bits")));
}
let frames = dlen / (u64::from(channels) * u64::from(bits / 8));
Ok(Self { source, frames, channels, rate, bits, float, time_reference: bext_tr.or(ixml_tr), data_off })
}
/// Bytes in one whole frame.
pub fn frame_bytes(&self) -> usize {
usize::from(self.channels) * usize::from(self.bits / 8)
}
/// Read the inclusive range of frames, capped to prevent unbounded allocation.
pub fn read_frames(&self, first: u64, last: u64) -> Result<Vec<u8>> {
if first > last || last >= self.frames {
return Err(AudioError::Malformed("WAV frame range outside data".into()));
}
let n = (last - first + 1).saturating_mul(self.frame_bytes() as u64);
if n > MAX_RANGE_BYTES {
return Err(AudioError::TooLarge("WAV frame range".into()));
}
let off = self.data_off.saturating_add(first.saturating_mul(self.frame_bytes() as u64));
read_range(&self.source, off, n as usize)
}
/// Decode a channel from a buffer returned by `read_frames`.
pub fn sample(&self, block: &[u8], frame: usize, ch: usize) -> f32 {
if ch >= usize::from(self.channels) {
return 0.0;
}
let bps = usize::from(self.bits / 8);
let Some(o) = frame.checked_mul(usize::from(self.channels)).and_then(|v| v.checked_add(ch)).and_then(|v| v.checked_mul(bps)) else {
return 0.0;
};
let Some(b) = block.get(o..o.saturating_add(bps)) else { return 0.0 };
match (self.bits, self.float, b) {
(8, _, &[a]) => (a as f32 - 128.0) / 128.0,
(16, _, &[a, b]) => i16::from_le_bytes([a, b]) as f32 / 32768.0,
(24, _, &[a, b, c]) => (i32::from_le_bytes([0, a, b, c]) >> 8) as f32 / 8_388_608.0,
(32, false, &[a, b, c, d]) => i32::from_le_bytes([a, b, c, d]) as f32 / 2_147_483_648.0,
(32, true, &[a, b, c, d]) => f32::from_le_bytes([a, b, c, d]),
(64, true, b) => f64::from_le_bytes(b.try_into().unwrap_or([0; 8])) as f32,
_ => 0.0,
}
}
}
fn ixml_time_reference(x: &[u8]) -> Option<u64> {
let text = String::from_utf8_lossy(x);
let field = |tag: &str| -> Option<u64> {
let open = format!("<{tag}>");
let a = text.find(&open)? + open.len();
let b = a + text[a..].find('<')?;
text[a..b].trim().parse().ok()
};
let low = field("BWF_TIME_REFERENCE_LOW")?;
Some(low | field("BWF_TIME_REFERENCE_HIGH").unwrap_or(0) << 32)
}
/// Write interleaved samples as 16-bit PCM WAVE, using symmetric 32767 scaling.
#[cfg(feature = "pcm-encode")]
pub fn write_wav16(samples: &[f32], channels: u16, rate: u32) -> Vec<u8> {
let data_len = samples.len().saturating_mul(2);
let mut v = crate::pcm_wav_header(u32::try_from(data_len).unwrap_or(u32::MAX), channels, rate, 16);
v.reserve(data_len);
for s in samples {
v.extend_from_slice(&((s.clamp(-1.0, 1.0) * 32767.0).round() as i16).to_le_bytes());
}
v
}
#[cfg(all(test, feature = "pcm-encode"))]
mod tests {
use super::*;
use std::sync::{Arc, Mutex};
struct Source {
bytes: Vec<u8>,
reads: Arc<Mutex<Vec<(u64, usize)>>>,
}
impl ByteSource for Source {
fn len(&self) -> u64 {
self.bytes.len() as u64
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result<()> {
self.reads.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push((offset, buf.len()));
self.bytes.read_at(offset, buf)
}
}
#[test]
fn bounded_reads_and_pcm_values() {
let bytes = write_wav16(&[-1.0, -0.5, 0.0, 0.5, 1.0], 1, 48_000);
let reads = Arc::new(Mutex::new(Vec::new()));
let wav = WavRange::open(Source { bytes, reads: reads.clone() }, 64, 384_000).unwrap();
assert_eq!((wav.frames, wav.channels, wav.rate, wav.bits), (5, 1, 48_000, 16));
assert!(reads.lock().unwrap().iter().all(|(_, n)| *n <= 40));
let data = wav.read_frames(1, 3).unwrap();
assert_eq!(wav.sample(&data, 0, 0), -0.5);
assert_eq!(wav.sample(&data, 2, 0), 0.5);
assert!(wav.read_frames(0, 5).is_err());
assert!(wav.read_frames(3, 1).is_err());
assert_eq!(reads.lock().unwrap().last(), Some(&(46, 6)));
}
#[test]
fn ixml_fallback_accepts_flat_fields() {
let base = write_wav16(&[0.0], 1, 48_000);
let mut bytes = base[..36].to_vec();
let xml = b"<BWF_TIME_REFERENCE_LOW>5</BWF_TIME_REFERENCE_LOW><BWF_TIME_REFERENCE_HIGH>1</BWF_TIME_REFERENCE_HIGH>";
bytes.extend_from_slice(b"iXML");
bytes.extend_from_slice(&(xml.len() as u32).to_le_bytes());
bytes.extend_from_slice(xml);
if xml.len() & 1 != 0 {
bytes.push(0);
}
bytes.extend_from_slice(&base[36..]);
assert_eq!(WavRange::open(bytes.as_slice(), 64, 384_000).unwrap().time_reference, Some((1 << 32) + 5));
}
}
+17 -10
View File
@@ -12,7 +12,7 @@ mod pcm;
#[cfg(feature = "vorbis-encode")]
mod vorbis;
#[cfg(feature = "vorbis-encode")]
pub use vorbis::{OGG_BITRATES, encode as encode_ogg};
pub use vorbis::{OGG_BITRATES, OggVorbisOptions, encode as encode_ogg, encode_ogg_vorbis, quality_for_bitrate};
#[cfg(feature = "pcm-encode")]
mod raw;
#[cfg(feature = "pcm-encode")]
@@ -66,15 +66,22 @@ pub fn encode(buf: &AudioBuffer, opts: &EncodeOptions) -> Result<Vec<u8>> {
EncodeFormat::Aac | EncodeFormat::Alac | EncodeFormat::Mp3 | EncodeFormat::Ac3 | EncodeFormat::Eac3 => coded::encode(buf, opts),
#[cfg(feature = "vorbis-encode")]
EncodeFormat::Ogg => {
if opts.layout.is_some_and(|layout| !EncodeFormat::Ogg.supports_layout(&layout)) {
return Err(AudioError::Unsupported("Vorbis requires a mono or stereo speaker layout".into()));
let count = u16::try_from(buf.channels.len()).unwrap_or(u16::MAX);
let layout = opts.layout.unwrap_or_else(|| ChannelLayout::default_for(count));
if layout.channels() != count {
return Err(AudioError::Unsupported(format!("a {}-channel layout for {count} channels", layout.channels())));
}
let kbps = match opts.rate_control {
None => 192,
Some(crate::RateControl::Bitrate(bps)) => bps / 1000,
Some(crate::RateControl::Quality(_)) => return Err(AudioError::Unsupported("Ogg export takes a target bitrate".into())),
let map = vorbis::vorbis_map(&layout)?;
let quality = match opts.rate_control {
None => vorbis::quality_for_bitrate(192, buf.channels.len()),
Some(crate::RateControl::Bitrate(bps)) => {
vorbis::quality_for_bitrate((bps / 1000).clamp(OGG_BITRATES.0, OGG_BITRATES.1), buf.channels.len())
}
Some(crate::RateControl::Quality(q)) if q.is_nan() => return Err(AudioError::Encode("the Vorbis quality is NaN".into())),
Some(crate::RateControl::Quality(q)) => -1.0 + 11.0 * q.clamp(0.0, 1.0),
};
vorbis::encode(buf, kbps)
let planar: Vec<&[f32]> = map.iter().filter_map(|&i| buf.channels.get(i).map(Vec::as_slice)).collect();
vorbis::encode_planar(buf.sample_rate, &planar, &OggVorbisOptions::new(quality))
}
#[allow(unreachable_patterns)]
f => Err(AudioError::Unsupported(format!("writing {f:?}"))),
@@ -106,7 +113,7 @@ const fn extensions() -> ([&'static str; 13], usize) {
add!(cfg!(feature = "mp3-encode"), "mp3");
add!(cfg!(feature = "ac3-encode"), "ac3");
add!(cfg!(feature = "eac3-encode"), "eac3", "ec3");
add!(cfg!(all(feature = "vorbis-encode", not(target_arch = "wasm32"))), "ogg");
add!(cfg!(feature = "vorbis-encode"), "ogg");
(all, n)
}
@@ -148,7 +155,7 @@ pub fn encode_format_for(name: &str) -> Option<EncodeFormat> {
"ac3" => Some(EncodeFormat::Ac3),
#[cfg(feature = "eac3-encode")]
"eac3" | "ec3" => Some(EncodeFormat::Eac3),
#[cfg(all(feature = "vorbis-encode", not(target_arch = "wasm32")))]
#[cfg(feature = "vorbis-encode")]
"ogg" => Some(EncodeFormat::Ogg),
_ => None,
}
+217 -80
View File
@@ -1,83 +1,198 @@
//! Ogg Vorbis encoding (RFC 5334): the reference libvorbis encoder, built from its C sources by
//! `vorbis_rs`. Stereo or mono, variable bitrate at the requested kbit/s (64–500), with a
//! fallback to a quality preset when the bitrate is not supported for the rate and channel count.
//! Ogg Vorbis files (Vorbis I §A, RFC 5334) from the pure-Rust [`crate::vorbis::encoder`] and
//! craft-media's Ogg page writer: 1 to 8 channels in Vorbis channel order, a quality from -1 to
//! 10 (variable bit rate), and the tags in the comment header. Builds for every target.
use crate::{AudioBuffer, AudioError, Result};
use craft_media::ogg::{PageWriter, vorbis_comment};
/// The bitrates an Ogg Vorbis export is asked for, in kbit/s.
use crate::vorbis::encoder::{DEFAULT_QUALITY, Encoder, EncoderConfig, Error as VorbisError};
use crate::{AudioBuffer, AudioError, ChannelLayout, Result};
/// The bit rates [`encode_ogg`](crate::encode_ogg) takes, in kbit/s. Vorbis is variable: the bit rate picks a
/// quality ([`quality_for_bitrate`]), and the file comes out near it for typical music.
pub const OGG_BITRATES: (u32, u32) = (64, 500);
/// The block size the encoder processes at a time: small enough that memory stays bounded
/// regardless of file length.
#[cfg(not(target_arch = "wasm32"))]
const BLOCK: usize = 4096;
/// The serial number of the stream when the options do not set one (`LCRF`): one stream per file,
/// so a constant keeps files byte-for-byte reproducible.
const SERIAL: u32 = 0x4C43_5246;
#[cfg(not(target_arch = "wasm32"))]
fn encode_error(e: vorbis_rs::VorbisError) -> AudioError {
AudioError::Encode(e.to_string())
/// Samples per channel fed to the encoder at a time.
const CHUNK: usize = 4096;
/// What [`encode_ogg_vorbis`] writes besides the audio.
///
/// `#[non_exhaustive]`: build it with `OggVorbisOptions::default()` or
/// [`OggVorbisOptions::new`] and the `with_*` methods.
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq)]
pub struct OggVorbisOptions {
/// -1 (smallest) to 10 (best); clamped, and NaN or infinite reads as
/// [`DEFAULT_QUALITY`] (6).
pub quality: f32,
/// The comment header's vendor string.
pub vendor: String,
/// `NAME=value` comments (Vorbis I §5), in order. A name that is empty, or holds `=` or a byte
/// outside 0x20–0x7D, is left out.
pub tags: Vec<(String, String)>,
/// The logical stream's serial number.
pub serial: u32,
}
/// Encode `buf` as an Ogg Vorbis stream at about `kbps` (variable, as Vorbis is): stereo, or mono
/// for a mono buffer. A bitrate the encoder has no setup for at that rate and channel count falls
/// back to a quality that suits it. `Err` says why there is nothing.
#[cfg(not(target_arch = "wasm32"))]
pub fn encode(buf: &AudioBuffer, kbps: u32) -> Result<Vec<u8>> {
use std::num::{NonZeroU8, NonZeroU32};
let channels = buf.channels.len();
if !(1..=2).contains(&channels) {
return Err(AudioError::Unsupported("Ogg Vorbis export requires mono or stereo audio".into()));
impl Default for OggVorbisOptions {
fn default() -> Self {
OggVorbisOptions { quality: DEFAULT_QUALITY, vendor: "craft-audio".into(), tags: Vec::new(), serial: SERIAL }
}
if buf.channels.iter().any(|c| c.len() != buf.frames() || c.iter().any(|s| !s.is_finite())) {
}
impl OggVorbisOptions {
/// The defaults at `quality`.
pub fn new(quality: f32) -> Self {
OggVorbisOptions { quality, ..Self::default() }
}
/// Sets [`quality`](Self::quality).
pub fn with_quality(mut self, quality: f32) -> Self {
self.quality = quality;
self
}
/// Sets [`vendor`](Self::vendor).
pub fn with_vendor(mut self, vendor: impl Into<String>) -> Self {
self.vendor = vendor.into();
self
}
/// Adds a comment ([`tags`](Self::tags)).
pub fn with_tag(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
self.tags.push((name.into(), value.into()));
self
}
/// Sets [`serial`](Self::serial).
pub fn with_serial(mut self, serial: u32) -> Self {
self.serial = serial;
self
}
}
fn encode_error(e: VorbisError) -> AudioError {
AudioError::Encode(format!("Vorbis: {e}"))
}
/// Encode `buf` as an Ogg Vorbis file with `opts`. The channels are in Vorbis channel order
/// (Vorbis I §4.3.9: L C R for three, L C R Ls Rs LFE for 5.1, L C R Ls Rs Rl Rr LFE for 7.1).
/// The last granule position is the buffer's length, so a decoder returns exactly its frames.
///
/// ```
/// use craft_audio::{AudioBuffer, OggVorbisOptions, encode_ogg_vorbis};
///
/// let buf = AudioBuffer::new(48_000, 6, 4_800); // 0.1 s of 5.1 silence
/// let opts = OggVorbisOptions::new(4.0).with_tag("TITLE", "Silence");
/// let ogg = encode_ogg_vorbis(&buf, &opts).unwrap();
/// assert!(ogg.starts_with(b"OggS"));
/// ```
pub fn encode_ogg_vorbis(buf: &AudioBuffer, opts: &OggVorbisOptions) -> Result<Vec<u8>> {
let planar: Vec<&[f32]> = buf.channels.iter().map(Vec::as_slice).collect();
encode_planar(buf.sample_rate, &planar, opts)
}
/// [`encode_ogg_vorbis`] over channels already in Vorbis order.
pub(crate) fn encode_planar(sample_rate: u32, planar: &[&[f32]], opts: &OggVorbisOptions) -> Result<Vec<u8>> {
if !(1..=8).contains(&planar.len()) {
return Err(AudioError::Unsupported(format!("Ogg Vorbis holds 1 to 8 channels, not {}", planar.len())));
}
let frames = planar.first().map_or(0, |c| c.len());
if planar.iter().any(|c| c.len() != frames || c.iter().any(|s| !s.is_finite())) {
return Err(AudioError::Encode("Vorbis samples must be finite and channels equally long".into()));
}
let (Some(rate), Some(count)) = (NonZeroU32::new(buf.sample_rate), NonZeroU8::new(channels as u8)) else {
if sample_rate == 0 {
return Err(AudioError::Encode("there is no audio to encode".into()));
};
let target = NonZeroU32::new(kbps.clamp(OGG_BITRATES.0, OGG_BITRATES.1) * 1000).unwrap_or(NonZeroU32::MIN);
let encode = |strategy: vorbis_rs::VorbisBitrateManagementStrategy| -> Result<Vec<u8>> {
let mut encoder = vorbis_rs::VorbisEncoderBuilder::new_with_serial(rate, count, Vec::new(), 0x4C43_5246)
.bitrate_management_strategy(strategy)
.build()
.map_err(encode_error)?;
let frames = buf.frames();
let mut at = 0usize;
while at < frames {
let end = (at + BLOCK).min(frames);
let block: Vec<&[f32]> = buf.channels.iter().take(channels).map(|side| side.get(at..end).unwrap_or_default()).collect();
encoder.encode_audio_block(&block).map_err(encode_error)?;
at = end;
}
let mut enc = Encoder::new(EncoderConfig::new(sample_rate, planar.len(), opts.quality)).map_err(encode_error)?;
let mut pages = PageWriter::new(opts.serial);
let tags: Vec<(&str, &str)> = opts.tags.iter().map(|(k, v)| (k.as_str(), v.as_str())).collect();
let mut out = pages.header(&enc.identification());
out.extend(pages.header(&vorbis_comment(&opts.vendor, &tags)));
out.extend(pages.header(&enc.setup()));
// The first audio packet (which outputs nothing) gets a page of its own: a stream whose first
// audio page is also its last would leave decoders to choose between start and end trimming
// (Vorbis I §A.2), and ffmpeg then plays the padding.
let mut first = true;
let mut add = |packets: Vec<(Vec<u8>, i64)>, out: &mut Vec<u8>| {
for (packet, granule) in packets {
out.extend(pages.packet(&packet, granule));
if std::mem::take(&mut first) {
out.extend(pages.flush());
}
}
encoder.finish().map_err(encode_error)
};
let mut at = 0;
loop {
let end = (at + CHUNK).min(frames);
let chunk: Vec<&[f32]> = planar.iter().map(|c| c.get(at..end).unwrap_or_default()).collect();
add(enc.encode(&chunk).map_err(encode_error)?, &mut out);
at = end;
if at >= frames {
break;
}
}
add(enc.finish().map_err(encode_error)?, &mut out);
out.extend(pages.finish(None));
Ok(out)
}
// Convert Rust unwinds to an error. This does not catch native crashes or aborts.
let attempt = |strategy| {
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| encode(strategy)))
.unwrap_or_else(|_| Err(AudioError::Encode("the Vorbis encoder failed".into())))
};
match attempt(vorbis_rs::VorbisBitrateManagementStrategy::Vbr { target_bitrate: target }) {
Ok(bytes) => Ok(bytes),
Err(_) => attempt(vorbis_rs::VorbisBitrateManagementStrategy::QualityVbr { target_quality: 0.6 })
.map_err(|e| AudioError::Encode(format!("the Vorbis encoder refused the audio: {e}"))),
/// Nominal kbit/s per channel at each whole quality from -1 to 10: this encoder's rate on
/// music-like test signals at 48 kHz (harmonic chords with noise), averaged.
const KBPS_PER_CHANNEL: [f32; 12] = [18.0, 23.0, 31.0, 43.0, 58.0, 75.0, 89.0, 100.0, 111.0, 121.0, 130.0, 140.0];
/// The quality whose nominal bit rate is `kbps` kbit/s for `channels` channels (clamped to -1..=10).
/// Vorbis is variable: the bit rate of a file depends on the audio.
pub fn quality_for_bitrate(kbps: u32, channels: usize) -> f32 {
let per = kbps as f32 / channels.max(1) as f32;
let mut q = -1.0;
for pair in KBPS_PER_CHANNEL.windows(2) {
if let [lo, hi] = *pair
&& per <= hi
{
return q + ((per - lo) / (hi - lo)).clamp(0.0, 1.0);
}
q += 1.0;
}
10.0
}
/// Encode `buf` as an Ogg Vorbis stream near `kbps` kbit/s ([`OGG_BITRATES`], clamped): the
/// quality [`quality_for_bitrate`] gives, no tags. 1 to 8 channels in Vorbis channel order.
/// `Err` says why there is nothing.
pub fn encode(buf: &AudioBuffer, kbps: u32) -> Result<Vec<u8>> {
let kbps = kbps.clamp(OGG_BITRATES.0, OGG_BITRATES.1);
encode_ogg_vorbis(buf, &OggVorbisOptions::new(quality_for_bitrate(kbps, buf.channels.len())))
}
/// WAV-order speakers ([`crate::EncodeFormat::layout_masks`]) to Vorbis order, per channel count.
const VORBIS_ORDER: [&[usize]; 8] =
[&[0], &[0, 1], &[0, 2, 1], &[0, 1, 2, 3], &[0, 2, 1, 3, 4], &[0, 2, 1, 4, 5, 3], &[0, 2, 1, 5, 6, 4, 3], &[0, 2, 1, 6, 7, 4, 5, 3]];
/// For each Vorbis channel, the channel of audio in `layout` it comes from.
pub(crate) fn vorbis_map(layout: &ChannelLayout) -> Result<Vec<usize>> {
let n = layout.channels();
let mask = crate::EncodeFormat::Ogg.layout_masks(n).first().copied();
let order = usize::from(n).checked_sub(1).and_then(|i| VORBIS_ORDER.get(i)).copied();
match (mask, order) {
(Some(mask), Some(order)) => crate::layout::channel_map(mask, Some(order), layout)
.ok_or_else(|| AudioError::Unsupported(format!("Vorbis has no layout of {n} channels with these speakers"))),
_ => Err(AudioError::Unsupported(format!("Ogg Vorbis holds 1 to 8 channels, not {n}"))),
}
}
/// Ogg Vorbis encoding is unavailable on WASM; returns an unsupported-format error.
#[cfg(target_arch = "wasm32")]
pub fn encode(_buf: &AudioBuffer, _kbps: u32) -> Result<Vec<u8>> {
Err(AudioError::Unsupported("Ogg Vorbis files are exported by the desktop app".into()))
}
#[cfg(all(test, not(target_arch = "wasm32")))]
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used)]
use super::*;
use super::{OGG_BITRATES, OggVorbisOptions, encode, encode_ogg_vorbis, quality_for_bitrate, vorbis_map};
use crate::{AudioBuffer, AudioError, ChannelLayout};
#[test]
fn rejects_unsupported_or_malformed_channels_without_dropping_audio() {
for channels in [0, 3, 6] {
for channels in [0, 9] {
assert!(matches!(encode(&AudioBuffer::new(48_000, channels, 16), 192), Err(AudioError::Unsupported(_))));
}
let ragged = AudioBuffer { sample_rate: 48_000, channels: vec![vec![0.0; 16], vec![0.0; 15]] };
@@ -85,46 +200,68 @@ mod tests {
assert!(encode(&AudioBuffer::new(0, 1, 16), 192).is_err());
let invalid = AudioBuffer { sample_rate: 48_000, channels: vec![vec![f32::NAN]] };
assert!(encode(&invalid, 192).is_err());
let infinite = AudioBuffer { sample_rate: 48_000, channels: vec![vec![f32::INFINITY]] };
assert!(encode(&infinite, 192).is_err());
}
#[test]
fn generic_and_direct_encoders_write_mono_and_stereo_ogg() {
for channels in [1, 2] {
fn generic_and_direct_encoders_write_one_to_eight_channels() {
for channels in 1..=8 {
let audio = AudioBuffer::new(48_000, channels, 512);
let opts = crate::EncodeOptions::new(crate::EncodeFormat::Ogg);
assert!(opts.check(48_000, channels).is_ok());
assert_eq!(opts.bitrate(48_000, channels).unwrap(), None);
let direct = encode(&audio, 192).unwrap();
let generic = crate::encode(&audio, &opts).unwrap();
assert!(direct.starts_with(b"OggS"));
assert_eq!(direct, generic);
// silence: the channel order changes nothing, so the generic path writes the same file
let generic = crate::encode(&audio, &opts.with_bitrate(192_000)).unwrap();
assert_eq!(direct, generic, "{channels} channels");
#[cfg(feature = "decode")]
{
let (info, decoded) = crate::decode(&generic, Some("ogg")).unwrap();
assert_eq!(info.channels, channels);
assert_eq!(decoded.num_channels(), channels);
assert_eq!(decoded.frames(), audio.frames());
}
}
}
#[cfg(feature = "decode")]
#[test]
fn distinct_signals_survive_full_and_partial_blocks() {
for frames in [4096, 4097, 8193] {
let channels: Vec<Vec<f32>> = [440.0f32, 880.0]
.into_iter()
.map(|hz| (0..frames).map(|i| (std::f32::consts::TAU * hz * i as f32 / 48_000.0).sin() * 0.4).collect())
.collect();
let audio = AudioBuffer { sample_rate: 48_000, channels };
let bytes = encode(&audio, 192).unwrap();
let (_, decoded) = crate::decode(&bytes, Some("ogg")).unwrap();
assert_eq!(decoded.frames(), frames);
assert_eq!(decoded.channels.len(), 2);
for (expected, actual) in audio.channels.iter().zip(&decoded.channels) {
let mse = expected.iter().zip(actual).map(|(a, b)| (a - b).powi(2)).sum::<f32>() / frames as f32;
// Vorbis is lossy; this tolerates codec error but rejects silence or swapped channels.
assert!(mse.sqrt() < 0.08, "{frames} frames: RMS error {}", mse.sqrt());
}
fn the_quality_follows_the_bit_rate() {
assert_eq!(quality_for_bitrate(0, 2), -1.0);
assert_eq!(quality_for_bitrate(10_000, 2), 10.0);
assert!((quality_for_bitrate(200, 2) - 6.0).abs() < 1e-6);
assert!((quality_for_bitrate(600, 6) - 6.0).abs() < 1e-6);
let mut last = -2.0;
for kbps in OGG_BITRATES.0..=OGG_BITRATES.1 {
let q = quality_for_bitrate(kbps, 2);
assert!(q >= last && (-1.0..=10.0).contains(&q));
last = q;
}
assert!(quality_for_bitrate(192, 0).is_finite());
}
#[test]
fn layouts_map_to_vorbis_order() {
// 5.1 in WAV order (L R C LFE Ls Rs) is written L C R Ls Rs LFE
assert_eq!(vorbis_map(&ChannelLayout::SURROUND_5_1).unwrap(), [0, 2, 1, 4, 5, 3]);
assert_eq!(vorbis_map(&ChannelLayout::SURROUND_5_1_SIDE).unwrap(), [0, 2, 1, 4, 5, 3]);
assert_eq!(vorbis_map(&ChannelLayout::from_mask(0x63F)).unwrap(), [0, 2, 1, 6, 7, 4, 5, 3]);
assert_eq!(vorbis_map(&ChannelLayout::from_mask(0x7)).unwrap(), [0, 2, 1]);
assert_eq!(vorbis_map(&ChannelLayout::STEREO).unwrap(), [0, 1]);
assert!(vorbis_map(&ChannelLayout::from_mask(0xC)).is_err());
assert!(vorbis_map(&ChannelLayout::unordered(9)).is_err());
assert!(vorbis_map(&ChannelLayout::unordered(0)).is_err());
}
#[test]
fn options_build_and_tags_reach_the_comment_header() {
let opts = OggVorbisOptions::default().with_quality(2.0).with_vendor("v").with_tag("TITLE", "t").with_tag("bad=name", "x").with_serial(9);
assert_eq!((opts.quality, opts.vendor.as_str(), opts.serial, opts.tags.len()), (2.0, "v", 9, 2));
let ogg = encode_ogg_vorbis(&AudioBuffer::new(44_100, 1, 100), &opts).unwrap();
let file = craft_media::ogg::open(&ogg).unwrap();
let s = &file.streams[0];
assert_eq!((s.serial, s.saw_eos, s.headers.len()), (9, true, 3));
assert_eq!(s.headers[1], craft_media::ogg::vorbis_comment("v", &[("TITLE", "t")]));
assert_eq!(s.last_granule, Some(100));
}
}
+2 -2
View File
@@ -20,8 +20,8 @@ fn only_formats_the_encoder_writes_are_named() {
}
#[test]
fn ogg_names_follow_the_native_feature() {
let built = cfg!(all(feature = "vorbis-encode", not(target_arch = "wasm32")));
fn ogg_names_follow_the_feature() {
let built = cfg!(feature = "vorbis-encode");
assert_eq!(EncodeFormat::Ogg.is_built(), built);
assert_eq!(encode_format_for("ogg"), built.then_some(EncodeFormat::Ogg));
assert_eq!(craft_audio::ENCODE_EXTENSIONS.contains(&"ogg"), built);
+1 -1
View File
@@ -527,7 +527,7 @@ fn the_format_queries_agree_with_the_writer() {
// The most channels is the most: one more is refused, by the layouts of a format that has
// a fixed set and by the writer of one that has not.
let most: Vec<u16> = EncodeFormat::ALL.iter().map(|f| f.max_channels()).collect();
assert_eq!(most, [32_767, 32_767, 8, 8, 8, 2, 6, 8, 2]);
assert_eq!(most, [32_767, 32_767, 8, 8, 8, 2, 6, 8, 8]);
for &format in EncodeFormat::ALL {
let most = format.max_channels();
match format.fixed_layouts(most) {
+2
View File
@@ -20,3 +20,5 @@ mod opus;
mod opusenc;
mod routing_oracle;
mod track_decoder;
#[cfg(feature = "vorbis-encode")]
mod vorbis_encoder_oracle;
@@ -0,0 +1,131 @@
//! ffmpeg as an external oracle for the pure-Rust Vorbis encoder: both of its decoders (its own
//! and libvorbis) read every file exactly as long as the input, surround comes out on the right
//! speakers, and the tags reach ffprobe. Skipped when ffmpeg is not installed.
use std::path::Path;
use std::process::Command;
use craft_audio::{AudioBuffer, ChannelLayout, EncodeFormat, EncodeOptions, OggVorbisOptions, encode, encode_ogg_vorbis};
/// A different tone on every channel, so each output channel can be traced to its input.
fn tones(rate: u32, channels: usize, frames: usize) -> AudioBuffer {
let channels = (0..channels)
.map(|c| {
let f = 220.0 * (c + 1) as f32;
(0..frames).map(|i| (std::f32::consts::TAU * f * i as f32 / rate as f32).sin() * 0.3).collect()
})
.collect();
AudioBuffer { sample_rate: rate, channels }
}
/// Decodes `file` with ffmpeg's `decoder`; planar output in ffmpeg's (WAV) channel order.
fn ffmpeg_decode(ff: &Path, decoder: &str, file: &[u8], channels: usize, tag: &str) -> Vec<Vec<f32>> {
let path = crate::testkit::fixtures_dir("vorbis-encoder").join(format!("{tag}.ogg"));
std::fs::write(&path, file).unwrap();
let out = Command::new(ff)
.args(["-v", "error", "-xerror", "-c:a", decoder, "-i"])
.arg(&path)
.args(["-f", "f32le", "-c:a", "pcm_f32le", "-"])
.output()
.unwrap();
assert!(out.status.success() && out.stderr.is_empty(), "{tag} {decoder}: {}", String::from_utf8_lossy(&out.stderr));
let pcm: Vec<f32> = out.stdout.as_chunks::<4>().0.iter().map(|&b| f32::from_le_bytes(b)).collect();
(0..channels).map(|c| pcm.iter().skip(c).step_by(channels).copied().collect()).collect()
}
/// ffmpeg's decoders to check: its own `vorbis` always, and `libvorbis` when ffmpeg has it.
fn decoders(ff: &Path) -> Vec<&'static str> {
let list =
Command::new(ff).args(["-hide_banner", "-decoders"]).output().map(|o| String::from_utf8_lossy(&o.stdout).into_owned()).unwrap_or_default();
if list.lines().any(|l| l.split_whitespace().nth(1) == Some("libvorbis")) {
vec!["vorbis", "libvorbis"]
} else {
eprintln!("SKIPPED (libvorbis decoder): this ffmpeg has no libvorbis; checking its own decoder only");
vec!["vorbis"]
}
}
fn correlation(a: &[f32], b: &[f32]) -> f64 {
let (mut ab, mut aa, mut bb) = (0f64, 0f64, 0f64);
for (&x, &y) in a.iter().zip(b) {
ab += f64::from(x) * f64::from(y);
aa += f64::from(x) * f64::from(x);
bb += f64::from(y) * f64::from(y);
}
ab / (aa * bb).sqrt().max(1e-20)
}
/// Vorbis order (§4.3.9) of the WAV-order layouts the test writes, per channel count.
const VORBIS_ORDER: [&[usize]; 8] =
[&[0], &[0, 1], &[0, 2, 1], &[0, 1, 2, 3], &[0, 2, 1, 3, 4], &[0, 2, 1, 4, 5, 3], &[0, 2, 1, 5, 6, 4, 3], &[0, 2, 1, 6, 7, 4, 5, 3]];
/// 1 to 8 channels through `encode` with a WAV-order layout, at exactly the input's length out of
/// both decoders: ffmpeg's own puts every channel back on its WAV speaker, and libvorbis (through
/// ffmpeg, which passes its order through) gives the file's Vorbis order.
#[test]
fn every_channel_count_decodes_on_the_right_speakers() {
let ff = require_ffmpeg!();
for channels in 1..=8u16 {
let layout = if channels == 7 { ChannelLayout::from_mask(0x70F) } else { ChannelLayout::default_for(channels) };
let buf = tones(48_000, channels.into(), 24_000 + 77);
let opts = EncodeOptions::new(EncodeFormat::Ogg).with_layout(layout).with_quality(0.6);
let file = encode(&buf, &opts).unwrap();
for decoder in decoders(&ff) {
let out = ffmpeg_decode(&ff, decoder, &file, channels.into(), &format!("tones{channels}-{decoder}"));
for (j, got) in out.iter().enumerate() {
let c = if decoder == "libvorbis" { VORBIS_ORDER[usize::from(channels) - 1][j] } else { j };
assert_eq!(got.len(), buf.frames(), "{channels} ch {decoder}: channel {j} length");
let best = (0..buf.channels.len()).max_by(|&a, &b| {
correlation(&buf.channels[a][2048..], &got[2048..]).total_cmp(&correlation(&buf.channels[b][2048..], &got[2048..]))
});
assert_eq!(best, Some(c), "{channels} ch {decoder}: output channel {c} carries input {best:?}");
let r = correlation(&buf.channels[c][2048..], &got[2048..]);
assert!(r > 0.95, "{channels} ch {decoder}: channel {c} correlation {r:.3}");
}
}
}
}
/// The comment header's tags, UTF-8 values included, are what ffprobe reports; a name that is not
/// a valid field name is left out.
#[test]
fn tags_reach_ffprobe() {
let Some(probe) = crate::common::tool("ffprobe").or_else(|| crate::common::skip("ffprobe not found")) else { return };
let opts = OggVorbisOptions::new(3.0)
.with_tag("TITLE", "Smörgåsbord ✓")
.with_tag("ARTIST", "Craft")
.with_tag("bad=name", "dropped")
.with_vendor("craft-audio test");
let file = encode_ogg_vorbis(&tones(44_100, 2, 44_100), &opts).unwrap();
let path = crate::testkit::fixtures_dir("vorbis-encoder").join("tags.ogg");
std::fs::write(&path, &file).unwrap();
let out = Command::new(probe).args(["-v", "error", "-of", "json", "-show_streams", "-show_format"]).arg(&path).output().unwrap();
assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr));
let j: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap();
let s = &j["streams"][0];
assert_eq!((s["codec_name"].as_str(), s["channels"].as_u64(), s["sample_rate"].as_str()), (Some("vorbis"), Some(2), Some("44100")));
let tags = &s["tags"];
assert_eq!((tags["TITLE"].as_str(), tags["ARTIST"].as_str()), (Some("Smörgåsbord ✓"), Some("Craft")));
assert!(tags.as_object().unwrap().keys().all(|k| !k.contains("bad")), "{tags}");
assert_eq!(j["format"]["duration"].as_str().map(|d| d.parse::<f64>().unwrap()), Some(1.0));
}
/// The lowest and highest qualities and the bit-rate path decode at the exact length, mono at
/// 44.1 kHz and stereo at 22.05 kHz.
#[test]
fn quality_extremes_and_rates_decode_exactly() {
let ff = require_ffmpeg!();
for (rate, channels, quality) in [(44_100u32, 1usize, -1.0f32), (44_100, 1, 10.0), (22_050, 2, -1.0), (22_050, 2, 10.0), (96_000, 2, 5.0)] {
let buf = tones(rate, channels, rate as usize / 2 + 13);
let file = encode_ogg_vorbis(&buf, &OggVorbisOptions::new(quality)).unwrap();
for decoder in decoders(&ff) {
let out = ffmpeg_decode(&ff, decoder, &file, channels, &format!("q{quality}-{rate}-{channels}-{decoder}"));
assert!(out.iter().all(|c| c.len() == buf.frames()), "q{quality} {rate} Hz {decoder}: {}", out[0].len());
let r = correlation(&buf.channels[0][2048..], &out[0][2048..]);
assert!(r > 0.9, "q{quality} {rate} Hz {decoder}: correlation {r:.3}");
}
}
let low = craft_audio::encode_ogg(&tones(48_000, 2, 48_000), craft_audio::OGG_BITRATES.0).unwrap();
let high = craft_audio::encode_ogg(&tones(48_000, 2, 48_000), craft_audio::OGG_BITRATES.1).unwrap();
assert!(low.len() < high.len(), "{} vs {} bytes", low.len(), high.len());
}
+211
View File
@@ -0,0 +1,211 @@
//! The native FLAC decoder: round trips through this crate's encoder over bit depths, channel
//! counts, block sizes and effort levels; hostile input; and ffmpeg as the reference (oracle;
//! skipped when ffmpeg is absent, a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`) for files
//! ffmpeg's encoder writes at every compression level.
use craft_audio::flac::{DecodeError, Decoder, Encoder, EncoderConfig, STREAMINFO_OFFSET, StreamInfo, decode};
use std::process::Command;
fn signal(channels: usize, frames: usize, bits: u32, seed: u32) -> Vec<Vec<i32>> {
let max = if bits == 32 { i32::MAX as f64 } else { ((1i64 << (bits - 1)) - 1) as f64 };
let mut s = seed;
(0..channels)
.map(|c| {
(0..frames)
.map(|i| {
s ^= s << 13;
s ^= s >> 17;
s ^= s << 5;
let t = i as f64;
let v = match (i / 3000) % 4 {
0 => (t * (0.01 + 0.007 * c as f64)).sin() * 0.6 + (t * 0.31).sin() * 0.1,
1 => (f64::from(s) / f64::from(u32::MAX) - 0.5) * 1.8,
2 => 0.0,
_ => {
if (i / 40) % 2 == 0 {
0.99
} else {
-0.99
}
}
};
(v * max).round() as i32
})
.collect()
})
.collect()
}
fn encode(channels: &[Vec<i32>], rate: u32, bits: u8, block: u16, level: u8) -> Vec<u8> {
let mut cfg = EncoderConfig::new(rate, channels.len() as u8, bits);
cfg.block_size = block;
cfg.level = level;
let mut enc = Encoder::new(cfg).expect("encoder");
let mut file = enc.header(&[]);
let refs: Vec<&[i32]> = channels.iter().map(Vec::as_slice).collect();
file.extend(enc.encode(&refs).expect("encode"));
file.extend(enc.finish());
let at = STREAMINFO_OFFSET as usize;
file[at..at + 34].copy_from_slice(&enc.streaminfo());
file
}
#[test]
fn round_trips_through_the_encoder() {
for (channels, bits) in [(1usize, 8u8), (2, 8), (1, 12), (2, 16), (2, 20), (3, 24), (6, 24), (8, 16), (2, 32), (1, 32)] {
for (block, level) in [(4096u16, 5u8), (16, 0), (1000, 8), (65535, 3), (576, 1)] {
let frames = 9000 + usize::from(block % 97);
let src = signal(channels, frames, u32::from(bits), 0x1234_5679 + u32::from(bits));
let file = encode(&src, 44_100, bits, block, level);
let got = decode(&file).unwrap_or_else(|e| panic!("{channels}ch {bits}-bit block {block} level {level}: {e}"));
assert_eq!((got.info.channels as usize, got.info.bits_per_sample, got.info.sample_rate), (channels, bits, 44_100));
assert_eq!(got.channels, src, "{channels}ch {bits}-bit block {block} level {level}");
}
}
}
#[test]
fn frames_decode_one_at_a_time() {
let src = signal(2, 10_000, 16, 7);
let file = encode(&src, 48_000, 16, 4096, 5);
let info = StreamInfo::parse(&file[STREAMINFO_OFFSET as usize..]).expect("streaminfo");
let dec = Decoder::new(info);
// The first frame follows the metadata blocks; find the first position a frame decodes at.
let at = (42..file.len()).find(|&p| dec.decode_frame(&file[p..]).is_ok()).expect("a frame");
let first = dec.decode_frame(&file[at..]).expect("frame");
assert_eq!((first.block_size, first.number, first.sample_rate, first.bits_per_sample), (4096, 0, 48_000, 16));
assert_eq!(first.channels[0], src[0][..4096]);
// Without the STREAMINFO a frame that spells out its own rate and depth still decodes.
let _ = Decoder::without_streaminfo().decode_frame(&file[at..]);
}
#[test]
fn hostile_input_is_an_error_not_a_panic() {
let src = signal(2, 6000, 16, 11);
let file = encode(&src, 44_100, 16, 1152, 5);
for n in 0..file.len().min(3000) {
let _ = decode(&file[..n]);
}
// A cut file is truncated, not silently shorter.
assert!(decode(&file[..file.len() - 5]).is_err());
// Every bit flip in the first frames is caught by a CRC or another check, never a panic.
for i in 0..file.len().min(2500) {
for bit in 0..8 {
let mut c = file.clone();
c[i] ^= 1 << bit;
let _ = decode(&c);
}
}
// A sample limit stops a stream that claims too much.
assert_eq!(craft_audio::flac::decode_with_limit(&file, 1000), Err(DecodeError::TooLarge));
assert!(decode(b"nope").is_err());
assert!(decode(b"fLaC").is_err());
}
// ---- ffmpeg ----
/// Little-endian 16- or 32-bit samples of raw PCM.
fn pcm_values(raw: &[u8], bytes: usize) -> Vec<i32> {
if bytes == 2 {
raw.as_chunks::<2>().0.iter().map(|b| i32::from(i16::from_le_bytes(*b))).collect()
} else {
raw.as_chunks::<4>().0.iter().map(|b| i32::from_le_bytes(*b)).collect()
}
}
/// Planar channels as interleaved samples.
fn interleave(channels: &[Vec<i32>]) -> Vec<i32> {
let frames = channels.first().map_or(0, Vec::len);
(0..frames).flat_map(|i| channels.iter().map(move |c| c[i])).collect()
}
fn ffmpeg_run(ff: &std::path::Path, args: &[&str]) -> Vec<u8> {
let out = Command::new(ff).args(["-v", "error", "-nostdin", "-y"]).args(args).output().expect("run ffmpeg");
assert!(out.status.success(), "ffmpeg {args:?}: {}", String::from_utf8_lossy(&out.stderr));
out.stdout
}
#[test]
fn ffmpeg_files_decode_to_ffmpegs_samples() {
let Some(ff) = craft_media_testkit::ffmpeg() else { return };
let dir = craft_media_testkit::fixture_subdir("flac_decode");
let tones = "aevalsrc=exprs=0.4*sin(2*PI*441*t)+0.2*sin(2*PI*3100*t)|0.3*sin(2*PI*660*t)*sin(2*PI*3*t):s=44100:d=1.3";
let noise = "anoisesrc=d=1.3:c=white:r=44100:a=0.5:s=3";
for (src, name) in [(tones, "tones"), (noise, "noise")] {
for channels in [1usize, 2, 6] {
for (fmt, bytes, bits) in [("s16", 2usize, 16u32), ("s32", 4, 24)] {
for level in [0, 3, 5, 8, 12] {
let file = dir.join(format!("ffmpeg_{name}_{channels}ch_{bits}_l{level}.flac"));
let f = file.to_str().expect("path");
let (ch, lv) = (channels.to_string(), level.to_string());
ffmpeg_run(&ff, &["-f", "lavfi", "-i", src, "-ac", &ch, "-sample_fmt", fmt, "-compression_level", &lv, "-c:a", "flac", f]);
let theirs = ffmpeg_run(&ff, &["-i", f, "-f", if bits == 16 { "s16le" } else { "s32le" }, "-"]);
let got =
decode(&std::fs::read(&file).expect("read")).unwrap_or_else(|e| panic!("{name} {channels}ch {bits}-bit level {level}: {e}"));
assert_eq!(got.info.channels as usize, channels);
let frames = got.channels[0].len();
assert_eq!(theirs.len(), frames * channels * bytes, "{name} {channels}ch {bits} l{level}: length");
// ffmpeg returns 24-bit audio left-aligned in 32 bits.
let shift = if bits == 24 { 8 } else { 0 };
let mine: Vec<i32> = interleave(&got.channels).into_iter().map(|v| v << shift).collect();
let ok = mine == pcm_values(&theirs, bytes);
assert!(ok, "{name} {channels}ch {bits}-bit level {level}: samples differ from ffmpeg's decode");
}
}
}
}
}
#[test]
fn ffmpeg_decodes_our_files_to_our_samples() {
let Some(ff) = craft_media_testkit::ffmpeg() else { return };
let dir = craft_media_testkit::fixture_subdir("flac_decode");
for (channels, bits) in [(1usize, 16u8), (2, 16), (2, 24), (6, 16)] {
let src = signal(channels, 20_000, u32::from(bits), 99);
let file = dir.join(format!("ours_{channels}ch_{bits}.flac"));
std::fs::write(&file, encode(&src, 44_100, bits, 4096, 5)).expect("write");
let theirs = ffmpeg_run(&ff, &["-i", file.to_str().expect("path"), "-f", if bits == 16 { "s16le" } else { "s32le" }, "-"]);
let got = decode(&std::fs::read(&file).expect("read")).expect("decode");
let bytes = if bits == 16 { 2 } else { 4 };
let shift = if bytes == 4 { 8 } else { 0 };
let want: Vec<i32> = pcm_values(&theirs, bytes).into_iter().map(|v| v >> shift).collect();
assert!(interleave(&got.channels) == want, "{channels}ch {bits}-bit: samples differ from ffmpeg's decode");
}
}
#[test]
fn rejects_unknown_length_corruption_and_streaminfo_format_mismatch() {
let src = signal(1, 100, 16, 7);
let original = encode(&src, 44_100, 16, 4096, 5);
let at = STREAMINFO_OFFSET as usize;
// STREAMINFO is not CRC-covered. Clear only the 36-bit total sample count.
let mut unknown = original.clone();
unknown[at + 13] &= 0xf0;
unknown[at + 14..at + 18].fill(0);
assert_eq!(decode(&unknown).unwrap().channels, src);
for tail in [vec![0xff], vec![0x00, 0x00]] {
let mut broken = unknown.clone();
broken.extend(tail);
assert!(decode(&broken).is_err());
}
// The encoder spells these in frame headers, independent of STREAMINFO.
let mut wrong_rate = original.clone();
let mut packed = u64::from_be_bytes(wrong_rate[at + 10..at + 18].try_into().unwrap());
packed = (packed & ((1u64 << 44) - 1)) | (48_000u64 << 44);
wrong_rate[at + 10..at + 18].copy_from_slice(&packed.to_be_bytes());
assert!(decode(&wrong_rate).is_err());
let mut wrong_depth = original;
wrong_depth[at + 12] = (wrong_depth[at + 12] & !1) | 1;
wrong_depth[at + 13] = (wrong_depth[at + 13] & 0x0f) | 0x70;
assert!(decode(&wrong_depth).is_err());
}
#[test]
fn caller_constructed_streaminfo_is_validated_before_decoding() {
let file = encode(&signal(1, 100, 16, 9), 44_100, 16, 4096, 5);
let info = StreamInfo::parse(&file[STREAMINFO_OFFSET as usize..]).unwrap();
for bits in [0, 1, 3, 33, 64, 255] {
let invalid = StreamInfo { bits_per_sample: bits, ..info };
assert!(matches!(Decoder::new(invalid).decode_frame(&[]), Err(DecodeError::Invalid(_))));
}
}
+2
View File
@@ -19,3 +19,5 @@ thiserror = { workspace = true }
proptest = { workspace = true }
# A second XML reader for the tests: checks of written TTML, and the comparison in tests/it/ttml_xml_views.rs.
roxmltree = "0.21"
# ffmpeg as the reference for the ASS tests (tests/it/ass.rs).
craft-media-testkit = { workspace = true }
+1
View File
@@ -55,6 +55,7 @@ lists at most 20 characters per stream.
| EBU STL `.stl` | yes | yes | EBU Tech 3264-E: GSI block + 128-byte TTI blocks, `STL25.01` (25 fps) or `STL30.01` (read/written as 29.97 fps, non-drop labels), ISO/IEC 6937 Latin text (accents as diacritic + letter), italics / underline codes as `<i>` / `<u>`, extension blocks for long subtitles, start-of-programme (TCP) subtracted. |
| TTML `.ttml` (IMSC1) | yes | yes | Writes the IMSC1 Text profile (TTML2 namespaces) with the document's language as `xml:lang` (read back on import): `<p>` per cue, `<br/>`, styled `<span>`s for `<i>` / `<b>` / `<u>`, `ttm:agent` speakers, exact frame times (`123f`) at the sequence rate. Reads any TTML (TTML1, TTML2, the 2006 `ttaf1` drafts): clock / frame / offset / tick times, `dur`, nested time containers (`body` / `div` begins offset and ends clip their contents; `seq` containers are read as `par` with a warning), timed spans, referenced styles. |
| DFXP `.dfxp` | yes | yes | TTML1 with the DFXP presentation profile; clock times in milliseconds (frame-exact after `snap_to_frames`). |
| ASS / SSA `.ass`, `.ssa` | yes | yes (ASS v4+) | The `[Events]` section: `Dialogue:` lines through the file's `Format:` line, `Name` as the speaker, `\N`, `\n`, `\h` and the `\i`, `\b`, `\u` override tags kept (as `<i>`, `<b>`, `<u>`); other override tags and drawing lines are dropped. Times are centiseconds. Written files have one `Default` style. |
All cue times are exact `Tick`s. Milliseconds convert exactly (1 ms = 254 016 000 ticks); SCC
frames map through `FrameRate::FPS_29_97`. `Document::snap_to_frames` snaps cues to a frame grid and
+244
View File
@@ -0,0 +1,244 @@
//! Advanced SubStation Alpha and SubStation Alpha (`.ass`, `.ssa`).
//!
//! Reading takes the `[Events]` section: its `Format:` line gives the field order (ASS v4+ and SSA
//! v4 differ in the first field only), each `Dialogue:` line becomes a cue; `Comment:` lines and
//! the other sections are skipped. A cue gets the `Name` field as its speaker. The text keeps what
//! the neutral model can show: `\N` and `\n` become a line break and a space, `\h` a no-break
//! space, and the override tags `\i`, `\b` and `\u` become `<i>`, `<b>` and `<u>`. Every other
//! override tag (colours, fonts, positions, karaoke, animation) is dropped, and a line that draws
//! vector shapes (`\p1`) is skipped with a warning.
//!
//! Writing gives a minimal ASS v4+ file: one `Default` style, one `Dialogue:` line per cue with
//! times to the centisecond. Braces in the text, which ASS reads as override blocks, are written
//! as the full-width `{` and `}`.
use crate::time::{TICKS_PER_SECOND, Tick};
use crate::{Cue, Document, Error, Result};
const TICKS_PER_CS: i64 = TICKS_PER_SECOND / 100;
/// `H:MM:SS.cc`, the form SSA and ASS use (any number of fraction digits is read).
fn parse_time(s: &str) -> Option<Tick> {
let s = s.trim();
let (hms, frac) = s.split_once('.').unwrap_or((s, ""));
let mut parts = hms.split(':');
let (h, m, sec) = (parts.next()?, parts.next()?, parts.next()?);
if parts.next().is_some() {
return None;
}
let num = |p: &str| -> Option<i64> { (!p.is_empty() && p.len() <= 9 && p.bytes().all(|b| b.is_ascii_digit())).then(|| p.parse().ok()).flatten() };
let (h, m, sec) = (num(h)?, num(m)?, num(sec)?);
if m >= 60 || sec >= 60 || h > 9999 {
return None;
}
let mut ticks = ((h * 60 + m) * 60 + sec) * TICKS_PER_SECOND;
if !frac.is_empty() {
if frac.len() > 9 || !frac.bytes().all(|b| b.is_ascii_digit()) {
return None;
}
let d: i64 = frac.parse().ok()?;
ticks += (i128::from(d) * i128::from(TICKS_PER_SECOND) / 10i128.pow(frac.len() as u32)) as i64;
}
Some(Tick(ticks))
}
fn format_time(t: Tick) -> String {
let cs = (i128::from(t.0.max(0)) + i128::from(TICKS_PER_CS) / 2) / i128::from(TICKS_PER_CS);
let (h, m, s, c) = (cs / 360_000, (cs / 6000) % 60, (cs / 100) % 60, cs % 100);
format!("{h}:{m:02}:{s:02}.{c:02}")
}
/// The Text field of a Dialogue line as cue text; `None` when the line draws shapes.
fn convert_text(raw: &str) -> Option<String> {
let mut out = String::new();
// Open `<i>`, `<b>`, `<u>` in the order they were opened.
let mut open: Vec<char> = Vec::new();
let mut chars = raw.chars().peekable();
let mut drawing = false;
let mut dropped_drawing = false;
while let Some(c) = chars.next() {
match c {
'{' => {
let mut block = String::new();
let mut closed = false;
for b in chars.by_ref() {
if b == '}' {
closed = true;
break;
}
block.push(b);
}
if !closed {
// An unterminated block is literal text.
out.push('{');
out.push_str(&block);
continue;
}
for tag in block.split('\\').skip(1) {
let tag = tag.trim_end();
let (name, val) = tag.split_at(tag.find(|c: char| !c.is_ascii_alphabetic()).unwrap_or(tag.len()));
match name {
"i" | "b" | "u" => {
let letter = name.chars().next().unwrap_or('i');
let on = val.trim() != "0";
if on && !open.contains(&letter) {
open.push(letter);
out.push_str(&format!("<{letter}>"));
} else if !on && open.contains(&letter) {
// Close back to it, reopening what was opened after.
let at = open.iter().position(|&o| o == letter).unwrap_or(0);
let after: Vec<char> = open.split_off(at + 1);
for o in after.iter().rev() {
out.push_str(&format!("</{o}>"));
}
open.pop();
out.push_str(&format!("</{letter}>"));
for o in after {
open.push(o);
out.push_str(&format!("<{o}>"));
}
}
}
"p" => drawing = val.trim().parse::<u32>().unwrap_or(0) != 0,
_ => {}
}
}
}
'\\' => match chars.peek() {
Some('N') => {
chars.next();
out.push('\n');
}
Some('n') => {
chars.next();
out.push(' ');
}
Some('h') => {
chars.next();
out.push('\u{a0}');
}
_ => out.push('\\'),
},
c => {
if drawing {
dropped_drawing = true;
} else {
out.push(c);
}
}
}
}
if dropped_drawing && out.trim().is_empty() {
return None;
}
while let Some(o) = open.pop() {
out.push_str(&format!("</{o}>"));
}
Some(out)
}
pub fn parse(text: &str) -> Result<Document> {
let text = text.strip_prefix('\u{feff}').unwrap_or(text);
let mut doc = Document::default();
let mut in_events = false;
let mut saw_events = false;
// Column of each field in a Dialogue line.
let mut cols: Vec<String> =
["layer", "start", "end", "style", "name", "marginl", "marginr", "marginv", "effect", "text"].map(String::from).to_vec();
let mut drawings = 0usize;
for (n, line) in text.lines().enumerate() {
let line = line.trim();
if line.starts_with('[') && line.ends_with(']') {
in_events = line.eq_ignore_ascii_case("[events]");
saw_events |= in_events;
continue;
}
if !in_events {
continue;
}
let Some((key, rest)) = line.split_once(':') else { continue };
let rest = rest.trim_start();
match key.trim().to_ascii_lowercase().as_str() {
"format" => cols = rest.split(',').map(|f| f.trim().to_ascii_lowercase()).collect(),
"dialogue" => {
// The last column (Text) keeps its commas.
let fields: Vec<&str> = rest.splitn(cols.len(), ',').collect();
let get = |name: &str| cols.iter().position(|c| c == name).and_then(|i| fields.get(i)).copied();
let (Some(start), Some(end), Some(raw)) = (get("start").and_then(parse_time), get("end").and_then(parse_time), get("text")) else {
doc.warnings.push(format!("line {}: not a Dialogue line this reader understands", n + 1));
continue;
};
let Some(text) = convert_text(raw) else {
drawings += 1;
continue;
};
let name = get("name").map(str::trim).filter(|s| !s.is_empty());
doc.cues.push(Cue { start, end, text, speaker: name.map(str::to_string), ..Default::default() });
}
_ => {}
}
}
if !saw_events {
return Err(Error::NotFormat("ASS"));
}
if drawings > 0 {
doc.warnings.push(format!("{drawings} vector drawing line(s) were skipped"));
}
Ok(doc)
}
fn escape(text: &str) -> String {
let mut out = String::with_capacity(text.len() + 8);
let mut rest = text;
while !rest.is_empty() {
if let Some(r) = rest.strip_prefix("\r\n") {
out.push_str("\\N");
rest = r;
} else if let Some(r) = rest.strip_prefix("<i>") {
out.push_str("{\\i1}");
rest = r;
} else if let Some(r) = rest.strip_prefix("</i>") {
out.push_str("{\\i0}");
rest = r;
} else if let Some(r) = rest.strip_prefix("<b>") {
out.push_str("{\\b1}");
rest = r;
} else if let Some(r) = rest.strip_prefix("</b>") {
out.push_str("{\\b0}");
rest = r;
} else if let Some(r) = rest.strip_prefix("<u>") {
out.push_str("{\\u1}");
rest = r;
} else if let Some(r) = rest.strip_prefix("</u>") {
out.push_str("{\\u0}");
rest = r;
} else if let Some(c) = rest.chars().next() {
match c {
'\n' | '\r' => out.push_str("\\N"),
// Empty override block separates a literal slash from N/n/h.
'\\' => out.push_str("\\{}"),
'{' => out.push('\u{ff5b}'),
'}' => out.push('\u{ff5d}'),
'\u{a0}' => out.push_str("\\h"),
c => out.push(c),
}
rest = &rest[c.len_utf8()..];
}
}
out
}
pub fn write(doc: &Document) -> String {
let mut out = String::from(
"[Script Info]\n; Written by craft-captions\nScriptType: v4.00+\nWrapStyle: 0\nScaledBorderAndShadow: yes\nPlayResX: 384\nPlayResY: 288\n\n\
[V4+ Styles]\nFormat: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding\n\
Style: Default,Arial,16,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,0,0,0,0,100,100,0,0,1,1,0,2,10,10,10,1\n\n\
[Events]\nFormat: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n",
);
for c in &doc.cues {
// A comma in the Name field would shift the columns; the other fields are fixed text.
let name = c.speaker.as_deref().unwrap_or("").replace([',', '\n', '\r'], " ");
out.push_str(&format!("Dialogue: 0,{},{},Default,{},0,0,0,,{}\n", format_time(c.start), format_time(c.end), name, escape(&c.text)));
}
out
}
+14 -3
View File
@@ -4,7 +4,7 @@
//! - **Files:** [SubRip](srt) (`.srt`), [WebVTT](vtt) (`.vtt`), [Scenarist SCC](scc) (`.scc`,
//! CEA-608 pop-on, roll-up and paint-on on read; pop-on on write), [MacCaption MCC](mcc)
//! (`.mcc`, SMPTE ST 334-2 packets with CEA-608 and CEA-708), [EBU STL](stl) (`.stl`, EBU
//! Tech 3264) and [TTML](ttml) (`.ttml` IMSC1 Text profile, `.dfxp` DFXP) parse into a
//! Tech 3264), [ASS/SSA](ass) (`.ass`, `.ssa`) and [TTML](ttml) (`.ttml` IMSC1 Text profile, `.dfxp` DFXP) parse into a
//! [`Document`] of [`Cue`]s and write back. Readers are forgiving (BOM, UTF-16, Windows-1252
//! fallback, CRLF/CR, missing SRT indexes, `,` or `.` milliseconds, missing hours, no blank
//! line between cues); writers are strict. WebVTT cue identifiers, cue settings, `<v Speaker>`
@@ -32,6 +32,7 @@
#[doc = include_str!("../README.md")]
struct ReadmeExamples;
pub mod ass;
pub mod cea608;
pub mod mcc;
pub mod scc;
@@ -69,10 +70,12 @@ pub enum Format {
Ttml,
/// DFXP (TTML1).
Dfxp,
/// Advanced SubStation Alpha (also reads SubStation Alpha).
Ass,
}
impl Format {
pub const ALL: [Format; 7] = [Format::Srt, Format::WebVtt, Format::Scc, Format::Mcc, Format::Stl, Format::Ttml, Format::Dfxp];
pub const ALL: [Format; 8] = [Format::Srt, Format::WebVtt, Format::Scc, Format::Mcc, Format::Stl, Format::Ttml, Format::Dfxp, Format::Ass];
pub fn name(self) -> &'static str {
match self {
Format::Srt => "SubRip (SRT)",
@@ -82,6 +85,7 @@ impl Format {
Format::Stl => "EBU STL",
Format::Ttml => "TTML (IMSC1)",
Format::Dfxp => "DFXP (TTML1)",
Format::Ass => "Advanced SubStation Alpha (ASS)",
}
}
pub fn extension(self) -> &'static str {
@@ -93,6 +97,7 @@ impl Format {
Format::Stl => "stl",
Format::Ttml => "ttml",
Format::Dfxp => "dfxp",
Format::Ass => "ass",
}
}
pub fn from_name(s: &str) -> Option<Format> {
@@ -104,6 +109,7 @@ impl Format {
"stl" | "ebu-stl" | "ebustl" | "ebu" => Some(Format::Stl),
"ttml" | "imsc" | "imsc1" | "xml" => Some(Format::Ttml),
"dfxp" => Some(Format::Dfxp),
"ass" | "ssa" => Some(Format::Ass),
_ => None,
}
}
@@ -251,6 +257,9 @@ pub fn detect(bytes: &[u8], ext: Option<&str>) -> Option<Format> {
if head.starts_with("File Format=MacCaption_MCC") {
return Some(Format::Mcc);
}
if head.starts_with("[Script Info]") {
return Some(Format::Ass);
}
let ext = ext.map(|e| e.trim_start_matches('.').to_ascii_lowercase());
if (head.starts_with('<') && (head.contains("<tt ") || head.contains("<tt>") || head.contains(":tt ")))
&& (head.contains("ns/ttml") || head.contains("ttaf1"))
@@ -262,7 +271,7 @@ pub fn detect(bytes: &[u8], ext: Option<&str>) -> Option<Format> {
match ext.as_deref() {
Some("srt") if has_arrow_timing || head.is_empty() => Some(Format::Srt),
Some("vtt") if has_arrow_timing => Some(Format::WebVtt),
Some("srt" | "vtt" | "scc" | "mcc" | "stl" | "ttml" | "dfxp" | "xml") => None,
Some("srt" | "vtt" | "scc" | "mcc" | "stl" | "ttml" | "dfxp" | "xml" | "ass" | "ssa") => None,
_ if has_arrow_timing => Some(Format::Srt),
_ => None,
}
@@ -296,6 +305,7 @@ fn parse_unbounded(bytes: &[u8], format: Format) -> Result<Document> {
Format::Scc => scc::parse(&text),
Format::Mcc => mcc::parse(&text),
Format::Ttml | Format::Dfxp => ttml::parse(&text),
Format::Ass => ass::parse(&text),
Format::Stl => stl::parse(bytes),
}
}
@@ -333,6 +343,7 @@ pub fn write(doc: &Document, format: Format, opts: WriteOptions) -> Vec<u8> {
Format::Stl => stl::write(doc, opts.rate, opts.program),
Format::Ttml => ttml::write(doc, ttml::Flavor::Imsc1, opts.rate, doc.language.as_deref().unwrap_or("")).into_bytes(),
Format::Dfxp => ttml::write(doc, ttml::Flavor::Dfxp, opts.rate, doc.language.as_deref().unwrap_or("")).into_bytes(),
Format::Ass => ass::write(doc).into_bytes(),
}
}
+142
View File
@@ -0,0 +1,142 @@
//! SubStation Alpha / Advanced SubStation Alpha: reading, writing, and ffmpeg as the reference
//! (oracle; skipped when ffmpeg is absent, a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`).
use craft_captions::{Cue, Document, Format, Tick, WriteOptions, detect, parse, write};
use std::process::Command;
const T: i64 = craft_captions::TICKS_PER_SECOND;
fn cs(n: i64) -> Tick {
Tick(n * (T / 100))
}
const SAMPLE: &str = "\u{feff}[Script Info]\n; a comment\nTitle: Test\nScriptType: v4.00+\n\n[V4+ Styles]\nFormat: Name, Fontname\nStyle: Default,Arial\n\n[Events]\nFormat: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\nComment: 0,0:00:00.00,0:00:01.00,Default,,0,0,0,,not a cue\nDialogue: 0,0:00:01.50,0:00:03.25,Default,Bob,0,0,0,,Hello, {\\i1}world{\\i0}\\Nsecond {\\b1}line{\\b0}\nDialogue: 0,1:02:03.04,1:02:04.5,Default,,0,0,0,,{\\pos(10,20)\\c&H0000FF&}Red\\hspace\\nsoft\nDialogue: 0,0:00:09.00,0:00:10.00,Default,,0,0,0,,{\\p1}m 0 0 l 100 0 100 100{\\p0}\n";
#[test]
fn reads_dialogue_lines_with_commas_overrides_and_line_breaks() {
let doc = parse(SAMPLE.as_bytes(), Format::Ass).unwrap();
assert_eq!(doc.cues.len(), 2);
assert_eq!(doc.cues[0].start, cs(150));
assert_eq!(doc.cues[0].end, cs(325));
assert_eq!(doc.cues[0].text, "Hello, <i>world</i>\nsecond <b>line</b>");
assert_eq!(doc.cues[0].speaker.as_deref(), Some("Bob"));
// 1:02:03.04, a one-digit fraction, an override block dropped, \h and \n.
assert_eq!(doc.cues[1].start, Tick((3600 + 120 + 3) * T + 4 * (T / 100)));
assert_eq!(doc.cues[1].end, Tick((3600 + 120 + 4) * T + T / 2));
assert_eq!(doc.cues[1].text, "Red\u{a0}space soft");
assert!(doc.warnings.iter().any(|w| w.contains("vector drawing")), "{:?}", doc.warnings);
}
#[test]
fn ssa_v4_files_read_through_their_own_format_line() {
let ssa = "[Script Info]\nScriptType: v4.00\n\n[V4 Styles]\nFormat: Name\n\n[Events]\nFormat: Marked, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\nDialogue: Marked=0,0:00:01.00,0:00:02.00,*Default,,0000,0000,0000,,Old style\n";
let doc = parse(ssa.as_bytes(), Format::Ass).unwrap();
assert_eq!((doc.cues[0].start, doc.cues[0].end, doc.cues[0].text.as_str()), (cs(100), cs(200), "Old style"));
}
#[test]
fn write_then_read_keeps_times_text_and_speaker() {
let cues = vec![
Cue { start: cs(100), end: cs(250), text: "Line one\n<i>line two</i>".into(), speaker: Some("Ann".into()), ..Default::default() },
Cue {
start: Tick(7200 * T + 99 * (T / 100)),
end: Tick(7203 * T),
text: "<b>bold</b> and <u>under</u> {braces}".into(),
..Default::default()
},
];
let doc = Document { cues, ..Default::default() };
let bytes = write(&doc, Format::Ass, WriteOptions::default());
assert_eq!(detect(&bytes, Some("ass")), Some(Format::Ass));
let back = parse(&bytes, Format::Ass).unwrap();
assert_eq!(back.cues[0], doc.cues[0]);
assert_eq!(back.cues[1].start, doc.cues[1].start);
// Braces are written full-width, as ASS reads `{` as the start of an override block.
assert_eq!(back.cues[1].text, "<b>bold</b> and <u>under</u> \u{ff5b}braces\u{ff5d}");
}
#[test]
fn files_without_an_events_section_are_not_ass() {
assert!(parse(b"[Script Info]\nTitle: x\n", Format::Ass).is_err());
assert!(parse(b"hello", Format::Ass).is_err());
assert_eq!(detect(b"hello", Some("ass")), None);
}
#[test]
fn bad_lines_are_skipped_with_a_warning() {
let text = "[Events]\nFormat: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\nDialogue: 0,garbage,0:00:02.00,Default,,0,0,0,,x\nDialogue: 0,0:00:01.00,0:00:02.00,Default,,0,0,0,,ok\n";
let doc = parse(text.as_bytes(), Format::Ass).unwrap();
assert_eq!(doc.cues.len(), 1);
assert_eq!(doc.warnings.len(), 1);
}
fn ffmpeg_run(ff: &std::path::Path, args: &[&str]) {
let out = Command::new(ff).args(["-v", "error", "-nostdin", "-y"]).args(args).output().expect("run ffmpeg");
assert!(out.status.success(), "ffmpeg {args:?}: {}", String::from_utf8_lossy(&out.stderr));
}
const SRT: &str = "1\n00:00:01,500 --> 00:00:03,250\nHello, <i>world</i>\nsecond <b>line</b>\n\n2\n00:00:04,000 --> 00:00:05,990\nPlain, with a comma\n\n3\n01:00:00,010 --> 01:00:02,999\n<u>Under</u> and <i>italic <b>both</b></i>\n";
#[test]
fn we_read_what_ffmpeg_writes() {
let Some(ff) = craft_media_testkit::ffmpeg() else { return };
let dir = craft_media_testkit::fixture_subdir("captions_ass");
let (srt, ass) = (dir.join("in.srt"), dir.join("ffmpeg.ass"));
std::fs::write(&srt, SRT).expect("write");
ffmpeg_run(&ff, &["-i", srt.to_str().expect("path"), ass.to_str().expect("path")]);
let from_ass = parse(&std::fs::read(&ass).expect("read"), Format::Ass).expect("parse ass");
let from_srt = parse(SRT.as_bytes(), Format::Srt).expect("parse srt");
assert_eq!(from_ass.cues.len(), from_srt.cues.len());
for (a, s) in from_ass.cues.iter().zip(&from_srt.cues) {
// ASS times are centiseconds; the SRT ones round to them.
assert_eq!(a.start, cs((s.start.0 + T / 200) / (T / 100)), "start of {:?}", s.text);
assert_eq!(a.end, cs((s.end.0 + T / 200) / (T / 100)), "end of {:?}", s.text);
assert_eq!(a.text, s.text);
}
}
#[test]
fn ffmpeg_reads_what_we_write() {
let Some(ff) = craft_media_testkit::ffmpeg() else { return };
let dir = craft_media_testkit::fixture_subdir("captions_ass");
let mut doc = parse(SRT.as_bytes(), Format::Srt).expect("parse srt");
doc.cues.push(Cue { start: cs(400_000), end: cs(400_100), text: r"C:\New\notes\hello and \N \n \h".into(), ..Default::default() });
let (ass, srt) = (dir.join("ours.ass"), dir.join("back.srt"));
std::fs::write(&ass, write(&doc, Format::Ass, WriteOptions::default())).expect("write");
ffmpeg_run(&ff, &["-i", ass.to_str().expect("path"), srt.to_str().expect("path")]);
let theirs = parse(&std::fs::read(&srt).expect("read"), Format::Srt).expect("parse back");
assert_eq!(theirs.cues.len(), doc.cues.len());
for (t, d) in theirs.cues.iter().zip(&doc.cues) {
assert_eq!(t.start, cs((d.start.0 + T / 200) / (T / 100)));
assert_eq!(t.end, cs((d.end.0 + T / 200) / (T / 100)));
assert_eq!(t.text, d.text, "text of the cue at {:?}", d.start);
}
}
#[test]
fn literal_backslashes_survive_ass_round_trip() {
let text = r"C:\New\notes\hello and \N \n \h";
let doc = Document { cues: vec![Cue { start: cs(100), end: cs(200), text: text.into(), ..Default::default() }], ..Default::default() };
let bytes = write(&doc, Format::Ass, WriteOptions::default());
assert_eq!(parse(&bytes, Format::Ass).unwrap().cues[0].text, text);
}
#[test]
fn carriage_returns_cannot_create_dialogue_records() {
let doc = Document {
cues: vec![Cue {
start: cs(100),
end: cs(200),
text: "first\rsecond\r\nthird".into(),
speaker: Some("Ann\rOther".into()),
..Default::default()
}],
..Default::default()
};
let bytes = write(&doc, Format::Ass, WriteOptions::default());
assert!(!bytes.contains(&b'\r'));
let back = parse(&bytes, Format::Ass).unwrap();
assert_eq!(back.cues.len(), 1);
assert_eq!(back.cues[0].text, "first\nsecond\nthird");
assert_eq!(back.cues[0].speaker.as_deref(), Some("Ann Other"));
}
+1
View File
@@ -1,3 +1,4 @@
mod ass;
mod limits;
mod literal_roundtrips;
mod mcc_708_timing;
+5
View File
@@ -29,11 +29,15 @@ pub mod lut3d;
pub mod lutfile;
pub mod math;
pub mod named;
mod normalize;
pub mod pipeline;
pub mod profile;
pub mod sony_lut;
pub mod spi1d;
pub mod synth;
pub mod transform;
mod write;
pub mod write_v2;
pub use builtin::Builtin;
pub use clut::Clut;
@@ -42,6 +46,7 @@ pub use gamut::GamutCheck;
pub use gpu_matrix_shaper::GpuMatrixShaper;
pub use lut3d::Lut3d;
pub use named::{NamedColor, NamedColors};
pub use normalize::normalize_for_embedding;
pub use profile::{ColorSpace, Pcs, Profile, ProfileClass};
pub use transform::{SampleKind, Transform, TransformOptions, black_point, cached};
pub use write::profile_id;
+171 -42
View File
@@ -150,62 +150,142 @@ fn decode_text(bytes: &[u8]) -> std::borrow::Cow<'_, str> {
}
}
/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table.
pub fn parse_cube(text: &str) -> Result<LutFile, LutError> {
let mut title = String::new();
let (mut size3, mut size1) = (0usize, 0usize);
let (mut dmin, mut dmax) = ([0.0f32; 3], [1.0f32; 3]);
let mut rows: Vec<f32> = Vec::new();
for line in text.lines() {
// Only a quote at the beginning of a TITLE value protects '#'. Quotes inside an
// unquoted title's comment must not turn that comment into part of the title.
if let Some(quoted) = line
.trim_start()
.strip_prefix("TITLE")
.filter(|rest| rest.starts_with(char::is_whitespace))
.and_then(|rest| rest.trim_start().strip_prefix('"'))
/// A bounded, three-dimensional Cube profile with RGB rows in file order.
/// Unlike [`parse_cube`], this preserves the optional title and does not resample 1D curves.
#[derive(Clone, Debug, PartialEq)]
pub struct Cube3d {
pub title: Option<String>,
pub size: usize,
pub data: Vec<[f32; 3]>,
pub domain_min: [f32; 3],
pub domain_max: [f32; 3],
}
/// Read the simple 3D profile dialect used by camera-profile consumers.
///
/// `max_size` must be in 2..=256; the row budget is checked before each allocation.
/// Whole-line comments and literal titles are preserved. Unknown alphabetic directives
/// are ignored; 1D tables are refused. Finite domains are returned unchanged so the
/// consumer retains its existing policy for equal or reversed endpoints.
/// Numeric fields must consist of exactly three finite numbers (never silently skip tokens).
pub fn parse_cube_3d(text: &str, max_size: usize) -> Result<Cube3d, LutError> {
let p = read_cube(text, Some(max_size))?;
if p.size3 == 0 {
return err("not a 3D .cube LUT (no LUT_3D_SIZE)");
}
let expected = p.size3.checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?;
if p.rows.len() != expected {
return err(format!("expected {expected} entries, found {}", p.rows.len()));
}
Ok(Cube3d { title: p.title, size: p.size3, data: p.rows, domain_min: p.min, domain_max: p.max })
}
struct CubeRows {
title: Option<String>,
size3: usize,
size1: usize,
min: [f32; 3],
max: [f32; 3],
rows: Vec<[f32; 3]>,
}
fn cube_triplet(text: &str) -> Result<[f32; 3], LutError> {
let mut it = text.split_whitespace();
let mut row = [0.0; 3];
for value in &mut row {
*value = it.next().and_then(|s| s.parse::<f32>().ok()).filter(|v| v.is_finite()).ok_or_else(|| LutError(format!("bad line `{text}`")))?;
}
if it.next().is_some() {
return err(format!("bad line `{text}`"));
}
Ok(row)
}
fn read_cube(text: &str, profile_max: Option<usize>) -> Result<CubeRows, LutError> {
if profile_max.is_some_and(|n| !(2..=256).contains(&n)) {
return err("3D profile size limit must be in 2..=256");
}
let mut p = CubeRows { title: None, size3: 0, size1: 0, min: [0.0; 3], max: [1.0; 3], rows: Vec::new() };
let row_limit = profile_max.unwrap_or(256).checked_pow(3).ok_or_else(|| LutError("LUT size overflow".into()))?;
for raw in text.lines() {
let profile = profile_max.is_some();
let line = if profile { raw.trim() } else { raw.split('#').next().unwrap_or("").trim() };
// The standard dialect protects '#' inside quoted titles; profile titles are literal.
if !profile
&& let Some(quoted) =
raw.trim_start().strip_prefix("TITLE").filter(|r| r.starts_with(char::is_whitespace)).and_then(|r| r.trim_start().strip_prefix('"'))
{
let Some((value, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") };
title = value.to_string();
let Some((title, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") };
p.title = Some(title.to_string());
continue;
}
let line = line.split('#').next().unwrap_or("").trim();
if line.is_empty() {
if line.is_empty() || line.starts_with('#') {
continue;
}
let mut it = line.split_whitespace();
let key = it.next().unwrap_or("");
let nums = |it: std::str::SplitWhitespace<'_>| -> Result<[f32; 3], LutError> {
let v: Vec<f32> = it.filter_map(|s| s.parse().ok()).collect();
if v.len() == 3 { Ok([v[0], v[1], v[2]]) } else { err(format!("bad line `{line}`")) }
let (key, rest) = if profile {
["TITLE", "LUT_3D_SIZE", "LUT_1D_SIZE", "DOMAIN_MIN", "DOMAIN_MAX"]
.into_iter()
.find_map(|key| line.strip_prefix(key).map(|rest| (key, rest.trim())))
.unwrap_or_else(|| line.split_once(char::is_whitespace).unwrap_or((line, "")))
} else {
line.split_once(char::is_whitespace).map_or((line, ""), |(key, rest)| (key, rest.trim()))
};
match key {
"TITLE" => title = line[5..].trim().trim_matches('"').to_string(),
"LUT_3D_SIZE" => size3 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_3D_SIZE".into()))?,
"LUT_1D_SIZE" => size1 = it.next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?,
"DOMAIN_MIN" => dmin = nums(it)?,
"DOMAIN_MAX" => dmax = nums(it)?,
"LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" => {
let v: Vec<f32> = it.filter_map(|s| s.parse().ok()).collect();
if v.len() == 2 {
dmin = [v[0]; 3];
dmax = [v[1]; 3];
"TITLE" => p.title = Some(rest.trim_matches('"').to_string()),
"LUT_3D_SIZE" => {
let value = if profile { rest } else { rest.split_whitespace().next().unwrap_or("") };
p.size3 = value.parse().map_err(|_| LutError(format!("bad LUT_3D_SIZE `{value}`")))?;
let limit = profile_max.unwrap_or(MAX_SIZE);
if !(2..=limit).contains(&p.size3) {
return err(format!("LUT_3D_SIZE {} out of range", p.size3));
}
}
k if k.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.') => {
let mut v = vec![k.parse::<f32>().map_err(|_| LutError(format!("bad number in `{line}`")))?];
for value in it {
v.push(value.parse::<f32>().map_err(|_| LutError(format!("bad number in `{line}`")))?);
"LUT_1D_SIZE" => {
if profile {
return err("1D LUTs are not supported (only 3D .cube files)");
}
if v.len() != 3 {
return err(format!("bad data line `{line}`"));
p.size1 = rest.split_whitespace().next().and_then(|s| s.parse().ok()).ok_or_else(|| LutError("bad LUT_1D_SIZE".into()))?;
if p.size1 > row_limit {
return err("1D LUT exceeds row budget");
}
}
"DOMAIN_MIN" => p.min = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MIN".into()))?,
"DOMAIN_MAX" => p.max = cube_triplet(rest).map_err(|_| LutError("bad DOMAIN_MAX".into()))?,
"LUT_3D_INPUT_RANGE" | "LUT_1D_INPUT_RANGE" if !profile => {
let mut it = rest.split_whitespace();
let min = it.next().and_then(|s| s.parse::<f32>().ok());
let max = it.next().and_then(|s| s.parse::<f32>().ok());
if let (Some(min), Some(max), None) = (min, max, it.next()) {
p.min = [min; 3];
p.max = [max; 3];
}
}
_ => {
let numeric = key.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.');
let nonfinite = key.parse::<f32>().is_ok();
if numeric || nonfinite || (profile && !key.starts_with(|c: char| c.is_ascii_alphabetic())) {
let row = cube_triplet(line)?;
if p.rows.len() >= row_limit {
return err("LUT exceeds row budget");
}
// Explicit fallible growth prevents capacity growth past the configured budget.
if p.rows.len() == p.rows.capacity() {
let grow = (p.rows.capacity().max(1)).min(row_limit - p.rows.len());
p.rows.try_reserve_exact(grow).map_err(|_| LutError("LUT allocation failed".into()))?;
}
p.rows.push(row);
}
rows.extend(v);
}
// Unknown keywords (e.g. Resolve's LUT_IN_VIDEO_RANGE) are ignored, as the spec allows.
_ => {}
}
}
Ok(p)
}
/// Adobe / Resolve `.cube`. 1D LUTs (`LUT_1D_SIZE`) are expanded to a 33³ table.
pub fn parse_cube(text: &str) -> Result<LutFile, LutError> {
let p = read_cube(text, None)?;
let (title, size3, size1, dmin, dmax) = (p.title.unwrap_or_default(), p.size3, p.size1, p.min, p.max);
let rows: Vec<f32> = p.rows.into_iter().flatten().collect();
let domain_ok = (0..3).all(|k| dmin[k] < dmax[k] && dmin[k].is_finite() && dmax[k].is_finite());
if (size3 > 0 || size1 >= 2) && !domain_ok {
return err(format!("DOMAIN_MIN {:?} must lie below DOMAIN_MAX {:?}", dmin, dmax));
@@ -1386,3 +1466,52 @@ LUT_3D_SIZE 2
assert_eq!(back.data, [0.0, 0.25, 1.0].repeat(8));
}
}
#[cfg(test)]
mod profile_cube_tests {
use super::*;
fn cube(prefix: &str) -> String {
format!("{prefix}\nLUT_3D_SIZE 2\n{}", "0.25 0.5 0.75\n".repeat(8))
}
#[test]
fn profile_preserves_optional_literal_title_and_domains() {
let no_title = parse_cube_3d(&cube(""), 256).unwrap();
assert_eq!(no_title.title, None);
let value = parse_cube_3d(&cube("TITLE \"look # literal\"\nDOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1"), 256).unwrap();
assert_eq!(value.title.as_deref(), Some("look # literal"));
assert_eq!(value.domain_min, [1.0, 0.0, 2.0]);
assert_eq!(value.domain_max, [0.0, 0.0, 1.0]);
assert_eq!(value.data, vec![[0.25, 0.5, 0.75]; 8]);
assert_eq!(parse_cube_3d(&cube("TITLE \"\""), 256).unwrap().title.as_deref(), Some(""));
assert!(parse_cube(&cube("DOMAIN_MIN 1 0 2\nDOMAIN_MAX 0 0 1")).is_err());
}
#[test]
fn profile_limits_are_checked_without_large_allocations() {
assert!(parse_cube_3d("LUT_3D_SIZE 256", 256).unwrap_err().0.contains("16777216"));
assert!(parse_cube_3d("LUT_3D_SIZE 257", 256).unwrap_err().0.contains("out of range"));
assert!(parse_cube_3d("LUT_3D_SIZE 256", 129).unwrap_err().0.contains("out of range"));
assert!(parse_cube_3d(&cube(""), usize::MAX).is_err());
assert!(parse_cube_3d(&cube(""), 1).is_err());
assert!(parse_cube_3d(&format!("{}0 0 0\n", cube("")), 2).unwrap_err().0.contains("budget"));
assert!(parse_cube_3d("LUT_1D_SIZE 2\n0 0 0\n1 1 1", 256).is_err());
}
#[test]
fn every_numeric_token_must_be_valid_and_finite() {
for row in ["0 ignored 0 0", "0 0 0 ignored", "NaN 0 0", "inf 0 0", "0 0 -inf", "1e39 0 0", "0 0", "0 0 0 0"] {
assert!(parse_cube_3d(&format!("LUT_3D_SIZE 2\n{row}\n{}", "0 0 0\n".repeat(7)), 256).is_err(), "{row}");
assert!(parse_cube_3d(&cube(&format!("DOMAIN_MIN {row}")), 256).is_err(), "{row}");
}
}
#[test]
fn standard_reader_keeps_quoted_comments_and_1d_expansion() {
assert_eq!(parse_cube(&cube("TITLE \"look # preserved\" # comment")).unwrap().title, "look # preserved");
let value = parse_cube("LUT_1D_SIZE 2\n0 0 0\n1 1 1").unwrap();
assert_eq!(value.size, 33);
assert!(parse_cube_3d(&cube("DOMAIN_MIN 0 0 0 # not a whole-line comment"), 256).is_err());
}
}
+142
View File
@@ -0,0 +1,142 @@
//! Bounded encoded ICC layout normalization, originating in LightCraft.
use std::collections::BTreeMap;
/// Zero the creation date, align tag payloads to four bytes, preserve shared payloads,
/// update the profile length and clear its stale profile ID. No colour tags are interpreted.
///
/// This is a layout operation, not profile validation. Malformed tables or layouts exceeding
/// 4096 tags or 64 MiB output are returned unchanged except for the creation date. The owned
/// input is reused on failure, so rejected input does not require an additional full copy.
/// Consumers needing rejection of invalid profiles must separately use `Profile::parse`.
pub fn normalize_for_embedding(mut bytes: Vec<u8>) -> Vec<u8> {
if let Some(date) = bytes.get_mut(24..36) {
date.fill(0);
}
align(&bytes).unwrap_or(bytes)
}
const MAX_BYTES: usize = 64 * 1024 * 1024;
const MAX_TAGS: usize = 4096;
fn be(bytes: &[u8], at: usize) -> Option<usize> {
let s = bytes.get(at..at.checked_add(4)?)?;
usize::try_from(u32::from_be_bytes([s[0], s[1], s[2], s[3]])).ok()
}
fn align(bytes: &[u8]) -> Option<Vec<u8>> {
if bytes.len() > MAX_BYTES {
return None;
}
let count = be(bytes, 128)?;
if count > MAX_TAGS {
return None;
}
let end = 132usize.checked_add(count.checked_mul(12)?)?;
bytes.get(..end)?;
let mut placements = BTreeMap::new();
let mut offsets = Vec::new();
offsets.try_reserve_exact(count).ok()?;
let mut size = end;
for index in 0..count {
let entry = 132 + index * 12;
let off = be(bytes, entry + 4)?;
let len = be(bytes, entry + 8)?;
if off < end {
return None;
}
bytes.get(off..off.checked_add(len)?)?;
let new_off = if let Some(&placed) = placements.get(&(off, len)) {
placed
} else {
let aligned = size.checked_add(3)? & !3;
size = aligned.checked_add(len)?;
if size > MAX_BYTES {
return None;
}
placements.insert((off, len), aligned);
aligned
};
offsets.push(new_off);
}
size = size.checked_add(3)? & !3;
if size > MAX_BYTES {
return None;
}
let mut out = Vec::new();
out.try_reserve_exact(size).ok()?;
out.resize(size, 0);
out[..end].copy_from_slice(&bytes[..end]);
for ((off, len), target) in placements {
out[target..target + len].copy_from_slice(&bytes[off..off + len]);
}
for (index, off) in offsets.into_iter().enumerate() {
let field = 132 + index * 12 + 4;
out[field..field + 4].copy_from_slice(&u32::try_from(off).ok()?.to_be_bytes());
}
out[..4].copy_from_slice(&u32::try_from(size).ok()?.to_be_bytes());
out[84..100].fill(0);
Some(out)
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
#[path = "normalize_legacy_tests.rs"]
mod legacy;
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use super::*;
#[test]
fn matches_frozen_writer_and_preserves_aliases() {
for count in 0..20usize {
let end = 132 + count * 12;
let mut bytes = vec![17; end + 31];
bytes[128..132].copy_from_slice(&(count as u32).to_be_bytes());
for index in 0..count {
let field = 132 + index * 12 + 4;
let off = end + index % 3;
bytes[field..field + 4].copy_from_slice(&(off as u32).to_be_bytes());
bytes[field + 4..field + 8].copy_from_slice(&7u32.to_be_bytes());
}
let expected = legacy::align_tags(&legacy::fixed_date(bytes.clone()));
let result = normalize_for_embedding(bytes);
assert_eq!(result, expected);
assert_eq!(normalize_for_embedding(result.clone()), result);
}
}
#[test]
fn real_profile_preserves_color_tags_and_legacy_bytes() {
let encoded = crate::Builtin::Srgb.profile().clone().with_encoded_bytes();
let input = encoded.to_bytes().to_vec();
let normalized = normalize_for_embedding(input.clone());
assert_eq!(normalized, legacy::align_tags(&legacy::fixed_date(input)));
assert_eq!(&normalized[24..36], &[0; 12]);
assert_eq!(&normalized[84..100], &[0; 16]);
assert!(crate::Profile::parse(&normalized).is_ok());
for index in 0..be(&normalized, 128).unwrap() {
assert_eq!(be(&normalized, 136 + index * 12).unwrap() % 4, 0);
}
}
#[test]
fn hostile_tables_return_owned_input_without_expansion() {
for len in 0..132 {
let bytes = vec![1; len];
assert_eq!(normalize_for_embedding(bytes.clone()), legacy::fixed_date(bytes));
}
for count in [4097u32, u32::MAX] {
let mut bytes = vec![1; 144];
bytes[128..132].copy_from_slice(&count.to_be_bytes());
assert_eq!(normalize_for_embedding(bytes.clone()), legacy::fixed_date(bytes));
}
let mut bytes = vec![1; 144];
bytes[128..132].copy_from_slice(&1u32.to_be_bytes());
bytes[136..140].copy_from_slice(&u32::MAX.to_be_bytes());
bytes[140..144].copy_from_slice(&u32::MAX.to_be_bytes());
assert_eq!(normalize_for_embedding(bytes.clone()), legacy::fixed_date(bytes));
}
}
+43
View File
@@ -0,0 +1,43 @@
//! Encoded ICC normalization extracted from LightCraft.
pub(super) fn align_tags(b: &[u8]) -> Vec<u8> {
let be = |i: usize| b.get(i..i + 4).map(|s| u32::from_be_bytes([s[0], s[1], s[2], s[3]]) as usize);
let Some(n) = be(128) else { return b.to_vec() };
let table_end = 132 + 12 * n;
let tags: Option<Vec<(usize, usize)>> = (0..n).map(|t| Some((be(132 + 12 * t + 4)?, be(132 + 12 * t + 8)?))).collect();
let Some(tags) = tags.filter(|t| table_end <= b.len() && t.iter().all(|&(o, l)| o >= table_end && o + l <= b.len())) else {
return b.to_vec();
};
let mut out = b[..table_end].to_vec();
let mut placed: Vec<((usize, usize), usize)> = Vec::new();
for (t, &(off, len)) in tags.iter().enumerate() {
let new_off = match placed.iter().find(|(k, _)| *k == (off, len)) {
// tags sharing data keep sharing it
Some(&(_, o)) => o,
None => {
while !out.len().is_multiple_of(4) {
out.push(0);
}
let o = out.len();
out.extend_from_slice(&b[off..off + len]);
placed.push(((off, len), o));
o
}
};
out[132 + 12 * t + 4..132 + 12 * t + 8].copy_from_slice(&(new_off as u32).to_be_bytes());
}
while !out.len().is_multiple_of(4) {
out.push(0);
}
let size = out.len() as u32;
out[0..4].copy_from_slice(&size.to_be_bytes());
// the profile ID (MD5) would no longer match: clear it (= "not computed")
out[84..100].fill(0);
out
}
pub(super) fn fixed_date(mut b: Vec<u8>) -> Vec<u8> {
if let Some(date) = b.get_mut(24..36) {
date.fill(0);
}
b
}
+82
View File
@@ -0,0 +1,82 @@
//! Sony Imageworks SPI3D and SPIMTX text readers, extracted from EffectCraft.
//!
//! These preserve the source reader's format tolerance. SPI3D sample values are not
//! clamped or sanitized; rendering policy belongs to the caller. Duplicate coordinates
//! retain their last value, and the total row count must match the lattice volume.
/// Maximum lattice side accepted by the source reader.
pub const MAX_SIDE: usize = 129;
/// Row-major homogeneous transform with a separate normalized code-value offset.
#[derive(Clone, Debug, PartialEq)]
pub struct Matrix {
pub matrix: [f64; 16],
pub offset: [f64; 4],
}
/// A cubic RGB table, red coordinate varying fastest.
#[derive(Clone, Debug, PartialEq)]
pub struct Lattice {
pub size: usize,
pub values: Vec<[f32; 3]>,
}
/// Read exactly twelve finite SPIMTX numbers, dividing the offset column by 65535.
/// Parsing uses a fixed array, not storage proportional to an oversized input.
pub fn parse_spimtx(text: &str) -> Option<Matrix> {
let mut words = text.split_whitespace();
let mut v = [0.0f64; 12];
for value in &mut v {
*value = words.next()?.parse().ok()?;
if !value.is_finite() {
return None;
}
}
if words.next().is_some() {
return None;
}
let mut matrix = [0.0; 16];
let mut offset = [0.0; 4];
for (r, row) in v.as_chunks::<4>().0.iter().enumerate() {
matrix[r * 4..r * 4 + 3].copy_from_slice(&row[..3]);
offset[r] = row[3] / 65535.0;
}
matrix[15] = 1.0;
Some(Matrix { matrix, offset })
}
/// Read a SPI3D table with a bounded 2..=129 cubic lattice and exactly six fields per row.
/// A `SPILUT` prefix is accepted and the second header line is ignored, as in EffectCraft.
pub fn parse_spi3d(text: &str) -> Option<Lattice> {
let mut lines = text.lines().map(str::trim).filter(|l| !l.is_empty());
if !lines.next()?.starts_with("SPILUT") {
return None;
}
lines.next()?;
let mut sizes = lines.next()?.split_whitespace();
let n: usize = sizes.next()?.parse().ok()?;
let g: usize = sizes.next()?.parse().ok()?;
let b: usize = sizes.next()?.parse().ok()?;
if sizes.next().is_some() || g != n || b != n || !(2..=MAX_SIDE).contains(&n) {
return None;
}
let count = n * n * n;
let mut values = vec![[0.0f32; 3]; count];
let mut seen = 0usize;
for line in lines {
if seen == count {
return None;
}
let mut fields = line.split_whitespace();
let r: usize = fields.next()?.parse().ok()?;
let g: usize = fields.next()?.parse().ok()?;
let b: usize = fields.next()?.parse().ok()?;
let color = [fields.next()?.parse().ok()?, fields.next()?.parse().ok()?, fields.next()?.parse().ok()?];
if fields.next().is_some() || r >= n || g >= n || b >= n {
return None;
}
*values.get_mut(r + g * n + b * n * n)? = color;
seen += 1;
}
(seen == count).then_some(Lattice { size: n, values })
}
+59
View File
@@ -0,0 +1,59 @@
//! Sony Imageworks SPI1D tables, preserving EffectCraft's permissive header syntax.
//!
//! Unknown header lines and trailing text after the closing brace are ignored. Table
//! samples are not clamped or sanitized: callers own their numerical rendering policy.
//! The declared length is validated before storing samples; surplus samples are refused.
/// Largest accepted table (the existing EffectCraft interpolation limit).
pub const MAX_ENTRIES: usize = 1 << 24;
/// A one-dimensional RGB lookup table and its common input domain.
#[derive(Clone, Debug, PartialEq)]
pub struct Table {
pub values: Vec<[f32; 3]>,
pub domain: [f32; 2],
}
/// Parse a SPI1D table with one or three components, expanding monochrome samples to RGB.
/// No allocation is based only on the claimed length. At most `MAX_ENTRIES` RGB entries
/// are retained. Missing final braces remain accepted, as in the source reader.
pub fn parse(text: &str) -> Option<Table> {
let (mut from, mut len, mut comps) = ([0.0f32, 1.0], 0usize, 1usize);
let mut values = Vec::new();
let mut body = false;
let mut sample = [0.0; 3];
let mut channel = 0;
for line in text.lines().map(str::trim).filter(|l| !l.is_empty()) {
if body {
if line.starts_with('}') {
break;
}
for t in line.split_whitespace() {
if values.len() == len {
return None;
}
sample[channel] = t.parse().ok()?;
channel += 1;
if channel == comps {
values.push(if comps == 1 { [sample[0]; 3] } else { sample });
channel = 0;
}
}
continue;
}
let mut t = line.split_whitespace();
match t.next()? {
"{" => {
if !(2..=MAX_ENTRIES).contains(&len) || !matches!(comps, 1 | 3) || from[1] <= from[0] || from.iter().any(|v| !v.is_finite()) {
return None;
}
body = true;
}
"From" => from = [t.next()?.parse().ok()?, t.next()?.parse().ok()?],
"Length" => len = t.next()?.parse().ok()?,
"Components" => comps = t.next()?.parse().ok()?,
_ => {}
}
}
(body && values.len() == len && channel == 0).then_some(Table { values, domain: from })
}
+195
View File
@@ -0,0 +1,195 @@
//! Bounded ICC v2.1 matrix/TRC display-profile writer, extracted from TextureCraft.
//! Colour-space selection and transfer-curve sampling remain caller policy.
const D50_XYZ: [f64; 3] = [0.9642, 1.0, 0.8249];
/// One common red/green/blue transfer curve.
#[derive(Clone, Copy)]
pub enum ToneCurve<'a> {
/// ICC gamma 1.0 (`curv` with one u8Fixed8 entry).
Linear,
/// Already quantized samples, inclusive endpoints, between 2 and 4096 entries.
Samples(&'a [u16]),
}
/// Parameters for an ICC v2.1 RGB display profile.
pub struct MatrixProfile<'a> {
/// RGB-to-XYZ matrix already adapted to PCS D50, in row-major order.
pub matrix: [[f64; 3]; 3],
/// Common RGB transfer curve; its storage is shared by all three tags.
pub curve: ToneCurve<'a>,
/// ASCII description, at most 1024 bytes with no embedded NUL.
pub description: &'a str,
/// ASCII copyright text, at most 1024 bytes with no embedded NUL.
pub copyright: &'a str,
/// Raw ICC date fields: year, month, day, hour, minute, second.
pub date: [u16; 6],
}
fn s15f16(v: f64) -> [u8; 4] {
((v * 65536.0).round().clamp(f64::from(i32::MIN), f64::from(i32::MAX)) as i32).to_be_bytes()
}
fn xyz_tag(v: [f64; 3]) -> Vec<u8> {
let mut b = b"XYZ \0\0\0\0".to_vec();
for c in v {
b.extend_from_slice(&s15f16(c));
}
b
}
fn text_tag(text: &str) -> Vec<u8> {
let mut b = b"text\0\0\0\0".to_vec();
b.extend_from_slice(text.as_bytes());
b.push(0);
b
}
/// `textDescriptionType` with the ASCII description only.
fn desc_tag(text: &str) -> Vec<u8> {
let mut b = b"desc\0\0\0\0".to_vec();
b.extend_from_slice(&(text.len() as u32 + 1).to_be_bytes());
b.extend_from_slice(text.as_bytes());
b.push(0);
// No Unicode text and no Macintosh ScriptCode text: empty fields and the 67-byte filler.
b.extend_from_slice(&[0; 4 + 4 + 2 + 1 + 67]);
b
}
fn curve_tag(curve: ToneCurve<'_>) -> Vec<u8> {
let mut b = b"curv\0\0\0\0".to_vec();
match curve {
ToneCurve::Linear => {
b.extend_from_slice(&1u32.to_be_bytes());
b.extend_from_slice(&0x0100u16.to_be_bytes());
}
ToneCurve::Samples(samples) => {
b.extend_from_slice(&(samples.len() as u32).to_be_bytes());
for sample in samples {
b.extend_from_slice(&sample.to_be_bytes());
}
}
}
b
}
/// Write a deterministic ICC v2.1 RGB display profile with one shared RGB TRC.
///
/// Returns `None` for nonfinite matrix values, descriptions longer than 1024 ASCII
/// bytes or containing NUL, or sampled curves outside 2..=4096 entries. These caps
/// bound every allocation independently of caller input. Matrix values outside the
/// signed 15.16 range saturate, matching the legacy exporter. The caller owns colour
/// adaptation, transfer sampling, names and timestamp policy.
pub fn matrix_profile(params: &MatrixProfile<'_>) -> Option<Vec<u8>> {
let valid_text = |s: &str| s.len() <= 1024 && s.is_ascii() && !s.contains('\0');
if !valid_text(params.description)
|| !valid_text(params.copyright)
|| !params.matrix.iter().flatten().all(|v| v.is_finite())
|| matches!(params.curve, ToneCurve::Samples(s) if !(2..=4096).contains(&s.len()))
{
return None;
}
let m = params.matrix;
let column = |c: usize| [m[0][c], m[1][c], m[2][c]];
let curve = curve_tag(params.curve);
// The green and blue curves are the red one's entry.
let tags: [(&[u8; 4], Vec<u8>); 9] = [
(b"desc", desc_tag(params.description)),
(b"cprt", text_tag(params.copyright)),
(b"wtpt", xyz_tag(D50_XYZ)),
(b"rXYZ", xyz_tag(column(0))),
(b"gXYZ", xyz_tag(column(1))),
(b"bXYZ", xyz_tag(column(2))),
(b"rTRC", curve.clone()),
(b"gTRC", curve.clone()),
(b"bTRC", curve),
];
// Header (128) + tag count (4) + 12 per tag, then the tag data, each tag padded to 4 bytes.
let table_end = 128 + 4 + 12 * tags.len();
let mut offsets = Vec::new();
let mut data: Vec<u8> = Vec::new();
for (i, (_, bytes)) in tags.iter().enumerate() {
let shared = i >= 7;
let at = if shared { offsets.get(6).copied().unwrap_or(0) } else { table_end + data.len() };
offsets.push(at);
if !shared {
data.extend_from_slice(bytes);
while !data.len().is_multiple_of(4) {
data.push(0);
}
}
}
let total = table_end + data.len();
let mut out = vec![0u8; 128];
out.get_mut(0..4)?.copy_from_slice(&(total as u32).to_be_bytes());
out.get_mut(8..12)?.copy_from_slice(&[2, 0x10, 0, 0]); // version 2.1.0
out.get_mut(12..16)?.copy_from_slice(b"mntr");
out.get_mut(16..20)?.copy_from_slice(b"RGB ");
out.get_mut(20..24)?.copy_from_slice(b"XYZ ");
// Serialize caller-supplied ICC date fields; no calendar arithmetic.
for (i, value) in params.date.iter().enumerate() {
out.get_mut(24 + i * 2..26 + i * 2)?.copy_from_slice(&value.to_be_bytes());
}
out.get_mut(36..40)?.copy_from_slice(b"acsp");
for (k, v) in D50_XYZ.iter().enumerate() {
out.get_mut(68 + 4 * k..72 + 4 * k)?.copy_from_slice(&s15f16(*v));
}
out.extend_from_slice(&(tags.len() as u32).to_be_bytes());
for ((sig, bytes), at) in tags.iter().zip(&offsets) {
out.extend_from_slice(*sig);
out.extend_from_slice(&(*at as u32).to_be_bytes());
out.extend_from_slice(&(bytes.len() as u32).to_be_bytes());
}
out.extend_from_slice(&data);
Some(out)
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used)]
use super::*;
fn params() -> MatrixProfile<'static> {
MatrixProfile {
matrix: [[0.4361, 0.3851, 0.1431], [0.2225, 0.7169, 0.0606], [0.0139, 0.0971, 0.7141]],
curve: ToneCurve::Linear,
description: "v2",
copyright: "free",
date: [2026, 1, 1, 0, 0, 0],
}
}
#[test]
fn valid_profile_parses_and_aliases_curves() {
for curve in [ToneCurve::Linear, ToneCurve::Samples(&[0, 32768, 65535])] {
let bytes = matrix_profile(&MatrixProfile { curve, ..params() }).unwrap();
let p = crate::Profile::parse(&bytes).unwrap();
assert_eq!(p.version, (2, 0x10));
assert_eq!(p.description, "v2");
assert_eq!(&bytes[24..28], &[7, 234, 0, 1]);
assert_eq!(&bytes[208..216], &bytes[220..228]);
assert_eq!(&bytes[208..216], &bytes[232..240]);
assert_eq!(bytes, matrix_profile(&MatrixProfile { curve, ..params() }).unwrap());
}
}
#[test]
fn bounded_inputs_reject_before_serializing() {
let long = "x".repeat(1025);
for description in [long.as_str(), "bad\0name", "non-ascii é"] {
assert!(matrix_profile(&MatrixProfile { description, ..params() }).is_none());
}
for samples in [&[][..], &[0][..], &vec![0; 4097][..]] {
assert!(matrix_profile(&MatrixProfile { curve: ToneCurve::Samples(samples), ..params() }).is_none());
}
for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
let mut p = params();
p.matrix[0][0] = value;
assert!(matrix_profile(&p).is_none());
}
let samples = vec![0; 4096];
let name = "x".repeat(1024);
let bytes = matrix_profile(&MatrixProfile { description: &name, copyright: &name, curve: ToneCurve::Samples(&samples), ..params() }).unwrap();
assert!(bytes.len() < 12 * 1024);
}
}
+46
View File
@@ -0,0 +1,46 @@
use craft_cms::sony_lut::{MAX_SIDE, parse_spi3d, parse_spimtx};
#[test]
fn spimtx_is_finite_exact_and_scales_offsets() {
let m = parse_spimtx("2 0 0 6553.5\n0 3 0 -65535\n0 0 4 0").unwrap();
assert_eq!(m.matrix, [2.0, 0.0, 0.0, 0.0, 0.0, 3.0, 0.0, 0.0, 0.0, 0.0, 4.0, 0.0, 0.0, 0.0, 0.0, 1.0]);
assert_eq!(m.offset, [0.1, -1.0, 0.0, 0.0]);
for bad in ["", "1 0 0 0", "NaN 0 0 0 0 1 0 0 0 0 1 0", "inf 0 0 0 0 1 0 0 0 0 1 0", "1 0 0 0 0 1 0 0 0 0 1 0 9"] {
assert!(parse_spimtx(bad).is_none());
}
}
fn table() -> String {
let mut text = String::from("SPILUT 1.0\n3 3\n2 2 2\n");
// Reverse row order to prove coordinates, rather than row order, determine storage.
for i in (0..8).rev() {
let (r, g, b) = (i & 1, (i >> 1) & 1, i >> 2);
text.push_str(&format!("{r} {g} {b} {r} {g} {b}\n"));
}
text
}
#[test]
fn spi3d_red_fast_coordinates_and_limits() {
let text = table();
let lut = parse_spi3d(&text).unwrap();
assert_eq!(lut.size, 2);
for (i, row) in lut.values.iter().enumerate() {
assert_eq!(*row, [(i & 1) as f32, ((i >> 1) & 1) as f32, (i >> 2) as f32]);
}
assert!(parse_spi3d(&format!("{text}0 0 0 0 0 0\n")).is_none());
assert!(parse_spi3d("SPILUT 1.0\n3 3\n2 2 2\n0 0 0 0 0 0\n").is_none());
assert!(parse_spi3d(&format!("SPILUT 1.0\n3 3\n{0} {0} {0}\n", MAX_SIDE + 1)).is_none());
assert!(parse_spi3d(&text.replace("2 2 2", "2 2 3")).is_none());
assert!(parse_spi3d(&text.replace("0 0 0 0 0 0", "2 0 0 0 0 0")).is_none());
assert!(parse_spi3d(&text.replace("0 0 0 0 0 0", "0 0 0 0 0 0 extra")).is_none());
}
#[test]
fn inherited_tolerance_is_explicit() {
let text = "SPILUT alternative\nignored\n2 2 2\n".to_owned() + &"0 0 0 -1 2 NaN\n".repeat(8);
let lut = parse_spi3d(&text).unwrap();
assert_eq!(&lut.values[0][..2], &[-1.0, 2.0]);
assert!(lut.values[0][2].is_nan());
assert_eq!(lut.values[1], [0.0; 3]);
}
+32
View File
@@ -0,0 +1,32 @@
use craft_cms::spi1d::{MAX_ENTRIES, Table, parse};
#[test]
fn monochrome_and_three_component_samples_preserve_domain() {
assert_eq!(
parse("Version 1\nFrom -1 2\nLength 2\nComponents 1\n{\n-2\n3\n}"),
Some(Table { values: vec![[-2.0; 3], [3.0; 3]], domain: [-1.0, 2.0] })
);
assert_eq!(
parse("Length 2\nComponents 3\n{\n0 1\n2 3 4 5\n}"),
Some(Table { values: vec![[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]], domain: [0.0, 1.0] })
);
}
#[test]
fn legacy_header_and_eof_tolerance_is_preserved() {
let table = parse("Ignored arbitrary header\nLength 2 extra\n{\n0 1");
assert!(table.is_some());
assert_eq!(table, parse("Length 2\n{\n0 1\n} ignored\ninvalid trailing text"));
assert!(parse("Length 2\n{\nNaN inf\n}").is_some(), "sample sanitization belongs to the caller");
}
#[test]
fn invalid_headers_partial_and_surplus_tables_are_refused() {
for header in ["Length 1", "Length 2\nComponents 2", "Length 2\nFrom 2 1", "Length 2\nFrom NaN 1"] {
assert!(parse(&format!("{header}\n{{\n0 1\n}}")).is_none());
}
assert!(parse(&format!("Length {}\n{{\n0 1", MAX_ENTRIES + 1)).is_none());
assert!(parse("Length 2\nComponents 3\n{\n0 1 2 3 4").is_none());
assert!(parse("Length 2\n{\n0 1 2").is_none());
assert!(parse("Length 2\n{\n").is_none());
}
+15
View File
@@ -0,0 +1,15 @@
[package]
name = "craft-control-protocol"
description = "Authenticated bounded JSON-lines control protocol"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
serde_json = { workspace = true }
subtle = "2"
+234
View File
@@ -0,0 +1,234 @@
//! Authenticated JSON-lines request framing and dispatch for local control services.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
use serde_json::{Value, json};
use std::io::{BufRead, BufReader, Read, Write};
use subtle::ConstantTimeEq;
/// Whether a session token has the control protocol's 64 hexadecimal character form.
pub fn valid_token(token: &str) -> bool {
token.len() == 64 && token.bytes().all(|b| b.is_ascii_hexdigit())
}
/// Encode one client request, including its terminating newline.
pub fn request_line(id: u64, token: Option<&str>, method: &str, params: &Value) -> String {
format!("{}\n", json!({"id": id, "token": token, "method": method, "params": params}))
}
/// Extract the result or application error from a parsed control reply.
pub fn reply_result(reply: &Value) -> Result<Value, String> {
if reply.get("ok").and_then(Value::as_bool) == Some(true) {
Ok(reply.get("result").cloned().unwrap_or(Value::Null))
} else {
Err(reply.get("error").and_then(Value::as_str).unwrap_or("error").to_owned())
}
}
/// One parsed line.
enum Line {
Request {
id: Value,
method: String,
params: Value,
},
Blank,
/// Not a request: reply with this error and close the connection.
Reject(String),
}
fn parse_line(bytes: &[u8], token: &str) -> Line {
let Ok(text) = std::str::from_utf8(bytes) else {
return Line::Reject("not a control request (invalid UTF-8); closing the connection".into());
};
let text = text.trim();
if text.is_empty() {
return Line::Blank;
}
let msg = match serde_json::from_str::<Value>(text) {
Ok(v) => v,
Err(e) => return Line::Reject(format!("bad JSON ({e}); expected one JSON request object per line; closing the connection")),
};
let Some(obj) = msg.as_object() else {
return Line::Reject("not a control request (expected a JSON object); closing the connection".into());
};
let Some(method) = obj.get("method").and_then(Value::as_str) else {
return Line::Reject("not a control request (missing string `method`); closing the connection".into());
};
// Authenticate every request before constructing or enqueueing a ControlRequest. A valid
// earlier request must not authorize a later one with a missing or incorrect credential.
let authenticated = obj.get("token").and_then(Value::as_str).is_some_and(|supplied| bool::from(supplied.as_bytes().ct_eq(token.as_bytes())));
if !authenticated {
return Line::Reject("control authentication failed; closing the connection".into());
}
Line::Request {
id: obj.get("id").cloned().unwrap_or(Value::Null),
method: method.to_string(),
params: obj.get("params").cloned().unwrap_or(json!({})),
}
}
#[derive(Debug, PartialEq, Eq)]
pub enum ReadFailure {
Timeout,
Rejected(String),
}
/// Read one `\n`-terminated line of at most `max` bytes into `buf` (newline stripped).
/// `Ok(false)` at end of stream; `Err` when the line is too long or the read fails.
pub fn read_bounded_line(reader: &mut impl BufRead, buf: &mut Vec<u8>, max: usize) -> Result<bool, ReadFailure> {
buf.clear();
// `take` bounds the bytes buffered for one line: a peer can't make us allocate more than `max`.
let limit = u64::try_from(max).unwrap_or(u64::MAX).saturating_add(1);
let n = reader.take(limit).read_until(b'\n', buf).map_err(|e| match e.kind() {
std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock => ReadFailure::Timeout,
_ => ReadFailure::Rejected(e.to_string()),
})?;
if n == 0 {
return Ok(false);
}
if buf.last() == Some(&b'\n') {
buf.pop();
if buf.last() == Some(&b'\r') {
buf.pop();
}
return Ok(true);
}
if buf.len() > max {
return Err(ReadFailure::Rejected(format!("request line longer than {max} bytes; closing the connection")));
}
// Last line without a newline, then end of stream.
Ok(true)
}
/// Serve requests from `read`, answering on `out`, until the peer closes, sends something that is
/// not a request, or `handle` returns `None` (app gone). `handle` gets `(method, params)` and
/// returns the reply object (the request's `id` is added here).
pub fn serve_lines<R: Read, W: Write>(read: R, mut out: W, token: &str, max_line: usize, mut handle: impl FnMut(String, Value) -> Option<Value>) {
let mut reader = BufReader::new(read);
let mut buf = Vec::new();
loop {
let reject = match read_bounded_line(&mut reader, &mut buf, max_line) {
Ok(false) => return,
// Expiration is not a request: a pooled client must see EOF and reconnect, rather
// than read a stale error reply. A partial line must not be parsed on timeout.
Err(ReadFailure::Timeout) => return,
Err(ReadFailure::Rejected(e)) => e,
Ok(true) => match parse_line(&buf, token) {
Line::Blank => continue,
Line::Reject(e) => e,
Line::Request { id, method, params } => {
let Some(mut r) = handle(method, params) else { return };
if let Some(o) = r.as_object_mut() {
o.insert("id".into(), id);
}
if writeln!(out, "{r}").and_then(|()| out.flush()).is_err() {
return;
}
continue;
}
},
};
let _ = writeln!(out, "{}", json!({"ok": false, "error": reject})).and_then(|()| out.flush());
return;
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn rejects_bad_request_shapes_without_dispatching() {
for input in [&b"\xff\n"[..], &b"[]\n"[..], &b"{}\n"[..]] {
let mut out = Vec::new();
serve_lines(input, &mut out, "secret", 1024, |_, _| panic!("must not dispatch"));
let reply: Value = serde_json::from_slice(&out).unwrap();
assert_eq!(reply["ok"], false);
assert!(reply["error"].as_str().unwrap().contains("closing the connection"));
}
}
#[test]
fn timeout_after_partial_line_never_dispatches() {
struct Partial(bool);
impl Read for Partial {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
if !self.0 {
self.0 = true;
buf[..2].copy_from_slice(b"{\"");
return Ok(2);
}
Err(std::io::ErrorKind::TimedOut.into())
}
}
let mut out = Vec::new();
serve_lines(Partial(false), &mut out, "secret", 1024, |_, _| panic!("must not dispatch"));
assert!(out.is_empty());
}
#[test]
fn line_boundaries_and_closed_dispatch_are_preserved() {
let mut buf = Vec::new();
assert_eq!(read_bounded_line(&mut &b"abc\r\n"[..], &mut buf, 4), Ok(true));
assert_eq!(buf, b"abc");
assert_eq!(read_bounded_line(&mut &b"abcd\n"[..], &mut buf, 4), Ok(true));
assert_eq!(buf, b"abcd");
let mut out = Vec::new();
serve_lines(&b"{\"method\":\"x\",\"token\":\"s\"}\n"[..], &mut out, "s", 1024, |_, _| None);
assert!(out.is_empty());
}
#[test]
fn output_failure_stops_following_requests() {
struct Closed;
impl Write for Closed {
fn write(&mut self, _: &[u8]) -> std::io::Result<usize> {
Err(std::io::ErrorKind::BrokenPipe.into())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
let input = b"{\"method\":\"x\",\"token\":\"s\"}\n{\"method\":\"y\",\"token\":\"s\"}\n";
let mut calls = 0;
serve_lines(&input[..], Closed, "s", 1024, |_, _| {
calls += 1;
Some(json!({"ok": true}))
});
assert_eq!(calls, 1);
}
#[test]
fn client_request_and_reply_keep_optional_fields() {
let line = request_line(7, None, "status", &json!({"a": 1}));
assert_eq!(serde_json::from_str::<Value>(&line).unwrap(), json!({"id": 7, "token": null, "method": "status", "params": {"a": 1}}));
assert_eq!(reply_result(&json!({"ok": true})).unwrap(), Value::Null);
assert_eq!(reply_result(&json!({"ok": false, "error": "busy"})).unwrap_err(), "busy");
}
#[test]
fn validates_session_token_shape() {
assert!(valid_token(&"aB09".repeat(16)));
assert!(!valid_token(&"a".repeat(63)));
assert!(!valid_token(&"g".repeat(64)));
}
#[test]
fn authenticates_each_request_and_closes_after_failure() {
let input =
b"{\"id\":1,\"method\":\"first\",\"token\":\"secret\"}\n{\"id\":2,\"method\":\"second\"}\n{\"method\":\"third\",\"token\":\"secret\"}\n";
let mut output = Vec::new();
let mut calls = Vec::new();
serve_lines(&input[..], &mut output, "secret", 4096, |method, _| {
calls.push(method);
Some(json!({"ok": true}))
});
assert_eq!(calls, ["first"]);
let lines: Vec<Value> = std::str::from_utf8(&output).unwrap().lines().map(|s| serde_json::from_str(s).unwrap()).collect();
assert_eq!(lines[0], json!({"ok": true, "id": 1}));
assert_eq!(lines[1]["error"], "control authentication failed; closing the connection");
}
#[test]
fn bounds_requests_and_preserves_last_unterminated_line() {
let mut buf = Vec::new();
assert_eq!(read_bounded_line(&mut &b"abc"[..], &mut buf, 3), Ok(true));
assert_eq!(buf, b"abc");
assert!(matches!(read_bounded_line(&mut &b"abcd"[..], &mut buf, 3), Err(ReadFailure::Rejected(_))));
let mut out = Vec::new();
serve_lines(&b"{broken}\n"[..], &mut out, "secret", 1024, |_, _| panic!("must not dispatch"));
let reply: Value = serde_json::from_slice(&out).unwrap();
assert!(reply["error"].as_str().unwrap().starts_with("bad JSON ("));
}
}
+18
View File
@@ -0,0 +1,18 @@
[package]
name = "craft-deckcraft-format"
description = "The DeckCraft .deckcraft package: identity, manifest envelope, NaN-safe JSON, member names and bounded reads"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
craft-zip = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
zip = { version = "9", default-features = false, features = ["deflate-flate2"] }
+25
View File
@@ -0,0 +1,25 @@
# craft-deckcraft-format
The DeckCraft `.deckcraft` package. A file is a ZIP that holds, in this order:
- `mimetype`, stored: `application/vnd.storyteller.deckcraft+zip`. Files from before the
SlideCraft rename carry the legacy type, and `sniff` accepts both.
- `presentation.json`, deflated: `{"format":"deckcraft","version":1,"generator":…,"presentation":…}`.
It is compact JSON, and non-finite numbers are written as `0` (`Finite`).
- `media/<id>` and `chart-parts/<index>` binary members. A member whose content type is already
compressed (JPEG, PNG, video, MP3/MP4 audio, anything `zip`) is stored, and any other member is deflated.
The presentation is a type parameter: DeckCraft owns the model, decides which media and chart
parts become members, and repairs a loaded deck. This crate owns the bytes:
- `write(presentation, generator, members)` writes the package with `craft-zip`'s deterministic writer.
- `Package::open_manifest(bytes, max)` reads and checks the manifest. A file with no
`presentation.json` or a different `format` is `NotOurs`. `parse_manifest` reads files written
before `Finite`, where serde_json saved NaN as `null`, as 0 within fixed patch and re-parse budgets.
- `Package::read_max` refuses a member that declares more than its limit and stops one that
inflates past it. `read_charged` charges a shared budget for every byte it inflates, including
the bytes of a read that then fails its CRC.
- `Budget` and the `MAX_*` constants are the format's limits: 2 GiB per member, 4 GiB of media and
chart parts, 100,000 items and a 256 MiB manifest.
There is no `unsafe` and no panics, and it builds for `wasm32-unknown-unknown`.
+185
View File
@@ -0,0 +1,185 @@
//! A serde serializer adapter that writes non-finite floats as 0.
//!
//! serde_json writes NaN and ±infinity as `null`, which doesn't read back as a number: a single
//! NaN anywhere in the model (a rotation, a crop, a stroke width) made a saved file fail to open
//! as "damaged". [`Finite`] wraps any `Serialize` value and replaces those floats on the way out;
//! everything else passes through unchanged.
use serde::ser::{self, Serialize, Serializer};
/// `T`, serialized with non-finite floats written as 0.
pub struct Finite<'a, T: ?Sized>(pub &'a T);
impl<T: ?Sized + Serialize> Serialize for Finite<'_, T> {
fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
self.0.serialize(Wrap(s))
}
}
struct Wrap<S>(S);
macro_rules! pass {
($($m:ident($t:ty)),* $(,)?) => {
$(fn $m(self, v: $t) -> Result<S::Ok, S::Error> { self.0.$m(v) })*
};
}
impl<S: Serializer> Serializer for Wrap<S> {
type Ok = S::Ok;
type Error = S::Error;
type SerializeSeq = Wrap<S::SerializeSeq>;
type SerializeTuple = Wrap<S::SerializeTuple>;
type SerializeTupleStruct = Wrap<S::SerializeTupleStruct>;
type SerializeTupleVariant = Wrap<S::SerializeTupleVariant>;
type SerializeMap = Wrap<S::SerializeMap>;
type SerializeStruct = Wrap<S::SerializeStruct>;
type SerializeStructVariant = Wrap<S::SerializeStructVariant>;
pass!(
serialize_bool(bool),
serialize_i8(i8),
serialize_i16(i16),
serialize_i32(i32),
serialize_i64(i64),
serialize_i128(i128),
serialize_u8(u8),
serialize_u16(u16),
serialize_u32(u32),
serialize_u64(u64),
serialize_u128(u128),
serialize_char(char),
serialize_str(&str),
serialize_bytes(&[u8]),
);
fn serialize_f32(self, v: f32) -> Result<S::Ok, S::Error> {
self.0.serialize_f32(if v.is_finite() { v } else { 0.0 })
}
fn serialize_f64(self, v: f64) -> Result<S::Ok, S::Error> {
self.0.serialize_f64(if v.is_finite() { v } else { 0.0 })
}
fn serialize_none(self) -> Result<S::Ok, S::Error> {
self.0.serialize_none()
}
fn serialize_some<T: ?Sized + Serialize>(self, v: &T) -> Result<S::Ok, S::Error> {
self.0.serialize_some(&Finite(v))
}
fn serialize_unit(self) -> Result<S::Ok, S::Error> {
self.0.serialize_unit()
}
fn serialize_unit_struct(self, name: &'static str) -> Result<S::Ok, S::Error> {
self.0.serialize_unit_struct(name)
}
fn serialize_unit_variant(self, name: &'static str, i: u32, variant: &'static str) -> Result<S::Ok, S::Error> {
self.0.serialize_unit_variant(name, i, variant)
}
fn serialize_newtype_struct<T: ?Sized + Serialize>(self, name: &'static str, v: &T) -> Result<S::Ok, S::Error> {
self.0.serialize_newtype_struct(name, &Finite(v))
}
fn serialize_newtype_variant<T: ?Sized + Serialize>(self, name: &'static str, i: u32, variant: &'static str, v: &T) -> Result<S::Ok, S::Error> {
self.0.serialize_newtype_variant(name, i, variant, &Finite(v))
}
fn serialize_seq(self, len: Option<usize>) -> Result<Self::SerializeSeq, S::Error> {
self.0.serialize_seq(len).map(Wrap)
}
fn serialize_tuple(self, len: usize) -> Result<Self::SerializeTuple, S::Error> {
self.0.serialize_tuple(len).map(Wrap)
}
fn serialize_tuple_struct(self, name: &'static str, len: usize) -> Result<Self::SerializeTupleStruct, S::Error> {
self.0.serialize_tuple_struct(name, len).map(Wrap)
}
fn serialize_tuple_variant(self, name: &'static str, i: u32, variant: &'static str, len: usize) -> Result<Self::SerializeTupleVariant, S::Error> {
self.0.serialize_tuple_variant(name, i, variant, len).map(Wrap)
}
fn serialize_map(self, len: Option<usize>) -> Result<Self::SerializeMap, S::Error> {
self.0.serialize_map(len).map(Wrap)
}
fn serialize_struct(self, name: &'static str, len: usize) -> Result<Self::SerializeStruct, S::Error> {
self.0.serialize_struct(name, len).map(Wrap)
}
fn serialize_struct_variant(
self,
name: &'static str,
i: u32,
variant: &'static str,
len: usize,
) -> Result<Self::SerializeStructVariant, S::Error> {
self.0.serialize_struct_variant(name, i, variant, len).map(Wrap)
}
fn is_human_readable(&self) -> bool {
self.0.is_human_readable()
}
}
macro_rules! compound {
($tr:ident, $m:ident) => {
impl<S: ser::$tr> ser::$tr for Wrap<S> {
type Ok = S::Ok;
type Error = S::Error;
fn $m<T: ?Sized + Serialize>(&mut self, v: &T) -> Result<(), S::Error> {
self.0.$m(&Finite(v))
}
fn end(self) -> Result<S::Ok, S::Error> {
self.0.end()
}
}
};
}
compound!(SerializeSeq, serialize_element);
compound!(SerializeTuple, serialize_element);
compound!(SerializeTupleStruct, serialize_field);
compound!(SerializeTupleVariant, serialize_field);
impl<S: ser::SerializeMap> ser::SerializeMap for Wrap<S> {
type Ok = S::Ok;
type Error = S::Error;
fn serialize_key<T: ?Sized + Serialize>(&mut self, k: &T) -> Result<(), S::Error> {
self.0.serialize_key(&Finite(k))
}
fn serialize_value<T: ?Sized + Serialize>(&mut self, v: &T) -> Result<(), S::Error> {
self.0.serialize_value(&Finite(v))
}
fn end(self) -> Result<S::Ok, S::Error> {
self.0.end()
}
}
macro_rules! fields {
($tr:ident) => {
impl<S: ser::$tr> ser::$tr for Wrap<S> {
type Ok = S::Ok;
type Error = S::Error;
fn serialize_field<T: ?Sized + Serialize>(&mut self, key: &'static str, v: &T) -> Result<(), S::Error> {
self.0.serialize_field(key, &Finite(v))
}
fn skip_field(&mut self, key: &'static str) -> Result<(), S::Error> {
self.0.skip_field(key)
}
fn end(self) -> Result<S::Ok, S::Error> {
self.0.end()
}
}
};
}
fields!(SerializeStruct);
fields!(SerializeStructVariant);
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn non_finite_floats_are_written_as_zero_and_nothing_else_changes() {
#[derive(serde::Serialize)]
struct T {
a: f64,
b: Option<f64>,
c: Vec<(f32, f64)>,
d: Option<String>,
e: std::collections::BTreeMap<String, f64>,
}
let t = T { a: f64::NAN, b: Some(f64::INFINITY), c: vec![(f32::NEG_INFINITY, 1.5)], d: None, e: [("k".to_string(), f64::NAN)].into() };
let s = serde_json::to_string(&Finite(&t)).unwrap_or_default();
assert_eq!(s, r#"{"a":0.0,"b":0.0,"c":[[0.0,1.5]],"d":null,"e":{"k":0.0}}"#);
}
}
+226
View File
@@ -0,0 +1,226 @@
//! The native `.deckcraft` package: a ZIP holding a stored `mimetype`, `presentation.json` (the
//! manifest: format name, version, generator and the presentation, minus media bytes), and
//! `media/<id>` and `chart-parts/<index>` binary members. Readers ignore unknown fields, so newer
//! files open in older builds with what those builds understand.
//!
//! DeckCraft owns the presentation model and its mapping: which media and chart parts become
//! members, and how a loaded deck is repaired. This crate owns the bytes: the package identity,
//! the manifest envelope, the NaN-safe JSON writer, the reader that opens files with NaN saved as
//! `null`, member names and compression, and the bounded member reads and their budgets.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
mod finite;
mod manifest;
#[cfg(test)]
mod tests;
use std::io::{Cursor, Read};
pub use finite::Finite;
pub use manifest::parse_manifest;
pub const MIMETYPE: &str = "application/vnd.storyteller.deckcraft+zip";
/// Files written before the SlideCraft → DeckCraft rename (`.slidecraft`) are still read.
pub const LEGACY_MIMETYPE: &str = concat!("application/vnd.storyteller.", "slide", "craft+zip");
pub const EXTENSION: &str = "deckcraft";
/// The manifest's `format`.
pub const FORMAT: &str = "deckcraft";
/// Format version written into the manifest.
pub const VERSION: u32 = 1;
/// The stored member naming the package type.
pub const MIMETYPE_ENTRY: &str = "mimetype";
/// The manifest member.
pub const MANIFEST_ENTRY: &str = "presentation.json";
/// Largest entry we will inflate (guards against zip bombs).
pub const MAX_ENTRY: u64 = 2 << 30;
/// All embedded media and chart sidecars together, inflated (storytold/deckcraft#393).
pub const MAX_TOTAL_MEDIA: u64 = 4 << 30;
/// Media items and chart sidecar parts one file may list.
pub const MAX_MEDIA_ITEMS: usize = 100_000;
/// Largest `presentation.json` we will inflate. The manifest holds no media bytes, so it is far
/// smaller than a media entry may be; inflating and parsing one is what dominates memory.
pub const MAX_MANIFEST: u64 = 256 << 20;
const MEDIA_PREFIX: &str = "media/";
const CHART_PARTS_PREFIX: &str = "chart-parts/";
/// What one load may inflate.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Budget {
/// Bytes all media and chart members may inflate to together.
pub total: u64,
/// Media items and chart parts the manifest may list.
pub items: usize,
/// Bytes `presentation.json` may inflate to.
pub manifest: u64,
}
impl Default for Budget {
fn default() -> Self {
Budget { total: MAX_TOTAL_MEDIA, items: MAX_MEDIA_ITEMS, manifest: MAX_MANIFEST }
}
}
impl Budget {
/// A media budget with the default manifest limit.
pub const fn media(total: u64, items: usize) -> Self {
Budget { total, items, manifest: MAX_MANIFEST }
}
}
#[derive(Debug, thiserror::Error)]
pub enum FormatError {
#[error("not a DeckCraft file: {0}")]
NotOurs(String),
#[error("damaged file: {0}")]
Damaged(String),
#[error("write failed: {0}")]
Write(String),
}
pub type Result<T> = std::result::Result<T, FormatError>;
/// `presentation.json`: the envelope around the app's presentation.
#[derive(serde::Serialize, serde::Deserialize)]
pub struct Manifest<T> {
pub format: String,
pub version: u32,
pub generator: String,
pub presentation: T,
}
/// The member holding media item `id`'s bytes.
pub fn media_entry(id: u32) -> String {
format!("{MEDIA_PREFIX}{id}")
}
/// The member holding the `index`th chart part a save separated out.
pub fn chart_part_entry(index: usize) -> String {
format!("{CHART_PARTS_PREFIX}{index}")
}
/// Whether `entry` is a name [`chart_part_entry`] gives, exactly: no other member, no path
/// tricks, no leading zeros.
pub fn is_chart_part_entry(entry: &str) -> bool {
entry.strip_prefix(CHART_PARTS_PREFIX).and_then(|s| s.parse::<usize>().ok()).is_some_and(|n| entry == chart_part_entry(n))
}
/// Whether a member of this content type is stored rather than deflated: it is compressed already.
pub fn is_compressed(content_type: &str) -> bool {
content_type.starts_with("image/jpeg")
|| content_type.starts_with("image/png")
|| content_type.starts_with("video/")
|| content_type.starts_with("audio/mpeg")
|| content_type.starts_with("audio/mp4")
|| content_type.contains("zip")
}
/// Does this look like a `.deckcraft` file?
pub fn sniff(bytes: &[u8]) -> bool {
bytes.starts_with(b"PK") && [MIMETYPE, LEGACY_MIMETYPE].iter().any(|m| bytes.windows(m.len()).take(200).any(|w| w == m.as_bytes()))
}
/// A binary member of a package being written.
#[derive(Clone, Copy, Debug)]
pub struct Member<'a> {
/// Its name, from [`media_entry`] or [`chart_part_entry`].
pub name: &'a str,
/// Its content type, which decides whether it is stored or deflated.
pub content_type: &'a str,
pub bytes: &'a [u8],
}
/// The package: `mimetype` (stored), the manifest naming `generator` around `presentation`
/// (deflated, with non-finite numbers written as 0), then `members` in order.
pub fn write<T: serde::Serialize>(presentation: &T, generator: &str, members: &[Member<'_>]) -> Result<Vec<u8>> {
use craft_zip::Compression::{Deflated, Stored};
let w = |e: craft_zip::Error| FormatError::Write(e.to_string());
let mut z = craft_zip::Writer::new();
z.add(MIMETYPE_ENTRY, MIMETYPE.as_bytes(), Stored).map_err(w)?;
let m = Manifest { format: FORMAT.to_string(), version: VERSION, generator: generator.to_string(), presentation };
let json = serde_json::to_vec(&Finite(&m)).map_err(|e| FormatError::Write(e.to_string()))?;
z.add(MANIFEST_ENTRY, &json, Deflated).map_err(w)?;
for member in members {
let compression = if is_compressed(member.content_type) { Stored } else { Deflated };
z.add(member.name, member.bytes, compression).map_err(w)?;
}
z.finish().map_err(w)
}
/// A package open for reading. Nothing is inflated until a member is read.
pub struct Package<'a> {
zip: zip::ZipArchive<Cursor<&'a [u8]>>,
}
impl<'a> Package<'a> {
/// Index `bytes` as a ZIP.
pub fn open(bytes: &'a [u8]) -> Result<Self> {
let zip = zip::ZipArchive::new(Cursor::new(bytes)).map_err(|e| FormatError::NotOurs(e.to_string()))?;
Ok(Package { zip })
}
/// Open `bytes` and read its manifest, inflating at most `max_manifest` bytes of it. Missing
/// `presentation.json` or a different `format` is [`FormatError::NotOurs`].
pub fn open_manifest<T: serde::de::DeserializeOwned>(bytes: &'a [u8], max_manifest: u64) -> Result<(Manifest<T>, Self)> {
let mut package = Package::open(bytes)?;
if !package.contains(MANIFEST_ENTRY) {
return Err(FormatError::NotOurs(format!("no {MANIFEST_ENTRY}")));
}
let json = package.read_max(MANIFEST_ENTRY, max_manifest)?;
let m: Manifest<T> = parse_manifest(&json)?;
if m.format != FORMAT {
return Err(FormatError::NotOurs(m.format));
}
Ok((m, package))
}
/// Whether the package has a member named `name`.
pub fn contains(&self, name: &str) -> bool {
self.zip.index_for_name(name).is_some()
}
/// The size `name` declares, inflated; `None` when there is no such readable member.
pub fn declared_size(&mut self, name: &str) -> Option<u64> {
self.zip.by_name(name).ok().map(|f| f.size())
}
/// Read one member within [`MAX_ENTRY`].
pub fn read(&mut self, name: &str) -> Result<Vec<u8>> {
self.read_max(name, MAX_ENTRY)
}
/// Read one member, refusing it before inflating if it declares more than `max` bytes, and
/// stopping at `max` if it inflates further than it declared.
pub fn read_max(&mut self, name: &str, max: u64) -> Result<Vec<u8>> {
self.read_counted(name, max, &mut 0)
}
/// [`Package::read_max`] within `left` (and [`MAX_ENTRY`]), charging `left` for every byte
/// inflated, including those inflated before a decode or CRC error, so a member that fails
/// late still spends the shared budget it used.
pub fn read_charged(&mut self, name: &str, left: &mut u64) -> Result<Vec<u8>> {
let mut spent = 0;
let out = self.read_counted(name, (*left).min(MAX_ENTRY), &mut spent);
*left = left.saturating_sub(spent);
out
}
/// [`Package::read_max`], adding the bytes inflated to `spent` whether or not the read succeeds.
fn read_counted(&mut self, name: &str, max: u64, spent: &mut u64) -> Result<Vec<u8>> {
let f = self.zip.by_name(name).map_err(|e| FormatError::Damaged(format!("{name}: {e}")))?;
if f.size() > max {
return Err(FormatError::Damaged(format!("{name} is too large")));
}
let mut out = Vec::with_capacity(usize::try_from(f.size().min(64 << 20)).unwrap_or(0));
let mut f = f.take(max.saturating_add(1));
let read = f.read_to_end(&mut out);
*spent = spent.saturating_add(out.len() as u64);
read.map_err(|e| FormatError::Damaged(format!("{name}: {e}")))?;
if out.len() as u64 > max {
return Err(FormatError::Damaged(format!("{name} is too large")));
}
Ok(out)
}
}
+99
View File
@@ -0,0 +1,99 @@
//! Reading `presentation.json`, including files written with NaN or infinity saved as `null`.
use serde::de::DeserializeOwned;
use crate::{FormatError, Manifest, Result};
/// Most `null`s standing in for numbers that [`parse_manifest`] patches in one file, and the
/// candidates it tries for each.
const MAX_NULL_FIXES: usize = 256;
const MAX_TRIES_PER_FIX: usize = 32;
/// Bytes [`parse_manifest`] may re-parse in all while patching, so a hostile file full of `null`s
/// can't keep it busy: past this the file is reported damaged.
const MAX_REPARSE_BYTES: usize = 256 * 1024 * 1024;
/// Where a parse failed, comparable: later is progress.
fn failure<T: DeserializeOwned>(src: &[u8]) -> std::result::Result<Manifest<T>, (String, (usize, usize))> {
serde_json::from_slice::<Manifest<T>>(src).map_err(|e| (e.to_string(), (e.line(), e.column())))
}
/// Parse `presentation.json`. serde_json writes a NaN or infinite number as `null`, which then
/// fails to read back as a number ("invalid type: null, expected f64"), so DeckCraft could save
/// a file it couldn't open. Each such `null` is read as 0 instead (the app then brings the
/// value into range); anything else that doesn't parse is damage.
///
/// serde reports the failing `null` exactly, except inside tagged enums (buffered), where it
/// reports the end of the enclosing object: there the `null`s before that point are tried
/// nearest first, and a patch is kept only if the parse then gets further. (Files saved since
/// [`crate::Finite`] never contain such `null`s; this reads older ones.)
pub fn parse_manifest<T: DeserializeOwned>(json: &[u8]) -> Result<Manifest<T>> {
// Parse the inflated text as it is: only a file that fails on a `null` number is copied for
// patching (the copy is bounded by `MAX_MANIFEST`).
let mut last = match failure(json) {
Ok(m) => return Ok(m),
Err(f) => f,
};
if !is_null_number(&last.0) {
return Err(FormatError::Damaged(last.0));
}
let mut src = json.to_vec();
let mut budget = MAX_REPARSE_BYTES;
'fixes: for _ in 0..MAX_NULL_FIXES {
let (msg, pos) = &last;
if !is_null_number(msg) {
break;
}
let end = offset(&src, pos.0, pos.1).unwrap_or(src.len());
let mut progressed = false;
let candidates: Vec<usize> = value_nulls_before(&src, end + 4).take(MAX_TRIES_PER_FIX).collect();
for at in candidates {
let (Some(head), Some(tail)) = (src.get(..at), src.get(at.saturating_add(4)..)) else { continue };
let Some(left) = budget.checked_sub(src.len()) else { break 'fixes };
budget = left;
let mut next = Vec::with_capacity(src.len());
next.extend_from_slice(head);
next.push(b'0');
next.extend_from_slice(tail);
match failure(&next) {
Ok(m) => return Ok(m),
// The patch shortened the text by 3 bytes before the error: compare positions
// allowing for that.
Err(f) if f.1 > (pos.0, pos.1.saturating_sub(3)) => {
src = next;
last = f;
progressed = true;
break;
}
Err(_) => {}
}
}
if !progressed {
break;
}
}
Err(FormatError::Damaged(last.0))
}
/// Is this parse error a `null` where a number belongs (a NaN or infinity serde_json wrote)?
fn is_null_number(msg: &str) -> bool {
msg.starts_with("invalid type: null, expected f") || msg.starts_with("invalid type: null, expected a number")
}
/// Byte offset of a 1-based `line`:`column`.
fn offset(src: &[u8], line: usize, column: usize) -> Option<usize> {
let mut start = 0;
for _ in 1..line {
start += src.get(start..)?.iter().position(|b| *b == b'\n')? + 1;
}
Some((start + column).min(src.len()))
}
/// Starts of `null` tokens in value position (after `:`, `[` or `,`) before `end`, nearest first.
fn value_nulls_before(src: &[u8], end: usize) -> impl Iterator<Item = usize> + '_ {
let end = end.min(src.len());
(0..end.saturating_sub(3)).rev().filter(move |&i| {
src.get(i..i + 4) == Some(b"null")
&& src[..i].iter().rev().find(|b| !b.is_ascii_whitespace()).is_some_and(|b| matches!(b, b':' | b'[' | b','))
&& src.get(i + 4).is_none_or(|b| !b.is_ascii_alphanumeric())
})
}
+235
View File
@@ -0,0 +1,235 @@
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use serde::{Deserialize, Serialize};
use super::*;
/// A stand-in for an app's presentation: plain numbers, an optional number and a tagged enum
/// (whose `null`s serde reports only at the end of the enclosing object).
#[derive(Debug, Default, PartialEq, Serialize, Deserialize)]
struct Deck {
#[serde(default)]
width: f64,
#[serde(default)]
zoom: Option<f64>,
#[serde(default)]
shapes: Vec<Shape>,
}
#[derive(Debug, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind")]
enum Shape {
Box { x: f64, y: f64 },
Line { len: f64 },
}
fn package(members: &[Member<'_>]) -> Vec<u8> {
write(&Deck { width: 960.0, ..Default::default() }, "Test 1", members).unwrap()
}
#[test]
fn a_package_holds_mimetype_then_manifest_then_members_in_order() {
let png = [0x89, b'P', b'N', b'G'];
let svg = b"<svg/>".repeat(20);
let bytes = package(&[
Member { name: &media_entry(7), content_type: "image/png", bytes: &png },
Member { name: &media_entry(8), content_type: "image/svg+xml", bytes: &svg },
Member { name: &chart_part_entry(0), content_type: "application/zip", bytes: b"PK" },
]);
assert!(sniff(&bytes));
let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes.as_slice())).unwrap();
assert_eq!(z.len(), 5);
let methods: Vec<_> = (0..z.len())
.map(|i| {
let f = z.by_index(i).unwrap();
(f.name().unwrap().into_owned(), f.compression())
})
.collect();
use zip::CompressionMethod::{Deflated, Stored};
assert_eq!(
methods,
[("mimetype", Stored), ("presentation.json", Deflated), ("media/7", Stored), ("media/8", Deflated), ("chart-parts/0", Stored)]
.map(|(n, m)| (n.to_string(), m))
);
// The mimetype is the first, stored member, readable in place.
assert_eq!(craft_zip::stored_mimetype(&bytes), Some(MIMETYPE.as_bytes()));
let (m, mut p) = Package::open_manifest::<Deck>(&bytes, MAX_MANIFEST).unwrap();
assert_eq!((m.format.as_str(), m.version, m.generator.as_str(), m.presentation.width), (FORMAT, VERSION, "Test 1", 960.0));
assert_eq!(p.read(&media_entry(8)).unwrap(), svg);
assert_eq!(p.declared_size(&media_entry(7)), Some(4));
assert_eq!(p.declared_size("media/99"), None);
}
#[test]
fn the_manifest_is_compact_json_with_its_fields_in_order() {
let bytes = package(&[]);
let mut p = Package::open(&bytes).unwrap();
let json = String::from_utf8(p.read(MANIFEST_ENTRY).unwrap()).unwrap();
assert_eq!(json, r#"{"format":"deckcraft","version":1,"generator":"Test 1","presentation":{"width":960.0,"zoom":null,"shapes":[]}}"#);
}
#[test]
fn non_finite_numbers_are_written_as_zero_and_read_back() {
let deck = Deck { width: f64::NAN, zoom: Some(f64::INFINITY), shapes: vec![Shape::Box { x: f64::NEG_INFINITY, y: 2.0 }] };
let bytes = write(&deck, "t", &[]).unwrap();
let (m, _) = Package::open_manifest::<Deck>(&bytes, MAX_MANIFEST).unwrap();
assert_eq!(m.presentation, Deck { width: 0.0, zoom: Some(0.0), shapes: vec![Shape::Box { x: 0.0, y: 2.0 }] });
}
/// Files written before [`Finite`] (plain serde_json: NaN as `null`) open too, including a
/// `null` inside a tagged enum, where serde reports only the enclosing object.
#[test]
fn older_manifests_with_null_numbers_parse() {
let cases: [(&str, Deck); 3] = [
(r#"{"format":"deckcraft","version":1,"generator":"old","presentation":{"width":null}}"#, Deck::default()),
(
r#"{"format":"deckcraft","version":1,"generator":"old","presentation":{"width":3,"shapes":[{"kind":"Box","x":null,"y":4}]}}"#,
Deck { width: 3.0, shapes: vec![Shape::Box { x: 0.0, y: 4.0 }], ..Default::default() },
),
(
"{\n \"format\": \"deckcraft\",\n \"version\": 1,\n \"generator\": \"t\",\n \"presentation\": {\"shapes\": [{\"kind\": \"Line\", \"len\": null}]}\n}",
Deck { shapes: vec![Shape::Line { len: 0.0 }], ..Default::default() },
),
];
for (json, want) in cases {
let m = parse_manifest::<Deck>(json.as_bytes()).unwrap_or_else(|e| panic!("{json}: {e}"));
assert_eq!(m.presentation, want, "{json}");
}
}
/// Only `null`s where numbers belong are patched: other damage still fails.
#[test]
fn other_manifest_damage_is_still_reported() {
let ok = br#"{"format": "deckcraft", "version": 1, "generator": "t", "presentation": {}}"#;
assert!(parse_manifest::<Deck>(ok).is_ok());
let bad = br#"{"format": null, "version": 1, "generator": "t", "presentation": {}}"#;
assert!(matches!(parse_manifest::<Deck>(bad), Err(FormatError::Damaged(_))));
assert!(parse_manifest::<Deck>(&ok[..ok.len() - 3]).is_err());
}
#[test]
fn a_manifest_full_of_nulls_is_refused_within_bounds() {
let shapes = (0..2000).map(|_| r#"{"kind":"Box","x":1,"y":"not a number"}"#).collect::<Vec<_>>().join(",");
let json = format!(r#"{{"format":"deckcraft","version":1,"generator":"t","presentation":{{"width":null,"shapes":[{shapes}]}}}}"#);
assert!(matches!(parse_manifest::<Deck>(json.as_bytes()), Err(FormatError::Damaged(_))));
}
#[test]
fn what_is_not_a_package_is_not_ours() {
for bytes in [&b""[..], b"PK\x03\x04garbage", &[0u8; 1000]] {
assert!(matches!(Package::open_manifest::<Deck>(bytes, MAX_MANIFEST), Err(FormatError::NotOurs(_))));
}
// A ZIP without the manifest, and one from another app.
let mut z = craft_zip::Writer::new();
z.add("mimetype", b"application/zip", craft_zip::Compression::Stored).unwrap();
let no_manifest = z.finish().unwrap();
let err = Package::open_manifest::<Deck>(&no_manifest, MAX_MANIFEST).err().unwrap();
assert_eq!(err.to_string(), "not a DeckCraft file: no presentation.json");
let mut z = craft_zip::Writer::new();
z.add(MANIFEST_ENTRY, br#"{"format":"other","version":1,"generator":"x","presentation":{}}"#, craft_zip::Compression::Deflated).unwrap();
let other = z.finish().unwrap();
let err = Package::open_manifest::<Deck>(&other, MAX_MANIFEST).err().unwrap();
assert_eq!(err.to_string(), "not a DeckCraft file: other");
// A truncated package is an error, not a panic.
let mut b = package(&[]);
b.truncate(b.len() / 2);
assert!(Package::open_manifest::<Deck>(&b, MAX_MANIFEST).is_err());
}
#[test]
fn legacy_slidecraft_packages_are_recognised() {
let mut z = craft_zip::Writer::new();
z.add(MIMETYPE_ENTRY, LEGACY_MIMETYPE.as_bytes(), craft_zip::Compression::Stored).unwrap();
assert!(sniff(&z.finish().unwrap()));
assert!(!sniff(b"PK\x03\x04 application/zip"));
assert!(!sniff(MIMETYPE.as_bytes()));
}
#[test]
fn an_oversized_manifest_is_refused_before_inflating() {
let bytes = package(&[]);
let size = Package::open(&bytes).unwrap().declared_size(MANIFEST_ENTRY).unwrap();
assert!(Package::open_manifest::<Deck>(&bytes, size).is_ok());
let err = Package::open_manifest::<Deck>(&bytes, size - 1).err().unwrap();
assert_eq!(err.to_string(), "damaged file: presentation.json is too large");
}
#[test]
fn a_member_that_inflates_past_its_declared_size_stops_at_the_limit() {
// Declared sizes are the file's word: a stored member whose header lies is still capped.
let data = vec![7u8; 100];
let bytes = package(&[Member { name: &media_entry(1), content_type: "image/png", bytes: &data }]);
let mut p = Package::open(&bytes).unwrap();
assert_eq!(p.read_max(&media_entry(1), 100).unwrap().len(), 100);
assert!(p.read_max(&media_entry(1), 99).is_err());
let err = p.read("media/2").unwrap_err();
assert!(matches!(err, FormatError::Damaged(ref m) if m.starts_with("media/2: ")), "{err}");
}
#[test]
fn corrupt_members_are_charged_for_what_they_inflated() {
let a = b"PICTURE-AAAAAAAAAAAA";
let b = b"PICTURE-BBBBBBBBBBBB";
let mut bytes = package(&[
Member { name: &media_entry(1), content_type: "image/png", bytes: a },
Member { name: &media_entry(2), content_type: "image/png", bytes: b },
]);
// Corrupt both stored members, so each fails its CRC only after inflating all its bytes.
for tag in [b"PICTURE-AAAA", b"PICTURE-BBBB"] {
let at = bytes.windows(tag.len()).position(|w| w == tag).unwrap();
bytes[at] ^= 0xff;
}
let mut p = Package::open(&bytes).unwrap();
let mut left = 30;
assert!(p.read_charged(&media_entry(1), &mut left).is_err());
assert_eq!(left, 10, "a failed read still spends what it inflated");
// The second fits a fresh budget on its own, but not what is left: refused before inflating.
let err = p.read_charged(&media_entry(2), &mut left).unwrap_err();
assert_eq!(err.to_string(), "damaged file: media/2 is too large");
assert_eq!(left, 10);
}
#[test]
fn chart_part_names_are_exact() {
assert_eq!(chart_part_entry(3), "chart-parts/3");
assert!(is_chart_part_entry("chart-parts/0"));
assert!(is_chart_part_entry("chart-parts/12"));
for not in ["media/1", "chart-parts/../0", "chart-parts/00", "chart-parts/-1", "chart-parts/", "chart-parts/+1", "chart-parts/1 "] {
assert!(!is_chart_part_entry(not), "{not}");
}
}
#[test]
fn compressed_content_types_are_stored() {
for stored in [
"image/jpeg",
"image/png",
"video/mp4",
"audio/mpeg",
"audio/mp4",
"application/zip",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet+zip",
] {
assert!(is_compressed(stored), "{stored}");
}
for deflated in ["image/svg+xml", "image/gif", "audio/wav", "application/octet-stream", "text/plain"] {
assert!(!is_compressed(deflated), "{deflated}");
}
}
#[test]
fn duplicate_members_are_a_write_error() {
let err = write(
&Deck::default(),
"t",
&[Member { name: "media/1", content_type: "x", bytes: b"a" }, Member { name: "media/1", content_type: "x", bytes: b"b" }],
)
.unwrap_err();
assert!(matches!(err, FormatError::Write(_)), "{err}");
}
#[test]
fn the_default_budget_is_the_formats_limits() {
assert_eq!(Budget::default(), Budget { total: 4 << 30, items: 100_000, manifest: 256 << 20 });
assert_eq!(Budget::media(10, 2), Budget { total: 10, items: 2, manifest: MAX_MANIFEST });
}
+362 -65
View File
@@ -6,7 +6,7 @@
//! [`spec_of`] turns a chart part back into the model only when writing that model again gives
//! the same markup, so a chart with anything the model leaves out stays as it was read.
use craft_wordmodel::chart::{ChartSeries, ChartSpec, ChartType, LegendPos, MAX_POINTS, MAX_SERIES};
use craft_wordmodel::chart::{ChartAxis, ChartSeries, ChartSpec, ChartType, LegendPos, MAX_POINTS, MAX_SERIES};
use craft_wordmodel::graphic::GraphicItem;
use craft_wordmodel::props::Rgb;
@@ -102,11 +102,24 @@ fn bounded(spec: &ChartSpec) -> ChartSpec {
.series
.iter()
.take(MAX_SERIES)
.map(|s| ChartSeries { name: s.name.clone(), values: s.values.iter().take(MAX_POINTS).copied().collect() })
.map(|s| ChartSeries {
name: s.name.clone(),
values: s.values.iter().take(MAX_POINTS).copied().collect(),
fill: s.fill,
no_fill: s.no_fill,
line_w: s.line_w,
no_line: s.no_line,
markers: s.markers,
smooth: s.smooth,
labels: s.labels,
fmt: s.fmt.clone(),
})
.collect(),
title: spec.title.clone(),
legend: spec.legend,
data_labels: spec.data_labels,
cat_axis: spec.cat_axis.clone(),
val_axis: spec.val_axis.clone(),
}
.sanitized()
}
@@ -185,17 +198,19 @@ fn plot(w: &mut W, spec: &ChartSpec) {
match k {
ChartType::Pie | ChartType::Doughnut => {}
ChartType::Scatter => {
value_axis(w, CAT_AX, VAL_AX, "b", false, "midCat");
value_axis(w, VAL_AX, CAT_AX, "l", true, "midCat");
value_axis(w, CAT_AX, VAL_AX, "b", "midCat", &spec.cat_axis);
value_axis(w, VAL_AX, CAT_AX, "l", "midCat", &spec.val_axis);
}
_ => {
category_axis(w, if horizontal { "l" } else { "b" });
value_axis(w, VAL_AX, CAT_AX, if horizontal { "b" } else { "l" }, true, "between");
category_axis(w, if horizontal { "l" } else { "b" }, &spec.cat_axis);
value_axis(w, VAL_AX, CAT_AX, if horizontal { "b" } else { "l" }, "between", &spec.val_axis);
}
}
}
/// One `c:ser` (ECMA-376 element order: idx, order, tx, spPr, marker, cat/xVal, val/yVal, smooth).
/// One `c:ser` (ECMA-376 element order: idx, order, tx, spPr, marker, invertIfNegative, dLbls,
/// cat/xVal, val/yVal, smooth): the series' formatting (fill, line, markers, its own labels) sits
/// in `c:spPr`, `c:marker` and `c:dLbls`; the values carry their number format.
fn series(w: &mut W, spec: &ChartSpec, i: usize, s: &ChartSeries) {
let k = spec.kind;
let i = i.to_string();
@@ -207,25 +222,26 @@ fn series(w: &mut W, spec: &ChartSpec, i: usize, s: &ChartSeries) {
w.leaf("c:v", &[], &s.name);
w.close("c:tx");
}
match k {
// Markers only: the series has no line.
ChartType::Scatter => {
w.open("c:spPr", &[]);
w.open("a:ln", &[("w", "19050")]);
w.empty("a:noFill", &[]);
w.close("a:ln");
w.close("c:spPr");
}
ChartType::Column | ChartType::ColumnStacked | ChartType::Bar | ChartType::BarStacked => {
w.empty("c:invertIfNegative", &[("val", "0")]);
}
ChartType::Line => {
w.open("c:marker", &[]);
w.empty("c:symbol", &[("val", "none")]);
w.close("c:marker");
}
_ => {}
series_props(w, s);
series_marker(w, k, s);
if matches!(k, ChartType::Column | ChartType::ColumnStacked | ChartType::Bar | ChartType::BarStacked) {
w.empty("c:invertIfNegative", &[("val", "0")]);
}
if let Some(on) = s.labels {
w.open("c:dLbls", &[]);
for (name, show) in [
("c:showLegendKey", false),
("c:showVal", on),
("c:showCatName", false),
("c:showSerName", false),
("c:showPercent", false),
("c:showBubbleSize", false),
] {
w.empty(name, &[("val", if show { "1" } else { "0" })]);
}
w.close("c:dLbls");
}
let code = s.fmt.as_deref().unwrap_or("General");
if k == ChartType::Scatter {
let xs: Option<Vec<Option<f64>>> = spec
.categories
@@ -234,27 +250,85 @@ fn series(w: &mut W, spec: &ChartSpec, i: usize, s: &ChartSeries) {
.collect();
w.open("c:xVal", &[]);
match xs {
Some(xs) => num_lit(w, &xs),
Some(xs) => num_lit(w, &xs, "General"),
None => str_lit(w, &spec.categories),
}
w.close("c:xVal");
w.open("c:yVal", &[]);
num_lit(w, &s.values);
num_lit(w, &s.values, code);
w.close("c:yVal");
} else {
w.open("c:cat", &[]);
str_lit(w, &spec.categories);
w.close("c:cat");
w.open("c:val", &[]);
num_lit(w, &s.values);
num_lit(w, &s.values, code);
w.close("c:val");
}
if matches!(k, ChartType::Line | ChartType::Scatter) {
w.empty("c:smooth", &[("val", "0")]);
w.empty("c:smooth", &[("val", if s.smooth { "1" } else { "0" })]);
}
w.close("c:ser");
}
/// `c:spPr`: the series' fill (`a:solidFill` / `a:noFill`) and line (`a:ln`), written when the
/// model sets them. `a:ln` without `w` is the default width, `w` a fixed one, and `a:noFill`
/// inside it "no line" (a scatter series keeps its markers with no line).
fn series_props(w: &mut W, s: &ChartSeries) {
let fill_set = s.no_fill || s.fill.is_some();
let ln_set = s.no_line || s.line_w.is_some();
if !(fill_set || ln_set) {
return;
}
w.open("c:spPr", &[]);
if fill_set {
// "No fill" wins over a stored colour, so the pair reads back as it was written.
match &s.fill {
Some(c) if !s.no_fill => {
w.open("a:solidFill", &[]);
w.empty("a:srgbClr", &[("val", &c.hex())]);
w.close("a:solidFill");
}
_ => w.empty("a:noFill", &[]),
}
}
if ln_set {
let w_attr = s.line_w.map(emu).unwrap_or_default();
let attrs = [("w", w_attr.as_str())];
w.open("a:ln", if w_attr.is_empty() { &[] } else { &attrs });
if s.no_line {
w.empty("a:noFill", &[]);
}
w.close("a:ln");
}
w.close("c:spPr");
}
/// `c:marker`: line and scatter series say which symbol each point gets; other types have none.
fn series_marker(w: &mut W, k: ChartType, s: &ChartSeries) {
match k {
ChartType::Line => {
w.open("c:marker", &[]);
w.empty("c:symbol", &[("val", if s.markers == Some(true) { "circle" } else { "none" })]);
w.close("c:marker");
}
// Unset: no `c:marker`, so Word picks its automatic symbol for each series.
ChartType::Scatter => {
if let Some(on) = s.markers {
w.open("c:marker", &[]);
w.empty("c:symbol", &[("val", if on { "circle" } else { "none" })]);
w.close("c:marker");
}
}
_ => {}
}
}
/// A line width in points as the `w` attribute DrawingML wants (EMU).
fn emu(pt: f32) -> String {
((pt * 12700.0).round().max(0.0) as i64).to_string()
}
fn str_lit(w: &mut W, items: &[String]) {
w.open("c:strLit", &[]);
w.empty("c:ptCount", &[("val", &items.len().to_string())]);
@@ -266,9 +340,9 @@ fn str_lit(w: &mut W, items: &[String]) {
w.close("c:strLit");
}
fn num_lit(w: &mut W, values: &[Option<f64>]) {
fn num_lit(w: &mut W, values: &[Option<f64>], code: &str) {
w.open("c:numLit", &[]);
w.leaf("c:formatCode", &[], "General");
w.leaf("c:formatCode", &[], code);
w.empty("c:ptCount", &[("val", &values.len().to_string())]);
for (i, v) in values.iter().enumerate() {
if let Some(v) = v.filter(|v| v.is_finite()) {
@@ -285,17 +359,21 @@ fn num(v: f64) -> String {
if v == 0.0 { "0".to_string() } else { format!("{v}") }
}
fn category_axis(w: &mut W, pos: &str) {
fn category_axis(w: &mut W, pos: &str, ax: &ChartAxis) {
w.open("c:catAx", &[]);
w.empty("c:axId", &[("val", CAT_AX)]);
w.open("c:scaling", &[]);
w.empty("c:orientation", &[("val", "minMax")]);
w.close("c:scaling");
scaling(w, ax);
w.empty("c:delete", &[("val", "0")]);
w.empty("c:axPos", &[("val", pos)]);
if ax.gridlines {
w.empty("c:majorGridlines", &[]);
}
axis_title(w, ax);
num_fmt(w, ax);
w.empty("c:majorTickMark", &[("val", "none")]);
w.empty("c:minorTickMark", &[("val", "none")]);
w.empty("c:tickLblPos", &[("val", "nextTo")]);
w.empty("c:tickLblPos", &[("val", if ax.labels { "nextTo" } else { "none" })]);
axis_line(w, ax);
w.empty("c:crossAx", &[("val", VAL_AX)]);
w.empty("c:crosses", &[("val", "autoZero")]);
w.empty("c:auto", &[("val", "1")]);
@@ -305,34 +383,81 @@ fn category_axis(w: &mut W, pos: &str) {
w.close("c:catAx");
}
/// A value axis: with gridlines and no axis line (`grid`), else with its line and no gridlines.
fn value_axis(w: &mut W, id: &str, cross: &str, pos: &str, grid: bool, between: &str) {
/// A value axis (scatter x axes included): its scale, gridlines, title, labels and line, as the
/// model sets them.
fn value_axis(w: &mut W, id: &str, cross: &str, pos: &str, between: &str, ax: &ChartAxis) {
w.open("c:valAx", &[]);
w.empty("c:axId", &[("val", id)]);
w.open("c:scaling", &[]);
w.empty("c:orientation", &[("val", "minMax")]);
w.close("c:scaling");
scaling(w, ax);
w.empty("c:delete", &[("val", "0")]);
w.empty("c:axPos", &[("val", pos)]);
if grid {
if ax.gridlines {
w.empty("c:majorGridlines", &[]);
}
axis_title(w, ax);
num_fmt(w, ax);
w.empty("c:majorTickMark", &[("val", "none")]);
w.empty("c:minorTickMark", &[("val", "none")]);
w.empty("c:tickLblPos", &[("val", "nextTo")]);
if grid {
w.open("c:spPr", &[]);
w.open("a:ln", &[]);
w.empty("a:noFill", &[]);
w.close("a:ln");
w.close("c:spPr");
}
w.empty("c:tickLblPos", &[("val", if ax.labels { "nextTo" } else { "none" })]);
axis_line(w, ax);
w.empty("c:crossAx", &[("val", cross)]);
w.empty("c:crosses", &[("val", "autoZero")]);
w.empty("c:crossBetween", &[("val", between)]);
w.close("c:valAx");
}
/// `c:scaling`: the orientation, then the fixed bounds when the model sets them.
fn scaling(w: &mut W, ax: &ChartAxis) {
w.open("c:scaling", &[]);
w.empty("c:orientation", &[("val", "minMax")]);
for (name, v) in [("c:max", ax.max), ("c:min", ax.min)] {
if let Some(v) = v.filter(|v| v.is_finite()) {
w.empty(name, &[("val", &num(v))]);
}
}
w.close("c:scaling");
}
/// The axis title, in the same rich-text shape the chart title uses.
fn axis_title(w: &mut W, ax: &ChartAxis) {
let Some(t) = &ax.title else { return };
w.open("c:title", &[]);
w.open("c:tx", &[]);
w.open("c:rich", &[]);
w.empty("a:bodyPr", &[]);
w.empty("a:lstStyle", &[]);
w.open("a:p", &[]);
w.open("a:r", &[]);
w.leaf("a:t", &[], t);
w.close("a:r");
w.close("a:p");
w.close("c:rich");
w.close("c:tx");
w.empty("c:overlay", &[("val", "0")]);
w.close("c:title");
}
/// Tick-label number format: the model's code, unlinked from the source, else the default.
fn num_fmt(w: &mut W, ax: &ChartAxis) {
match &ax.fmt {
Some(code) => w.empty("c:numFmt", &[("formatCode", code), ("sourceLinked", "0")]),
None => w.empty("c:numFmt", &[("formatCode", "General"), ("sourceLinked", "1")]),
}
}
/// `c:spPr`: an axis line only when the model keeps it (the default writes nothing, which reads
/// as a line); `a:noFill` when the axis line is off.
fn axis_line(w: &mut W, ax: &ChartAxis) {
if ax.line {
return;
}
w.open("c:spPr", &[]);
w.open("a:ln", &[]);
w.empty("a:noFill", &[]);
w.close("a:ln");
w.close("c:spPr");
}
/// The chart a chart part holds, when the part is exactly what [`chart_xml`] writes for it (so
/// editing and saving it loses nothing). `None` for any other chart.
pub(crate) fn spec_of(space: &El) -> Option<ChartSpec> {
@@ -365,15 +490,40 @@ pub(crate) fn spec_of(space: &El) -> Option<ChartSpec> {
let categories = sers.first().and_then(|s| s.child(if scatter { "c:xVal" } else { "c:cat" })).map(literal).unwrap_or_default();
let series = sers
.iter()
.map(|s| ChartSeries {
name: s.child("c:tx").and_then(|t| t.child("c:v")).map(|v| v.text()).unwrap_or_default(),
values: s
.child(if scatter { "c:yVal" } else { "c:val" })
.map(literal)
.unwrap_or_default()
.iter()
.map(|v| v.parse::<f64>().ok().filter(|v| v.is_finite()))
.collect(),
.map(|s| {
let ln = s.child("c:spPr").and_then(|p| p.child("a:ln"));
let symbol = s.child("c:marker").and_then(|m| m.child("c:symbol")).and_then(|m| m.attr("val"));
ChartSeries {
name: s.child("c:tx").and_then(|t| t.child("c:v")).map(|v| v.text()).unwrap_or_default(),
values: s
.child(if scatter { "c:yVal" } else { "c:val" })
.map(literal)
.unwrap_or_default()
.iter()
.map(|v| v.parse::<f64>().ok().filter(|v| v.is_finite()))
.collect(),
fill: s
.child("c:spPr")
.and_then(|p| p.child("a:solidFill"))
.and_then(|f| f.child("a:srgbClr"))
.and_then(|c| c.attr("val"))
.and_then(Rgb::parse),
no_fill: s.child("c:spPr").and_then(|p| p.child("a:noFill")).is_some(),
line_w: ln.and_then(|l| l.attr("w")).and_then(|w| w.parse::<f64>().ok()).filter(|w| w.is_finite()).map(|w| (w / 12700.0) as f32),
no_line: ln.and_then(|l| l.child("a:noFill")).is_some(),
// Line series show no markers by default: only a setting that differs from that
// belongs in the model. A scatter series keeps what the file says, so a missing
// `c:marker` (Word's automatic symbols) stays unset and is not written back.
markers: symbol.map(|v| v != "none").filter(|m| scatter || *m),
smooth: s.child("c:smooth").and_then(|v| v.attr("val")).is_some_and(|v| v == "1"),
labels: s.child("c:dLbls").and_then(|d| d.child("c:showVal")).and_then(|v| v.attr("val")).map(|v| v == "1"),
fmt: s
.child(if scatter { "c:yVal" } else { "c:val" })
.and_then(|v| v.els().find(|e| e.name == "c:numLit"))
.and_then(|l| l.child("c:formatCode"))
.map(|f| f.text())
.filter(|f| *f != "General"),
}
})
.collect();
let title = chart.child("c:title").and_then(|t| t.find("a:t")).map(|t| t.text());
@@ -384,12 +534,48 @@ pub(crate) fn spec_of(space: &El) -> Option<ChartSpec> {
_ => LegendPos::Right,
});
let data_labels = ty.child("c:dLbls").and_then(|d| d.child("c:showVal")).and_then(|v| v.attr("val")) == Some("1");
let spec = ChartSpec { kind, categories, series, title, legend, data_labels }.sanitized();
// Editable only when nothing would be lost: writing the model again gives the same part.
let again = xml::parse(chart_xml(&spec).as_bytes()).ok()?;
let (cat_a, val_a) = crate::read::chart::chart_axes(plot, scatter);
let axis = |a: crate::read::chart::Axis| ChartAxis {
title: a.title,
min: a.min,
max: a.max,
fmt: a.fmt,
gridlines: a.grid,
labels: a.labels,
line: a.line,
};
let spec = ChartSpec { kind, categories, series, title, legend, data_labels, cat_axis: axis(cat_a), val_axis: axis(val_a) }.sanitized();
// Editable only when nothing would be lost: writing the model again gives the same part, as
// this writer or the earlier one wrote it.
let mut again = xml::parse(chart_xml(&spec).as_bytes()).ok()?;
if same(space, &again, 0) {
return Some(spec);
}
as_previous_writer(&mut again);
same(space, &again, 0).then_some(spec)
}
/// `part` (what [`chart_xml`] writes) as the earlier WordCraft writer wrote the same model: it
/// had no axis `c:numFmt`. Only that element goes, and only with the value its absence reads as
/// (General, linked to the source), so a part that matches still holds nothing the model leaves out.
fn as_previous_writer(part: &mut El) {
fn els_mut(e: &mut El) -> impl Iterator<Item = &mut El> {
e.kids.iter_mut().filter_map(|n| match n {
Node::El(e) => Some(e),
Node::Text(_) => None,
})
}
fn drop_els(e: &mut El, drop: impl Fn(&El) -> bool) {
e.kids.retain(|n| !matches!(n, Node::El(c) if drop(c)));
}
let Some(plot) = els_mut(part).find(|e| e.name == "c:chart").and_then(|c| els_mut(c).find(|e| e.name == "c:plotArea")) else { return };
for e in els_mut(plot) {
if matches!(e.name.as_str(), "c:catAx" | "c:valAx") {
drop_els(e, |c| c.name == "c:numFmt" && c.attr("sourceLinked") == Some("1"));
}
}
}
/// The points of a `c:strLit` / `c:numLit` (or a cache) as texts by index, `ptCount` long.
fn literal(e: &El) -> Vec<String> {
let Some(lit) = e.els().find(|c| matches!(c.name.as_str(), "c:strLit" | "c:numLit")) else { return Vec::new() };
@@ -461,10 +647,11 @@ mod tests {
let spec = ChartSpec {
kind: ChartType::Column,
categories: (0..4500).map(|i| i.to_string()).collect(),
series: (0..300).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 4500] }).collect(),
series: (0..300).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 4500], ..Default::default() }).collect(),
title: None,
legend: None,
data_labels: false,
..Default::default()
};
let x = chart_xml(&spec);
let series = x.matches("<c:ser>").count();
@@ -491,6 +678,33 @@ mod tests {
assert_eq!(spec_of(&parse(&plain)), Some(plain));
}
#[test]
fn no_fill_wins_over_a_stored_colour() {
let mut spec = ChartSpec::sample(ChartType::Column);
spec.series[0].fill = Some(Rgb(255, 0, 0));
spec.series[0].no_fill = true;
let x = chart_xml(&spec);
assert!(!x.contains("FF0000"), "colour written next to no_fill");
let back = spec_of(&parse(&spec)).expect("stays editable");
assert!(back.series[0].no_fill);
assert_eq!(back.series[0].fill, None);
}
#[test]
fn an_axis_format_is_limited_in_characters_not_bytes() {
use craft_wordmodel::chart::MAX_FMT;
let cjk = format!("0\"{}\"", "\u{5143}".repeat(90));
let ascii = format!("0\"{}\"", "x".repeat(90));
for fmt in [cjk, ascii] {
assert!(fmt.chars().count() <= MAX_FMT);
let mut spec = ChartSpec::sample(ChartType::Column);
spec.val_axis.fmt = Some(fmt.clone());
let spec = spec.sanitized();
assert_eq!(spec.val_axis.fmt.as_deref(), Some(fmt.as_str()));
assert_eq!(spec_of(&parse(&spec)).as_ref(), Some(&spec), "{} chars", fmt.chars().count());
}
}
#[test]
fn a_chart_with_more_than_the_model_stays_read_only() {
let spec = ChartSpec::sample(ChartType::Column);
@@ -500,6 +714,61 @@ mod tests {
assert_eq!(spec_of(&xml::parse(x.as_bytes()).unwrap()), None);
}
/// Chart parts as the previous WordCraft chart writer saved them (its output for
/// `ChartSpec::sample`): no axis `c:numFmt`, and scatter series with no `c:marker`.
const PREVIOUS_COLUMN: &str = r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?><c:chartSpace xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><c:roundedCorners val="0"/><c:chart><c:autoTitleDeleted val="1"/><c:plotArea><c:layout/><c:barChart><c:barDir val="col"/><c:grouping val="clustered"/><c:varyColors val="0"/><c:ser><c:idx val="0"/><c:order val="0"/><c:tx><c:v>Series 1</c:v></c:tx><c:invertIfNegative val="0"/><c:cat><c:strLit><c:ptCount val="4"/><c:pt idx="0"><c:v>Q1</c:v></c:pt><c:pt idx="1"><c:v>Q2</c:v></c:pt><c:pt idx="2"><c:v>Q3</c:v></c:pt><c:pt idx="3"><c:v>Q4</c:v></c:pt></c:strLit></c:cat><c:val><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>3.2</c:v></c:pt><c:pt idx="1"><c:v>4.1</c:v></c:pt><c:pt idx="2"><c:v>3.6</c:v></c:pt><c:pt idx="3"><c:v>5</c:v></c:pt></c:numLit></c:val></c:ser><c:ser><c:idx val="1"/><c:order val="1"/><c:tx><c:v>Series 2</c:v></c:tx><c:invertIfNegative val="0"/><c:cat><c:strLit><c:ptCount val="4"/><c:pt idx="0"><c:v>Q1</c:v></c:pt><c:pt idx="1"><c:v>Q2</c:v></c:pt><c:pt idx="2"><c:v>Q3</c:v></c:pt><c:pt idx="3"><c:v>Q4</c:v></c:pt></c:strLit></c:cat><c:val><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>2</c:v></c:pt><c:pt idx="1"><c:v>2.7</c:v></c:pt><c:pt idx="2"><c:v>3.9</c:v></c:pt><c:pt idx="3"><c:v>3.3</c:v></c:pt></c:numLit></c:val></c:ser><c:ser><c:idx val="2"/><c:order val="2"/><c:tx><c:v>Series 3</c:v></c:tx><c:invertIfNegative val="0"/><c:cat><c:strLit><c:ptCount val="4"/><c:pt idx="0"><c:v>Q1</c:v></c:pt><c:pt idx="1"><c:v>Q2</c:v></c:pt><c:pt idx="2"><c:v>Q3</c:v></c:pt><c:pt idx="3"><c:v>Q4</c:v></c:pt></c:strLit></c:cat><c:val><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>1.5</c:v></c:pt><c:pt idx="1"><c:v>2.2</c:v></c:pt><c:pt idx="2"><c:v>2.4</c:v></c:pt><c:pt idx="3"><c:v>3.8</c:v></c:pt></c:numLit></c:val></c:ser><c:gapWidth val="150"/><c:axId val="500000001"/><c:axId val="500000002"/></c:barChart><c:catAx><c:axId val="500000001"/><c:scaling><c:orientation val="minMax"/></c:scaling><c:delete val="0"/><c:axPos val="b"/><c:majorTickMark val="none"/><c:minorTickMark val="none"/><c:tickLblPos val="nextTo"/><c:crossAx val="500000002"/><c:crosses val="autoZero"/><c:auto val="1"/><c:lblAlgn val="ctr"/><c:lblOffset val="100"/><c:noMultiLvlLbl val="0"/></c:catAx><c:valAx><c:axId val="500000002"/><c:scaling><c:orientation val="minMax"/></c:scaling><c:delete val="0"/><c:axPos val="l"/><c:majorGridlines/><c:majorTickMark val="none"/><c:minorTickMark val="none"/><c:tickLblPos val="nextTo"/><c:spPr><a:ln><a:noFill/></a:ln></c:spPr><c:crossAx val="500000001"/><c:crosses val="autoZero"/><c:crossBetween val="between"/></c:valAx></c:plotArea><c:legend><c:legendPos val="b"/><c:overlay val="0"/></c:legend><c:plotVisOnly val="1"/><c:dispBlanksAs val="gap"/></c:chart><c:spPr><a:solidFill><a:schemeClr val="bg1"/></a:solidFill><a:ln><a:noFill/></a:ln></c:spPr></c:chartSpace>"#;
const PREVIOUS_SCATTER: &str = r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?><c:chartSpace xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><c:roundedCorners val="0"/><c:chart><c:autoTitleDeleted val="1"/><c:plotArea><c:layout/><c:scatterChart><c:scatterStyle val="lineMarker"/><c:varyColors val="0"/><c:ser><c:idx val="0"/><c:order val="0"/><c:tx><c:v>Series 1</c:v></c:tx><c:spPr><a:ln w="19050"><a:noFill/></a:ln></c:spPr><c:xVal><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>1</c:v></c:pt><c:pt idx="1"><c:v>2</c:v></c:pt><c:pt idx="2"><c:v>3</c:v></c:pt><c:pt idx="3"><c:v>4</c:v></c:pt></c:numLit></c:xVal><c:yVal><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>2.1</c:v></c:pt><c:pt idx="1"><c:v>3.4</c:v></c:pt><c:pt idx="2"><c:v>3</c:v></c:pt><c:pt idx="3"><c:v>4.6</c:v></c:pt></c:numLit></c:yVal><c:smooth val="0"/></c:ser><c:ser><c:idx val="1"/><c:order val="1"/><c:tx><c:v>Series 2</c:v></c:tx><c:spPr><a:ln w="19050"><a:noFill/></a:ln></c:spPr><c:xVal><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>1</c:v></c:pt><c:pt idx="1"><c:v>2</c:v></c:pt><c:pt idx="2"><c:v>3</c:v></c:pt><c:pt idx="3"><c:v>4</c:v></c:pt></c:numLit></c:xVal><c:yVal><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="4"/><c:pt idx="0"><c:v>1.2</c:v></c:pt><c:pt idx="1"><c:v>1.9</c:v></c:pt><c:pt idx="2"><c:v>2.8</c:v></c:pt><c:pt idx="3"><c:v>3.1</c:v></c:pt></c:numLit></c:yVal><c:smooth val="0"/></c:ser><c:axId val="500000001"/><c:axId val="500000002"/></c:scatterChart><c:valAx><c:axId val="500000001"/><c:scaling><c:orientation val="minMax"/></c:scaling><c:delete val="0"/><c:axPos val="b"/><c:majorTickMark val="none"/><c:minorTickMark val="none"/><c:tickLblPos val="nextTo"/><c:crossAx val="500000002"/><c:crosses val="autoZero"/><c:crossBetween val="midCat"/></c:valAx><c:valAx><c:axId val="500000002"/><c:scaling><c:orientation val="minMax"/></c:scaling><c:delete val="0"/><c:axPos val="l"/><c:majorGridlines/><c:majorTickMark val="none"/><c:minorTickMark val="none"/><c:tickLblPos val="nextTo"/><c:spPr><a:ln><a:noFill/></a:ln></c:spPr><c:crossAx val="500000001"/><c:crosses val="autoZero"/><c:crossBetween val="midCat"/></c:valAx></c:plotArea><c:legend><c:legendPos val="b"/><c:overlay val="0"/></c:legend><c:plotVisOnly val="1"/><c:dispBlanksAs val="gap"/></c:chart><c:spPr><a:solidFill><a:schemeClr val="bg1"/></a:solidFill><a:ln><a:noFill/></a:ln></c:spPr></c:chartSpace>"#;
#[test]
fn charts_saved_by_the_previous_writer_stay_editable() {
for (x, kind) in [(PREVIOUS_COLUMN, ChartType::Column), (PREVIOUS_SCATTER, ChartType::Scatter)] {
let spec = spec_of(&xml::parse(x.as_bytes()).unwrap()).unwrap_or_else(|| panic!("{kind:?}: the earlier part is read-only"));
assert_eq!((spec.kind, spec.series.len()), (kind, if kind == ChartType::Scatter { 2 } else { 3 }));
assert_eq!((spec.cat_axis.fmt.as_deref(), spec.val_axis.fmt.as_deref()), (None, None), "{kind:?}");
assert!(spec.series.iter().all(|s| s.markers.is_none()), "{kind:?}: markers unset");
// Saved again, it is in the current shape and still reads back as the same model.
assert_eq!(spec_of(&parse(&spec)).as_ref(), Some(&spec), "{kind:?}");
}
// What neither writer writes still keeps such a part read-only: an axis format linked to
// the source that isn't General, a scatter marker other than the default, a marker size.
let edited = |x: &str, from: &str, to: &str| {
assert!(x.contains(from), "{from}");
spec_of(&xml::parse(x.replacen(from, to, 1).as_bytes()).unwrap())
};
let ax = r#"<c:axPos val="l"/><c:majorGridlines/>"#;
assert_eq!(edited(PREVIOUS_COLUMN, ax, &format!(r#"{ax}<c:numFmt formatCode="0.0" sourceLinked="1"/>"#)), None);
let ln = r#"<a:ln w="19050"><a:noFill/></a:ln></c:spPr>"#;
assert_eq!(edited(PREVIOUS_SCATTER, ln, &format!(r#"{ln}<c:marker><c:symbol val="square"/></c:marker>"#)), None);
assert_eq!(edited(PREVIOUS_SCATTER, ln, &format!(r#"{ln}<c:marker><c:symbol val="circle"/><c:size val="9"/></c:marker>"#)), None);
assert_eq!(edited(PREVIOUS_COLUMN, "<c:roundedCorners val=\"0\"/>", "<c:roundedCorners val=\"0\"/><c:style val=\"2\"/>"), None);
}
/// A scatter series with no `c:marker` (Word's automatic symbols) stays without one after an
/// unrelated edit and a save; an explicit circle or "none" stays explicit.
#[test]
fn scatter_series_without_a_marker_stay_without_after_a_save() {
let marker_of = |x: &El| -> Vec<Option<String>> {
let sc = x.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:scatterChart").unwrap();
sc.children("c:ser").map(|s| s.child("c:marker").map(|m| m.child("c:symbol").unwrap().attr("val").unwrap().to_string())).collect()
};
let mut spec = spec_of(&xml::parse(PREVIOUS_SCATTER.as_bytes()).unwrap()).unwrap();
assert_eq!(marker_of(&xml::parse(PREVIOUS_SCATTER.as_bytes()).unwrap()), [None, None]);
spec.title = Some("Edited".into());
spec.series[0].name = "Renamed".into();
let saved = parse(&spec);
assert_eq!(marker_of(&saved), [None, None]);
let again = spec_of(&saved).unwrap();
assert_eq!(again, spec);
assert!(again.series.iter().all(|s| s.markers.is_none()));
// Explicit choices are kept as written.
spec.series[0].markers = Some(true);
spec.series[1].markers = Some(false);
let saved = parse(&spec);
assert_eq!(marker_of(&saved), [Some("circle".to_string()), Some("none".to_string())]);
let again = spec_of(&saved).unwrap();
assert_eq!((again.series[0].markers, again.series[1].markers), (Some(true), Some(false)));
}
#[test]
fn elements_come_in_schema_order() {
let mut spec = ChartSpec::sample(ChartType::ColumnStacked);
@@ -517,6 +786,11 @@ mod tests {
["c:barDir", "c:grouping", "c:varyColors", "c:ser", "c:ser", "c:ser", "c:dLbls", "c:gapWidth", "c:overlap", "c:axId", "c:axId"]
);
assert_eq!(names(bar.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:invertIfNegative", "c:cat", "c:val"]);
// A series with its own labels keeps `c:invertIfNegative` ahead of `c:dLbls` (CT_BarSer).
spec.series[0].labels = Some(true);
let labelled = parse(&spec);
let ser = labelled.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:barChart").unwrap().child("c:ser").unwrap();
assert_eq!(names(ser), ["c:idx", "c:order", "c:tx", "c:invertIfNegative", "c:dLbls", "c:cat", "c:val"]);
let val_ax = plot.child("c:valAx").unwrap();
assert_eq!(
names(val_ax),
@@ -526,6 +800,7 @@ mod tests {
"c:delete",
"c:axPos",
"c:majorGridlines",
"c:numFmt",
"c:majorTickMark",
"c:minorTickMark",
"c:tickLblPos",
@@ -535,13 +810,35 @@ mod tests {
"c:crossBetween"
]
);
let cat_ax = plot.child("c:catAx").unwrap();
assert_eq!(
names(cat_ax),
[
"c:axId",
"c:scaling",
"c:delete",
"c:axPos",
"c:numFmt",
"c:majorTickMark",
"c:minorTickMark",
"c:tickLblPos",
"c:crossAx",
"c:crosses",
"c:auto",
"c:lblAlgn",
"c:lblOffset",
"c:noMultiLvlLbl"
]
);
let line = parse(&ChartSpec { data_labels: true, ..ChartSpec::sample(ChartType::Line) });
let lc = line.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:lineChart").unwrap();
assert_eq!(names(lc), ["c:grouping", "c:varyColors", "c:ser", "c:ser", "c:ser", "c:dLbls", "c:marker", "c:axId", "c:axId"]);
assert_eq!(names(lc.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:marker", "c:cat", "c:val", "c:smooth"]);
let sc = parse(&ChartSpec::sample(ChartType::Scatter));
let mut scatter = ChartSpec::sample(ChartType::Scatter);
scatter.series.iter_mut().for_each(|s| s.markers = Some(true));
let sc = parse(&scatter);
let sc = sc.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:scatterChart").unwrap();
assert_eq!(names(sc.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:spPr", "c:xVal", "c:yVal", "c:smooth"]);
assert_eq!(names(sc.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:spPr", "c:marker", "c:xVal", "c:yVal", "c:smooth"]);
let dn = parse(&ChartSpec::sample(ChartType::Doughnut));
let dn = dn.child("c:chart").unwrap().child("c:plotArea").unwrap();
assert_eq!(names(dn), ["c:layout", "c:doughnutChart"]);
+3 -1
View File
@@ -29,7 +29,9 @@ pub(super) use canvas::text_w;
use canvas::{Rect, clip_text, legend, text};
#[cfg(test)]
use format::{civil, excel_stamp, format_value};
use model::{Axis, Grouping, Kind, Pos, Series};
pub(crate) use model::Axis;
use model::{Grouping, Kind, Pos, Series};
pub(crate) use parse::chart_axes;
#[cfg(test)]
use parse::{axis, category_text, labels, numbers};
use parse::{read_series, text_size, title_of};
+6 -1
View File
@@ -41,8 +41,13 @@ pub(super) fn widest(items: &[String], size: f32) -> f32 {
items.iter().map(|s| text_w(s, size)).fold(0.0, f32::max)
}
/// Text in `rect`; `angle` degrees clockwise turns it about the rect's centre (axis titles).
pub(super) fn text(out: &mut Vec<GraphicItem>, rect: Rect, s: &str, size: f32, align: TextAlign, bold: bool) {
out.push(GraphicItem::Text { rect: rect.arr(), text: s.to_string(), size, color: TEXT, bold, align, font: None });
text_angle(out, rect, s, size, align, bold, 0.0);
}
pub(super) fn text_angle(out: &mut Vec<GraphicItem>, rect: Rect, s: &str, size: f32, align: TextAlign, bold: bool, angle: f32) {
out.push(GraphicItem::Text { rect: rect.arr(), text: s.to_string(), size, color: TEXT, bold, align, font: None, angle });
}
/// Text centred on `cx` with its top at `y`.
+94 -8
View File
@@ -7,6 +7,33 @@ pub(super) struct DataLabel {
pub(super) y: f32,
pub(super) text: String,
pub(super) inside: bool,
/// Where a label that is not `inside` sits relative to (`x`, `y`).
pub(super) pos: LabelPos,
/// Distance from (`x`, `y`) to the label for `pos` other than `Center`, points (half the marker).
pub(super) off: f32,
}
/// Where a data label sits beside its point (`c:dLblPos`: `t`, `b`, `l`, `r`, `ctr`).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum LabelPos {
Above,
Below,
Left,
Right,
Center,
}
impl LabelPos {
pub(super) fn parse(v: &str) -> Option<LabelPos> {
Some(match v.trim() {
"t" => LabelPos::Above,
"b" => LabelPos::Below,
"l" => LabelPos::Left,
"r" => LabelPos::Right,
"ctr" => LabelPos::Center,
_ => return None,
})
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -52,16 +79,56 @@ pub(super) enum Pos {
/// One value axis (or category axis) as the file states it.
#[derive(Default)]
pub(super) struct Axis {
pub(super) deleted: bool,
pub(super) grid: bool,
pub(super) line: bool,
pub(crate) struct Axis {
pub(crate) deleted: bool,
pub(crate) grid: bool,
pub(crate) line: bool,
/// `c:crossBetween val="midCat"`: categories sit on the tick marks, not between them.
pub(super) mid: bool,
pub(super) min: Option<f64>,
pub(super) max: Option<f64>,
pub(crate) min: Option<f64>,
pub(crate) max: Option<f64>,
/// Number format of the tick labels, when not linked to the source data.
pub(super) fmt: Option<String>,
pub(crate) fmt: Option<String>,
/// The axis title, when the file gives it one.
pub(crate) title: Option<String>,
/// The title's `a:bodyPr/@rot` in degrees clockwise (`-5400000` is 90° counter-clockwise),
/// when the file gives one.
pub(crate) title_rot: Option<f32>,
/// Tick labels, as `c:tickLblPos` says (`none` hides them; a file that says nothing shows
/// them, as Word does).
pub(crate) labels: bool,
}
/// The shape of a series' point markers (`ST_MarkerStyle`, ECMA-376 §21.2.3.28).
#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)]
pub(super) enum Symbol {
#[default]
Circle,
Square,
Diamond,
Triangle,
X,
Plus,
Star,
Dash,
Dot,
}
impl Symbol {
/// The symbol a `c:symbol` value names; anything unknown (`auto`, `picture`) is a circle.
pub(super) fn parse(name: &str) -> Symbol {
match name {
"square" => Symbol::Square,
"diamond" => Symbol::Diamond,
"triangle" => Symbol::Triangle,
"x" => Symbol::X,
"plus" => Symbol::Plus,
"star" => Symbol::Star,
"dash" => Symbol::Dash,
"dot" => Symbol::Dot,
_ => Symbol::Circle,
}
}
}
pub(super) struct Series {
@@ -70,9 +137,21 @@ pub(super) struct Series {
/// when missing or not a valid number.
pub(super) idx: Option<usize>,
pub(super) color: Option<Rgb>,
pub(super) line_w: f32,
/// Fixed line width in points, `None`: the chart's default.
pub(super) line_w: Option<f32>,
/// Whether the series is drawn as a line (scatter series without one get markers).
pub(super) line: bool,
/// The series' own fill: `Some` colour, `None` colour with `no_fill` set, else the chart's
/// accent for the series.
pub(super) no_fill: bool,
/// Markers on the series' points, when the file says.
pub(super) markers: Option<bool>,
/// The marker shape (`c:marker/c:symbol`; a circle when missing or unknown) and its size in
/// points (`c:marker/c:size`), when the file gives one.
pub(super) symbol: Symbol,
pub(super) marker_size: Option<f32>,
/// Draw the series' line as a smooth curve (`c:smooth`).
pub(super) smooth: bool,
pub(super) values: Vec<Option<f64>>,
/// Scatter x values (index + 1 when the file has none).
pub(super) xs: Vec<Option<f64>>,
@@ -83,6 +162,13 @@ pub(super) struct Series {
pub(super) x_fmt: Option<String>,
/// Data labels on or off for this series (`c:dLbls` of its own), when it says.
pub(super) labels: Option<bool>,
/// The labels' number format (`c:dLbls/c:numFmt`): `None` when not given (the chart type's
/// applies), `Some(None)` when linked to the source, else `Some(Some(code))`.
pub(super) label_fmt: Option<Option<String>>,
/// The data labels' `c:dLblPos`, when the file gives a known one.
pub(super) label_pos: Option<LabelPos>,
/// The marker's own fill (`c:marker/c:spPr`); without it markers take the automatic colour.
pub(super) marker_color: Option<Rgb>,
/// The values as the file has them, for data labels (`values` become shares in a 100%
/// stacked chart); empty when the series shows no labels.
pub(super) raw: Vec<Option<f64>>,
+76 -7
View File
@@ -1,7 +1,7 @@
use super::format::{date_text, excel_stamp, format_value, is_date_code};
use super::model::{Axis, Grouping, Kind, Pos, Series};
use super::model::{Axis, Grouping, Kind, LabelPos, Pos, Series, Symbol};
use super::plot::label;
use super::{Chart, El, FONT, LINE_W, MAX_CELLS, MAX_POINTS, MAX_RADAR_SERIES, MAX_SERIES, MAX_SPOKES, Rgb, fill_colour, int, measure, slot};
use super::{Chart, El, FONT, MAX_CELLS, MAX_POINTS, MAX_RADAR_SERIES, MAX_SERIES, MAX_SPOKES, Rgb, fill_colour, int, measure, slot};
impl Chart {
pub(super) fn parse(space: &El, theme: &[Rgb]) -> Option<Chart> {
@@ -39,10 +39,15 @@ impl Chart {
cats.resize(n, String::new());
let radar_style = ty.child("c:radarStyle").and_then(|r| r.attr("val"));
let type_labels = show_values(ty.child("c:dLbls"));
let type_fmt = label_format(ty.child("c:dLbls"));
for s in &mut series {
s.values.resize(n, None);
s.xs.resize(n, None);
s.labels = Some(s.labels.or(type_labels).unwrap_or(false));
// Absent (`None`) inherits the chart type's; explicitly source-linked (`Some(None)`) does not.
if s.label_fmt.is_none() {
s.label_fmt.clone_from(&type_fmt);
}
if s.labels == Some(true) {
s.raw = s.values.clone();
}
@@ -98,6 +103,13 @@ impl Chart {
s.and_then(|s| s.color).unwrap_or_else(|| self.accent(s.and_then(|s| s.idx).unwrap_or(j)))
}
/// Colour of series `j`'s markers: the marker's own fill, else the automatic colour of the
/// series index (not the series' explicit line colour, as Word draws them).
pub(super) fn marker_colour(&self, j: usize) -> Rgb {
let s = self.series.get(j);
s.and_then(|s| s.marker_color).unwrap_or_else(|| self.accent(s.and_then(|s| s.idx).unwrap_or(j)))
}
/// Colour of point `i` of series `j`: its `c:dPt` override, else the varied accent (pie) or
/// the series colour.
pub(super) fn point_colour(&self, j: usize, i: usize) -> Rgb {
@@ -149,7 +161,7 @@ fn to_percent(series: &mut [Series], n: usize) {
}
/// The category axis (scatter: the horizontal value axis) and the value axis of a plot area.
fn chart_axes(plot: &El, scatter: bool) -> (Axis, Axis) {
pub(crate) fn chart_axes(plot: &El, scatter: bool) -> (Axis, Axis) {
let vals_ax: Vec<&El> = plot.children("c:valAx").collect();
let horiz_pos = |a: &&El| matches!(a.child("c:axPos").and_then(|p| p.attr("val")), Some("b" | "t"));
let (val_el, x_el) = if scatter {
@@ -180,6 +192,17 @@ fn show_values(d: Option<&El>) -> Option<bool> {
d.child("c:showVal").map(|v| v.attr("val").is_none_or(truthy))
}
/// The number format of data labels (`c:dLbls/c:numFmt`): `None` when the labels say nothing,
/// `Some(None)` when they are linked to the source (so a chart-level format does not apply),
/// `Some(Some(code))` for their own format code.
fn label_format(d: Option<&El>) -> Option<Option<String>> {
let f = d?.child("c:numFmt")?;
if f.attr("sourceLinked").is_some_and(truthy) {
return Some(None);
}
f.attr("formatCode").filter(|f| f.chars().count() <= craft_wordmodel::chart::MAX_FMT).map(|f| Some(f.to_string()))
}
fn legend_pos(chart: &El) -> Option<Pos> {
chart.child("c:legend").map(|l| match l.child("c:legendPos").and_then(|p| p.attr("val")).unwrap_or("r") {
"l" => Pos::Left,
@@ -206,9 +229,24 @@ impl Series {
let sp = ser.child("c:spPr");
let ln = sp.and_then(|p| p.child("a:ln"));
let marker = ser.child("c:marker").and_then(|m| m.child("c:spPr")).and_then(|p| fill_colour(p, theme));
let marker_color = marker;
let color = sp.and_then(|p| fill_colour(p, theme)).or_else(|| ln.and_then(|l| fill_colour(l, theme))).or(marker);
let line_w = ln.and_then(|l| l.attr("w")).and_then(|v| measure(v, 12_700.0)).map_or(LINE_W, |w| w.clamp(0.0, 20.0));
let no_fill = sp.and_then(|p| p.child("a:noFill")).is_some();
let line_w = ln.and_then(|l| l.attr("w")).and_then(|v| measure(v, 12_700.0)).map(|w| w.clamp(0.0, 20.0));
let line = ln.is_none_or(|l| l.child("a:noFill").is_none());
// `c:marker/c:symbol`: which symbol each point gets (`none` for no markers). Scatter
// series show markers unless the file says not; line series only when it says so.
let symbol = ser.child("c:marker").and_then(|m| m.child("c:symbol")).and_then(|s| s.attr("val")).map(str::to_string);
let markers = symbol.as_deref().map(|s| s != "none").or_else(|| scatter.then_some(true));
let symbol = symbol.as_deref().map_or(Symbol::Circle, Symbol::parse);
let marker_size = ser
.child("c:marker")
.and_then(|m| m.child("c:size"))
.and_then(|s| s.attr("val"))
.and_then(|v| v.trim().parse::<f32>().ok())
.filter(|v| v.is_finite())
.map(|v| v.clamp(2.0, 72.0));
let smooth = ser.child("c:smooth").and_then(|s| s.attr("val")).is_some_and(truthy);
let (values, fmt) = numbers(ser.child(if scatter { "c:yVal" } else { "c:val" }), cap);
let (mut xs, x_fmt) = if scatter { numbers(ser.child("c:xVal"), cap) } else { (Vec::new(), None) };
if scatter && xs.is_empty() {
@@ -223,7 +261,30 @@ impl Series {
}
}
let labels = show_values(ser.child("c:dLbls"));
Series { name, idx, color, line_w, line, values, xs, points, fmt, x_fmt, labels, raw: Vec::new() }
let label_fmt = label_format(ser.child("c:dLbls"));
let label_pos = ser.child("c:dLbls").and_then(|d| d.child("c:dLblPos")).and_then(|p| p.attr("val")).and_then(LabelPos::parse);
Series {
name,
idx,
color,
line_w,
line,
no_fill,
markers,
symbol,
marker_size,
smooth,
values,
xs,
points,
fmt,
x_fmt,
labels,
label_fmt,
label_pos,
marker_color,
raw: Vec::new(),
}
}
}
@@ -290,7 +351,7 @@ fn pts(cache: &El, cap: usize) -> Vec<Option<String>> {
}
pub(super) fn axis(a: Option<&El>) -> Axis {
let Some(a) = a else { return Axis { deleted: true, ..Default::default() } };
let Some(a) = a else { return Axis { deleted: true, labels: true, line: true, ..Default::default() } };
let scaling = a.child("c:scaling");
let num =
|n: &str| scaling.and_then(|s| s.child(n)).and_then(|e| e.attr("val")).and_then(|v| v.trim().parse::<f64>().ok()).filter(|v| v.is_finite());
@@ -305,8 +366,16 @@ pub(super) fn axis(a: Option<&El>) -> Axis {
.child("c:numFmt")
.filter(|f| f.attr("sourceLinked") != Some("1"))
.and_then(|f| f.attr("formatCode"))
.filter(|f| f.len() <= MAX_FORMAT_CODE)
.filter(|f| f.chars().count() <= craft_wordmodel::chart::MAX_FMT)
.map(str::to_string),
title: a.child("c:title").and_then(title_text),
title_rot: a
.child("c:title")
.and_then(|t| t.find("a:bodyPr"))
.and_then(|b| b.attr("rot"))
.and_then(|v| v.trim().parse::<i64>().ok())
.map(|v| (v as f64 / 60_000.0).clamp(-360.0, 360.0) as f32),
labels: a.child("c:tickLblPos").and_then(|p| p.attr("val")) != Some("none"),
}
}
+246 -32
View File
@@ -1,9 +1,12 @@
use super::canvas::{Rect, fill, rect_segs, runs, seg, stroke, text, text_at, text_w, widest, wrap};
use super::canvas::{Rect, clip_text, fill, rect_segs, runs, seg, stroke, text, text_angle, text_at, text_w, widest, wrap};
use super::format::format_value;
use super::model::{DataLabel, Grouping, Kind, Series};
use super::model::{DataLabel, Grouping, Kind, LabelPos, Series, Symbol};
use super::parse::category_text;
use super::polar::{draw_data_labels, label_of};
use super::{AXIS, Chart, El, GAP, GRID, GraphicItem, LINE, MAX_LABEL, MAX_TICKS, PathSeg, Rgb, ShapeKind, TextAlign, fill_colour, measure};
use super::{AXIS, Chart, El, GAP, GRID, GraphicItem, LINE, LINE_W, MAX_LABEL, MAX_TICKS, PathSeg, Rgb, ShapeKind, TextAlign, fill_colour, measure};
/// Line width of the stroked marker symbols (x, plus, star), points.
const MARKER_STROKE: f32 = 1.5;
pub(super) fn label(s: &str) -> String {
s.chars().take(MAX_LABEL).collect()
@@ -142,6 +145,8 @@ impl Axes {
/// The room around the plot for axis labels, and the category labels as they'll be drawn.
struct Margins {
left: f32,
/// Part of `left` taken by the rotated category-axis title (bars): the labels start after it.
title: f32,
top: f32,
bottom: f32,
/// Category labels in lines (none for scatter charts or a deleted category axis).
@@ -151,22 +156,25 @@ struct Margins {
}
/// The margins the axis labels need around the plot in `area`. Category labels under a column
/// chart wrap to the band; labels that still don't fit show every k-th category.
/// chart wrap to the band; labels that still don't fit show every k-th category. Hidden tick
/// labels take no room; an axis title adds one line on its side.
fn axis_margins(c: &Chart, ax: &Axes, area: Rect) -> Margins {
let fs = c.text_size;
let lh = fs * LINE;
let scatter = c.kind == Kind::Scatter;
let (cat_on, val_on) = (!c.cat.deleted, !c.val.deleted);
let (cat_labels, val_labels) = (cat_on && c.cat.labels, val_on && c.val.labels);
let title = title_left(c, cat_on, val_on, area.w);
let left = if c.horizontal {
if cat_on { widest(&c.cats, fs).min(area.w * 0.4) + GAP } else { 0.0 }
} else if val_on {
if cat_labels { widest(&c.cats, fs).min(area.w * 0.4) + GAP } else { 0.0 }
} else if val_labels {
ax.vticks.iter().map(|(_, s)| text_w(s, fs)).fold(0.0, f32::max) + GAP
} else {
0.0
};
} + title;
let band_w = (area.w - left).max(1.0) / c.n as f32;
let wrapped: Vec<Vec<String>> = if cat_on && !scatter {
let width = if c.horizontal { left - GAP } else { band_w };
let wrapped: Vec<Vec<String>> = if cat_labels && !scatter {
let width = if c.horizontal { left - title - GAP } else { band_w };
c.cats.iter().map(|s| wrap(s, width, fs)).collect()
} else {
Vec::new()
@@ -174,15 +182,48 @@ fn axis_margins(c: &Chart, ax: &Axes, area: Rect) -> Margins {
let widest_line = wrapped.iter().flatten().map(|s| text_w(s, fs)).fold(0.0, f32::max);
let skip = ((widest_line / band_w).ceil() as usize).max(1);
let label_lines = wrapped.iter().map(Vec::len).max().unwrap_or(1) as f32;
let bottom = if cat_on && !c.horizontal {
let bottom = (if cat_labels && !c.horizontal {
if scatter { lh + GAP } else { label_lines * lh + GAP }
} else if c.horizontal && val_on {
} else if c.horizontal && val_labels {
lh + GAP
} else {
0.0
};
let top = if !c.horizontal && val_on { lh / 2.0 } else { 0.0 };
Margins { left, top, bottom, wrapped, skip }
}) + title_bottom(c, cat_on, val_on, lh);
let top = if !c.horizontal && val_labels { lh / 2.0 } else { 0.0 };
Margins { left, title, top, bottom, wrapped, skip }
}
/// Room for the vertical axis title on the plot's left: one line when the axis that sits there
/// (the value axis for columns, the category axis for bars) has a title.
/// A horizontal title (`rot="0"`) gets a strip as wide as its wrapped lines, at most a quarter of
/// `area_w`, so the plot keeps its room.
fn title_left(c: &Chart, cat_on: bool, val_on: bool, area_w: f32) -> f32 {
let side = if c.horizontal { cat_on && c.cat.title.is_some() } else { val_on && c.val.title.is_some() };
if !side {
return 0.0;
}
match flat_title_lines(c, area_w) {
Some(lines) => widest(&lines, c.text_size).min((area_w * 0.25).max(0.0)) + GAP,
None => c.text_size * LINE,
}
}
/// The wrapped lines of the left axis's title when the file keeps it horizontal (`rot="0"`).
fn flat_title_lines(c: &Chart, area_w: f32) -> Option<Vec<String>> {
let left = if c.horizontal { &c.cat } else { &c.val };
let r = left.title_rot?.rem_euclid(360.0);
if !(r < 1.0 || r > 359.0) {
return None;
}
let width = if area_w.is_finite() { (area_w * 0.18).max(1.0) } else { 1.0 };
left.title.as_ref().map(|t| wrap(t, width, c.text_size))
}
/// Room for the horizontal axis title under the plot: one line when the axis that sits there
/// (the category axis for columns, the value axis for bars) has a title.
fn title_bottom(c: &Chart, cat_on: bool, val_on: bool, lh: f32) -> f32 {
let side = if c.horizontal { val_on && c.val.title.is_some() } else { cat_on && c.cat.title.is_some() };
if side { lh } else { 0.0 }
}
/// A bar, column, line, area or scatter chart in `area`: gridlines, series, axes, then labels.
@@ -200,6 +241,9 @@ pub(super) fn cartesian(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
if c.val.grid {
draw_grid(out, &g, &ax.vticks);
}
if c.cat.grid {
draw_cat_grid(out, &g, &ax.xticks, c.kind);
}
let mut labels = Vec::new();
match c.kind {
Kind::Bar => bars(out, c, &g, &mut labels),
@@ -210,12 +254,49 @@ pub(super) fn cartesian(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
}
draw_axes(out, c, &g);
draw_data_labels(out, &labels, c.text_size, g.horiz && c.kind == Kind::Bar);
if !c.val.deleted {
draw_value_labels(out, c, &g, &ax.vticks, Rect { w: m.left, ..area });
if !c.val.deleted && c.val.labels {
let room = title_left(c, !c.cat.deleted, true, area.w);
draw_value_labels(out, c, &g, &ax.vticks, Rect { x: area.x + room, w: m.left - room, ..area });
}
if !c.cat.deleted {
if !c.cat.deleted && c.cat.labels {
draw_category_labels(out, c, &g, &ax.xticks, &m, area);
}
draw_axis_titles(out, c, &g, area);
}
/// The axis titles, one line: the title of the vertical axis (the value axis for columns, the
/// category axis for bars) is turned 90° on the plot's left; the horizontal axis's title sits
/// under the plot.
fn draw_axis_titles(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, area: Rect) {
let lh = c.text_size * LINE;
// The axis drawn along the plot's left (the value axis for columns, the category axis for
// bars) gets its title rotated there; the other axis's title sits under the plot.
let (left, bottom) = if c.horizontal { (&c.cat, &c.val) } else { (&c.val, &c.cat) };
// Word turns the vertical axis's title counter-clockwise (reading upwards) unless the file
// turns it the other way or keeps it horizontal (`rot="0"`, wrapped in its own strip); any
// other rotation would not fit the strip and is read as counter-clockwise.
let up = if left.title_rot.is_some_and(|r| (r.rem_euclid(360.0) - 90.0).abs() < 1.0) { 90.0 } else { -90.0 };
if !left.deleted
&& let Some(lines) = flat_title_lines(c, area.w)
{
let strip = (g.plot.x - area.x - GAP).max(1.0);
let top = g.plot.y + g.plot.h / 2.0 - lines.len() as f32 * lh / 2.0;
for (k, l) in lines.iter().enumerate() {
let l = clip_text(l, strip, c.text_size);
text_angle(out, Rect { x: area.x, y: top + k as f32 * lh, w: strip, h: lh }, &l, c.text_size, TextAlign::Center, false, 0.0);
}
} else if !left.deleted
&& let Some(t) = &left.title
{
let t = clip_text(t, g.plot.h, c.text_size);
text_angle(out, Rect { x: area.x, y: g.plot.y, w: lh, h: g.plot.h }, &t, c.text_size, TextAlign::Center, false, up);
}
if !bottom.deleted
&& let Some(t) = &bottom.title
{
let t = clip_text(t, g.plot.w, c.text_size);
text_angle(out, Rect { x: g.plot.x, y: area.bottom() - lh, w: g.plot.w, h: lh }, &t, c.text_size, TextAlign::Center, false, 0.0);
}
}
/// Major gridlines at the value ticks.
@@ -228,6 +309,31 @@ fn draw_grid(out: &mut Vec<GraphicItem>, g: &Frame, ticks: &[(f64, String)]) {
}
}
/// Gridlines along the category axis: across the plot at each category's centre (where line and
/// area points sit, on the tick marks when the axis crosses "midCat"), at the x ticks of a
/// scatter chart.
fn draw_cat_grid(out: &mut Vec<GraphicItem>, g: &Frame, xticks: &[(f64, String)], kind: Kind) {
let plot = g.plot;
if kind == Kind::Scatter {
for (t, _) in xticks {
let p = g.scat_x(*t);
stroke(out, seg(p, plot.y, p, plot.bottom()), GRID, 0.75);
}
return;
}
for i in 0..g.n {
let segs = if g.horiz {
let p = g.band_start(i) + g.band() / 2.0;
seg(plot.x, p, plot.right(), p)
} else {
// Bars stay in their bands; only line and area points move onto the ticks.
let p = if matches!(kind, Kind::Line | Kind::Area) { g.cat_x(i) } else { g.band_start(i) + g.band() / 2.0 };
seg(p, plot.y, p, plot.bottom())
};
stroke(out, segs, GRID, 0.75);
}
}
/// The category axis line, the plot area's outline and the value axis line, as the file shows them.
fn draw_axes(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame) {
let plot = g.plot;
@@ -271,7 +377,7 @@ fn draw_category_labels(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, xticks
for (i, lines) in m.wrapped.iter().enumerate() {
let top = g.band_start(i) + g.band() / 2.0 - lines.len() as f32 * lh / 2.0;
for (k, s) in lines.iter().enumerate() {
text(out, Rect { x: area.x, y: top + k as f32 * lh, w: m.left - GAP, h: lh }, s, fs, TextAlign::Right, false);
text(out, Rect { x: area.x + m.title, y: top + k as f32 * lh, w: m.left - m.title - GAP, h: lh }, s, fs, TextAlign::Right, false);
}
}
} else {
@@ -361,7 +467,13 @@ fn bars(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<DataL
let (p0, p1) = (g.val(v0), g.val(v1));
let (p0, p1) = (p0.min(p1), p0.max(p1));
let segs = if g.horiz { rect_segs(p0, a, p1, a + thick) } else { rect_segs(a, p0, a + thick, p1) };
fill(out, segs, c.point_colour(j, i));
if !s.no_fill {
fill(out, segs.clone(), c.point_colour(j, i));
}
// An explicit outline shows without a fill too (the series' colour is the line's here).
if let (true, true, Some(w)) = (s.line, s.no_fill, s.line_w) {
stroke(out, segs, c.point_colour(j, i), w);
}
if let Some(text) = label_of(s, i) {
let mid = a + thick / 2.0;
let (x, y) = match (g.horiz, stacked) {
@@ -371,7 +483,7 @@ fn bars(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<DataL
(true, false) => (if v >= 0.0 { p1 } else { p0 }, mid),
(false, false) => (mid, if v >= 0.0 { p0 } else { p1 }),
};
labels.push(DataLabel { x, y, text, inside: stacked });
labels.push(DataLabel { x, y, text, inside: stacked, pos: LabelPos::Above, off: 0.0 });
}
}
}
@@ -401,13 +513,109 @@ fn lines(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<Data
.collect();
for (i, p) in pts.iter().enumerate() {
if let (Some((x, y)), Some(text)) = (p, label_of(s, i)) {
labels.push(DataLabel { x: *x, y: *y, text, inside: false });
labels.push(DataLabel {
x: *x,
y: *y,
text,
inside: false,
pos: s.label_pos.unwrap_or(LabelPos::Right),
off: marker_half(s, s.markers == Some(true)) + 2.0,
});
}
}
stroke(out, runs(pts.into_iter()), c.series_colour(j), s.line_w);
if s.line {
stroke(out, curve(pts.iter().copied(), s.smooth), c.series_colour(j), s.line_w.unwrap_or(LINE_W));
}
if s.markers == Some(true) {
markers(out, pts.iter().flatten().copied(), c.marker_colour(j), s);
}
}
}
/// A series' point markers in the series' symbol and size (`colour` fills or strokes them), for
/// lines and scatter charts.
/// Half a series' marker size, points; zero when it draws no markers.
fn marker_half(s: &Series, drawn: bool) -> f32 {
if drawn { s.marker_size.filter(|v| v.is_finite()).unwrap_or(6.0).clamp(2.0, 72.0) / 2.0 } else { 0.0 }
}
fn markers(out: &mut Vec<GraphicItem>, pts: impl Iterator<Item = (f32, f32)>, colour: Rgb, s: &Series) {
const MARKER: f32 = 6.0;
let size = s.marker_size.filter(|v| v.is_finite()).unwrap_or(MARKER).clamp(2.0, 72.0);
let h = size / 2.0;
let shape = |out: &mut Vec<GraphicItem>, rect: [f32; 4], kind: ShapeKind| {
out.push(GraphicItem::Shape { rect, kind, fill: Some(colour), stroke: None, stroke_width: 0.0 });
};
let mut strokes = Vec::new();
for (x, y) in pts {
match s.symbol {
Symbol::Circle => shape(out, [x - h, y - h, size, size], ShapeKind::Ellipse),
Symbol::Square => shape(out, [x - h, y - h, size, size], ShapeKind::Rectangle),
Symbol::Dot => shape(out, [x - h / 2.0, y - h / 2.0, h, h], ShapeKind::Ellipse),
Symbol::Dash => shape(out, [x - h, y - size / 6.0, size, size / 3.0], ShapeKind::Rectangle),
Symbol::Diamond => fill(
out,
vec![PathSeg::Move(x, y - h), PathSeg::Line(x + h, y), PathSeg::Line(x, y + h), PathSeg::Line(x - h, y), PathSeg::Close],
colour,
),
Symbol::Triangle => {
fill(out, vec![PathSeg::Move(x, y - h), PathSeg::Line(x + h, y + h), PathSeg::Line(x - h, y + h), PathSeg::Close], colour)
}
Symbol::X | Symbol::Plus | Symbol::Star => {
if s.symbol != Symbol::Plus {
strokes.extend(seg(x - h, y - h, x + h, y + h));
strokes.extend(seg(x - h, y + h, x + h, y - h));
}
if s.symbol != Symbol::X {
strokes.extend(seg(x - h, y, x + h, y));
strokes.extend(seg(x, y - h, x, y + h));
}
}
}
}
stroke(out, strokes, colour, MARKER_STROKE);
}
/// A polyline through `pts`, a smooth curve (Catmull-Rom pieces) when `smooth` and there are at
/// least three points in a row; gaps (`None`) end a run.
fn curve(pts: impl Iterator<Item = Option<(f32, f32)>>, smooth: bool) -> Vec<PathSeg> {
if !smooth {
return runs(pts);
}
let pts: Vec<Option<(f32, f32)>> = pts.collect();
let mut segs = Vec::new();
let mut i = 0;
while i < pts.len() {
let Some(first) = pts[i] else {
i += 1;
continue;
};
// Collect one run of points.
let mut run = vec![first];
while let Some(p) = pts.get(i + 1).copied().flatten() {
run.push(p);
i += 1;
}
i += 1;
if run.len() < 3 {
segs.extend(runs(run.into_iter().map(Some)));
continue;
}
segs.push(PathSeg::Move(run[0].0, run[0].1));
for k in 0..run.len() - 1 {
// Catmull-Rom: tangent at p1 from p0 to p2 (ends mirror their own tangent).
let p0 = run[k.max(1) - 1];
let p1 = run[k];
let p2 = run[k + 1];
let p3 = run.get(k + 2).copied().unwrap_or(p2);
let c1 = (p1.0 + (p2.0 - p0.0) / 6.0, p1.1 + (p2.1 - p0.1) / 6.0);
let c2 = (p2.0 - (p3.0 - p1.0) / 6.0, p2.1 - (p3.1 - p1.1) / 6.0);
segs.push(PathSeg::Cubic(c1.0, c1.1, c2.0, c2.1, p2.0, p2.1));
}
}
segs
}
fn areas(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<DataLabel>) {
let mut base = vec![0.0f64; g.n];
for (j, s) in c.series.iter().enumerate().filter(|(_, s)| has_values(s)) {
@@ -422,10 +630,12 @@ fn areas(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<Data
segs.push(PathSeg::Line(g.cat_x(i), g.val(*b)));
}
segs.push(PathSeg::Close);
fill(out, segs, c.series_colour(j));
if !s.no_fill {
fill(out, segs, c.series_colour(j));
}
for (i, (t, b)) in top.iter().zip(&base).enumerate() {
if let Some(text) = label_of(s, i) {
labels.push(DataLabel { x: g.cat_x(i), y: g.val((t + b) / 2.0), text, inside: true });
labels.push(DataLabel { x: g.cat_x(i), y: g.val((t + b) / 2.0), text, inside: true, pos: LabelPos::Above, off: 0.0 });
}
}
if c.stacked() {
@@ -434,9 +644,8 @@ fn areas(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<Data
}
}
/// Scatter series as lines, or as small markers when the series has no line.
/// Scatter series as lines, and as small markers when the series asks for them.
fn draw_scatter(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut Vec<DataLabel>) {
const MARKER: f32 = 6.0;
for (j, s) in c.series.iter().enumerate() {
let pts: Vec<Option<(f32, f32)>> =
s.xs.iter()
@@ -448,16 +657,21 @@ fn draw_scatter(out: &mut Vec<GraphicItem>, c: &Chart, g: &Frame, labels: &mut V
.collect();
for (i, p) in pts.iter().enumerate() {
if let (Some((x, y)), Some(text)) = (p, label_of(s, i)) {
labels.push(DataLabel { x: *x, y: *y - MARKER / 2.0, text, inside: false });
labels.push(DataLabel {
x: *x,
y: *y,
text,
inside: false,
pos: s.label_pos.unwrap_or(LabelPos::Right),
off: marker_half(s, s.markers != Some(false)) + 2.0,
});
}
}
if s.line {
stroke(out, runs(pts.iter().copied()), c.series_colour(j), s.line_w);
continue;
stroke(out, curve(pts.iter().copied(), s.smooth), c.series_colour(j), s.line_w.unwrap_or(LINE_W));
}
for (x, y) in pts.iter().flatten() {
let rect = [x - MARKER / 2.0, y - MARKER / 2.0, MARKER, MARKER];
out.push(GraphicItem::Shape { rect, kind: ShapeKind::Ellipse, fill: Some(c.series_colour(j)), stroke: None, stroke_width: 0.0 });
if s.markers != Some(false) {
markers(out, pts.iter().flatten().copied(), c.marker_colour(j), s);
}
}
}
+17 -8
View File
@@ -1,8 +1,8 @@
use super::canvas::{Rect, clip_text, fill, stroke, text, text_w, widest};
use super::format::format_value;
use super::model::{DataLabel, Kind, Series};
use super::model::{DataLabel, Kind, LabelPos, Series};
use super::plot::{Axes, has_values};
use super::{Chart, FRAC_PI_2, GAP, GRID, GraphicItem, LINE, PathSeg, Rgb, ShapeKind, TAU, TEXT, TextAlign};
use super::{Chart, FRAC_PI_2, GAP, GRID, GraphicItem, LINE, LINE_W, PathSeg, Rgb, ShapeKind, TAU, TEXT, TextAlign};
pub(super) fn pie(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
let Some(s) = c.series.first() else { return };
@@ -38,7 +38,7 @@ pub(super) fn pie(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
// Halfway across the ring (pie: two thirds out from the centre).
let at = if inner > 0.0 { (r + inner) / 2.0 } else { r * 0.65 };
let (x, y) = polar(cx, cy, at, (a + b) / 2.0);
labels.push(DataLabel { x: x as f32, y: y as f32, text, inside: true });
labels.push(DataLabel { x: x as f32, y: y as f32, text, inside: true, pos: LabelPos::Above, off: 0.0 });
}
a = b;
}
@@ -92,14 +92,14 @@ pub(super) fn radar(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
if c.radar_fill {
fill(out, segs.clone(), col);
}
stroke(out, segs, col, s.line_w);
stroke(out, segs, col, s.line_w.unwrap_or(LINE_W));
for (i, p) in ring.iter().enumerate() {
if c.radar_marks && s.values.get(i).copied().flatten().is_some() {
let rect = [p.0 as f32 - MARKER / 2.0, p.1 as f32 - MARKER / 2.0, MARKER, MARKER];
out.push(GraphicItem::Shape { rect, kind: ShapeKind::Ellipse, fill: Some(col), stroke: None, stroke_width: 0.0 });
}
if let Some(text) = label_of(s, i) {
labels.push(DataLabel { x: p.0 as f32, y: p.1 as f32, text, inside: false });
labels.push(DataLabel { x: p.0 as f32, y: p.1 as f32, text, inside: false, pos: LabelPos::Above, off: 0.0 });
}
}
}
@@ -134,13 +134,13 @@ pub(super) fn radar(out: &mut Vec<GraphicItem>, c: &Chart, area: Rect) {
}
/// The data label of point `i` of series `s`, when it shows one: the value as the file has it,
/// in the series' number format.
/// in the labels' own number format, else the series' one.
pub(super) fn label_of(s: &Series, i: usize) -> Option<String> {
if s.labels != Some(true) {
return None;
}
let v = s.raw.get(i).copied().flatten()?;
let fmt = s.fmt.as_deref().filter(|f| !f.eq_ignore_ascii_case("General"));
let fmt = s.label_fmt.as_ref().map_or(s.fmt.as_deref(), |f| f.as_deref().or(s.fmt.as_deref())).filter(|f| !f.eq_ignore_ascii_case("General"));
Some(match fmt {
Some(f) => format_value(v, f, 1.0),
None => general_value(v),
@@ -165,13 +165,22 @@ pub(super) fn draw_data_labels(out: &mut Vec<GraphicItem>, labels: &[DataLabel],
let w = text_w(&l.text, fs) + 4.0;
let (rect, align) = if l.inside {
(Rect { x: l.x - w / 2.0, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Center)
} else if l.off > 0.0 || l.pos != LabelPos::Above {
let o = l.off;
match l.pos {
LabelPos::Right => (Rect { x: l.x + o, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Left),
LabelPos::Left => (Rect { x: l.x - o - w, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Right),
LabelPos::Below => (Rect { x: l.x - w / 2.0, y: l.y + o, w, h: lh }, TextAlign::Center),
LabelPos::Center => (Rect { x: l.x - w / 2.0, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Center),
LabelPos::Above => (Rect { x: l.x - w / 2.0, y: l.y - o - lh, w, h: lh }, TextAlign::Center),
}
} else if beside_right {
(Rect { x: l.x + GAP / 2.0, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Left)
} else {
(Rect { x: l.x - w / 2.0, y: l.y - lh, w, h: lh }, TextAlign::Center)
};
let color = if l.inside { Rgb::WHITE } else { TEXT };
out.push(GraphicItem::Text { rect: rect.arr(), text: l.text.clone(), size: fs, color, bold: false, align, font: None });
out.push(GraphicItem::Text { rect: rect.arr(), text: l.text.clone(), size: fs, color, bold: false, align, font: None, angle: 0.0 });
}
}
+311 -2
View File
@@ -386,9 +386,13 @@ fn axis_format_code_length_is_capped() {
let x = format!(r#"<c:valAx {NS}><c:numFmt formatCode="{code}" sourceLinked="0"/></c:valAx>"#);
axis(Some(&crate::xml::parse(x.as_bytes()).unwrap())).fmt
};
let ok = format!("{}0", "x".repeat(254));
use craft_wordmodel::chart::MAX_FMT;
// The limit is the model's, counted in characters (a CJK character is one).
let ok = format!("{}0", "x".repeat(MAX_FMT - 1));
assert_eq!(fmt(&ok), Some(ok.clone()));
assert_eq!(fmt(&format!("{}0", "x".repeat(300))), None);
let cjk = format!("{}0", "\u{5143}".repeat(MAX_FMT - 1));
assert_eq!(fmt(&cjk), Some(cjk.clone()));
assert_eq!(fmt(&format!("{}0", "x".repeat(MAX_FMT))), None);
}
#[test]
@@ -590,3 +594,308 @@ fn series_without_a_usable_c_idx_take_their_place() {
let big = [renumbered(&a, &format!(r#"<c:idx val="{}"/>"#, u32::MAX))];
assert_eq!(column_colours(&big).0.len(), 1);
}
#[test]
fn bar_category_labels_start_after_the_axis_title() {
let long = "A category label that is much too long for its column";
let title = r#"<c:title><c:tx><c:rich><a:p><a:r><a:t>Region</a:t></a:r></a:p></c:rich></c:tx></c:title>"#;
let plot = format!(
r#"<c:barChart><c:barDir val="bar"/><c:grouping val="clustered"/>{}<c:axId val="1"/><c:axId val="2"/></c:barChart><c:catAx><c:axId val="1"/>{title}</c:catAx><c:valAx><c:axId val="2"/></c:valAx>"#,
ser(0, "S", Some("FF0000"), &[long, "Short"], &["4", "8"]),
);
let items = chart_items(&space(&plot), &theme(), 300.0, 200.0);
let rect_of = |f: &dyn Fn(&str, &TextAlign) -> bool| -> Vec<[f32; 4]> {
items
.iter()
.filter_map(|it| if let GraphicItem::Text { rect, text, align, .. } = it { f(text, align).then_some(*rect) } else { None })
.collect()
};
let title_rect = rect_of(&|t, _| t == "Region");
assert_eq!(title_rect.len(), 1);
let strip_end = title_rect[0][0] + title_rect[0][2];
let labels = rect_of(&|t, a| matches!(a, TextAlign::Right) && t != "Region" && (long.contains(t) || t == "Short"));
assert!(!labels.is_empty());
assert!(labels.iter().all(|r| r[0] >= strip_end - 1e-3), "labels {labels:?} overlap the title strip ending at {strip_end}");
assert!(labels.iter().all(|r| r[2] <= 0.4 * 300.0), "{labels:?}");
}
#[test]
fn category_gridlines_follow_the_points_when_the_axis_crosses_mid() {
let line = |cross: &str| {
let plot = format!(
r#"<c:lineChart><c:grouping val="standard"/>{}<c:axId val="1"/><c:axId val="2"/></c:lineChart><c:catAx><c:axId val="1"/><c:majorGridlines/></c:catAx><c:valAx><c:axId val="2"/><c:crossBetween val="{cross}"/></c:valAx>"#,
ser(0, "L", Some("FF0000"), &["Q1", "Q2", "Q3"], &["1", "5", "3"]),
);
let items = chart_items(&space(&plot), &theme(), 300.0, 200.0);
let grid: Vec<f32> = items
.iter()
.filter_map(|it| match it {
GraphicItem::Path { segs, stroke: Some(c), .. } if *c == GRID => match segs.first() {
Some(PathSeg::Move(x, _)) => Some(*x),
_ => None,
},
_ => None,
})
.collect();
let pts: Vec<f32> = items
.iter()
.filter_map(|it| match it {
GraphicItem::Path { segs, stroke: Some(c), .. } if *c == Rgb(255, 0, 0) => Some(
segs.iter().filter_map(|s| if let PathSeg::Move(x, _) | PathSeg::Line(x, _) = s { Some(*x) } else { None }).collect::<Vec<f32>>(),
),
_ => None,
})
.next()
.unwrap_or_default();
(grid, pts)
};
for cross in ["midCat", "between"] {
let (grid, pts) = line(cross);
assert_eq!(grid.len(), 3, "{cross}");
assert_eq!(pts.len(), 3, "{cross}");
assert!(grid.iter().zip(&pts).all(|(g, p)| (g - p).abs() < 1e-3), "{cross}: grid {grid:?} points {pts:?}");
}
}
/// A line chart with one series; `extra` goes into the series (`c:marker`, `c:dLbls`), `axes`
/// into the value axis.
fn line_with(extra: &str, val_axis: &str) -> Vec<GraphicItem> {
let s = ser(0, "S", Some("FF0000"), &["Q1", "Q2", "Q3"], &["3.2", "4.1", "5"]).replace("<c:cat>", &format!("{extra}<c:cat>"));
let plot = format!(
r#"<c:lineChart><c:grouping val="standard"/>{s}<c:axId val="1"/><c:axId val="2"/></c:lineChart><c:catAx><c:axId val="1"/></c:catAx><c:valAx><c:axId val="2"/>{val_axis}</c:valAx>"#
);
chart_items(&space(&plot), &theme(), 300.0, 200.0)
}
fn marker_xml(symbol: &str, size: u32) -> String {
format!(r#"<c:marker><c:symbol val="{symbol}"/><c:size val="{size}"/></c:marker>"#)
}
#[test]
fn marker_symbols_are_drawn_as_their_shape() {
let shapes = |items: &[GraphicItem], kind: ShapeKind| {
items
.iter()
.filter(|it| matches!(it, GraphicItem::Shape { kind: k, rect, .. } if *k == kind && rect[2] > 3.0 && rect[2] < 20.0 && rect[3] > 1.0))
.count()
};
let paths = |items: &[GraphicItem], filled: bool| {
items
.iter()
.filter(|it| matches!(it, GraphicItem::Path { fill, segs, .. } if fill.is_some() == filled && segs.len() <= 12 && segs.len() >= 3))
.count()
};
// One marker per point, in the series' size.
let circle = line_with(&marker_xml("circle", 9), "");
assert_eq!(shapes(&circle, ShapeKind::Ellipse), 3);
assert!(circle.iter().any(|it| matches!(it, GraphicItem::Shape { kind: ShapeKind::Ellipse, rect, .. } if (rect[2] - 9.0).abs() < 1e-3)));
let square = line_with(&marker_xml("square", 7), "");
assert_eq!(shapes(&square, ShapeKind::Rectangle) - shapes(&circle, ShapeKind::Rectangle), 3, "a square per point");
assert_eq!(shapes(&square, ShapeKind::Ellipse), 0, "no circles");
// Diamonds and triangles are filled polygons; x, plus and star are strokes.
let base = paths(&line_with("", ""), true);
assert_eq!(paths(&line_with(&marker_xml("diamond", 7), ""), true) - base, 3);
assert_eq!(paths(&line_with(&marker_xml("triangle", 7), ""), true) - base, 3);
for symbol in ["x", "plus", "star"] {
let items = line_with(&marker_xml(symbol, 7), "");
assert!(items.iter().any(|it| matches!(it, GraphicItem::Path { fill: None, stroke: Some(_), segs, .. } if segs.len() >= 6)), "{symbol}");
assert_eq!(shapes(&items, ShapeKind::Ellipse), 0, "{symbol} is not a circle");
}
// Dash and dot are small shapes; `none` and unknown names never crash (unknown: a circle).
assert_eq!(shapes(&line_with(&marker_xml("dash", 7), ""), ShapeKind::Rectangle) - shapes(&circle, ShapeKind::Rectangle), 3);
assert_eq!(shapes(&line_with(&marker_xml("dot", 7), ""), ShapeKind::Ellipse), 3);
assert_eq!(shapes(&line_with(&marker_xml("none", 7), ""), ShapeKind::Ellipse), 0);
assert_eq!(shapes(&line_with(&marker_xml("wibble", 7), ""), ShapeKind::Ellipse), 3);
// Hostile sizes are clamped.
for size in ["0", "99999999999", "-4", "x"] {
let items = line_with(&format!(r#"<c:marker><c:symbol val="square"/><c:size val="{size}"/></c:marker>"#), "");
assert!(items.iter().all(|it| match it {
GraphicItem::Shape { rect, .. } => rect.iter().all(|v| v.is_finite()) && rect[2] <= 300.0,
_ => true,
}));
}
}
#[test]
fn data_labels_use_their_own_format_code() {
let dl = |num: &str| format!(r#"<c:dLbls>{num}<c:showVal val="1"/></c:dLbls>"#);
let shown = |items: &[GraphicItem]| -> Vec<String> {
items.iter().filter_map(|it| if let GraphicItem::Text { text, .. } = it { Some(text.clone()) } else { None }).collect()
};
let own = shown(&line_with(&dl(r#"<c:numFmt formatCode="0.00" sourceLinked="0"/>"#), ""));
assert!(own.contains(&"3.20".to_string()) && own.contains(&"5.00".to_string()), "{own:?}");
// Linked to the source: the series' format (General here).
let linked = shown(&line_with(&dl(r#"<c:numFmt formatCode="0.00" sourceLinked="1"/>"#), ""));
assert!(linked.contains(&"3.2".to_string()) && linked.contains(&"5".to_string()), "{linked:?}");
// A format code beyond the cap is ignored, not applied.
let long = "0".repeat(craft_wordmodel::chart::MAX_FMT + 1);
let capped = shown(&line_with(&dl(&format!(r#"<c:numFmt formatCode="{long}" sourceLinked="0"/>"#)), ""));
assert!(capped.contains(&"3.2".to_string()), "{capped:?}");
}
#[test]
fn value_axis_title_turns_counter_clockwise_like_word() {
let title = |body: &str| format!(r#"<c:title><c:tx><c:rich>{body}<a:p><a:r><a:t>Revenue</a:t></a:r></a:p></c:rich></c:tx></c:title>"#);
let angle = |items: &[GraphicItem]| {
items.iter().find_map(|it| if let GraphicItem::Text { text, angle, .. } = it { (text == "Revenue").then_some(*angle) } else { None })
};
// The file's rotation (`-5400000` is 90° counter-clockwise), or Word's default when it says none.
assert_eq!(angle(&line_with("", &title(r#"<a:bodyPr rot="-5400000" vert="horz"/>"#))), Some(-90.0));
assert_eq!(angle(&line_with("", &title("<a:bodyPr/>"))), Some(-90.0));
assert_eq!(angle(&line_with("", &title(r#"<a:bodyPr rot="5400000" vert="horz"/>"#))), Some(90.0));
// A horizontal title (`rot="0"`) stays horizontal: the plot makes room for it (see below).
assert_eq!(angle(&line_with("", &title(r#"<a:bodyPr rot="0" vert="horz"/>"#))), Some(0.0));
// Hostile values stay finite.
let odd = angle(&line_with("", &title(r#"<a:bodyPr rot="99999999999999999999"/>"#)));
assert!(odd.is_some_and(f32::is_finite));
}
fn first_ellipse(items: &[GraphicItem]) -> Option<([f32; 4], Rgb)> {
items.iter().find_map(|it| match it {
GraphicItem::Shape { kind: ShapeKind::Ellipse, rect, fill: Some(c), .. } if rect[2] > 3.0 && rect[2] < 20.0 => Some((*rect, *c)),
_ => None,
})
}
#[test]
fn markers_use_the_automatic_series_colour_not_the_line_colour() {
// The series has an explicit red line/fill but no marker fill: Word draws the markers in the
// palette colour of the series index (accent 1 here).
let items = line_with(&marker_xml("circle", 9), "");
assert_eq!(first_ellipse(&items).map(|e| e.1), Some(theme()[4]));
// A marker's own fill wins.
let own =
r#"<c:marker><c:symbol val="circle"/><c:size val="9"/><c:spPr><a:solidFill><a:srgbClr val="00AA00"/></a:solidFill></c:spPr></c:marker>"#;
assert_eq!(first_ellipse(&line_with(own, "")).map(|e| e.1), Some(Rgb(0, 0xAA, 0)));
}
#[test]
fn line_data_labels_default_to_the_right_and_honour_dlblpos() {
let dl = |pos: &str| format!(r#"{}<c:dLbls>{pos}<c:showVal val="1"/></c:dLbls>"#, marker_xml("circle", 9));
let text_rect = |items: &[GraphicItem]| {
items.iter().find_map(|it| if let GraphicItem::Text { text, rect, .. } = it { (text == "3.2").then_some(*rect) } else { None })
};
let at = |pos: &str| {
let items = line_with(&dl(pos), "");
let (m, _) = first_ellipse(&items).unwrap();
let t = text_rect(&items).unwrap();
// Centre of the marker and of the label box.
((t[0] + t[2] / 2.0) - (m[0] + m[2] / 2.0), (t[1] + t[3] / 2.0) - (m[1] + m[3] / 2.0), t, m)
};
let (dx, dy, t, m) = at("");
assert!(t[0] >= m[0] + m[2] && dy.abs() < 1.0, "default is right of the marker: {dx} {dy}");
let (dx, dy, t, m) = at(r#"<c:dLblPos val="r"/>"#);
assert!(t[0] >= m[0] + m[2] && dy.abs() < 1.0, "{dx} {dy}");
let (dx, dy, _, _) = at(r#"<c:dLblPos val="t"/>"#);
assert!(dx.abs() < 1.0 && dy < -4.0, "{dx} {dy}");
let (dx, dy, _, _) = at(r#"<c:dLblPos val="b"/>"#);
assert!(dx.abs() < 1.0 && dy > 4.0, "{dx} {dy}");
let (dx, dy, t, m) = at(r#"<c:dLblPos val="l"/>"#);
assert!(t[0] + t[2] <= m[0] && dy.abs() < 1.0, "{dx} {dy}");
let (dx, dy, _, _) = at(r#"<c:dLblPos val="ctr"/>"#);
assert!(dx.abs() < 1.0 && dy.abs() < 1.0, "{dx} {dy}");
// Unknown positions fall back to the default.
let (_, _, t, m) = at(r#"<c:dLblPos val="wibble"/>"#);
assert!(t[0] >= m[0] + m[2]);
}
#[test]
fn horizontal_value_axis_title_gets_its_own_strip() {
let title = |rot: &str, name: &str| {
format!(r#"<c:title><c:tx><c:rich><a:bodyPr{rot}/><a:p><a:r><a:t>{name}</a:t></a:r></a:p></c:rich></c:tx></c:title>"#)
};
let plot_left = |items: &[GraphicItem]| {
// The first gridline's start is the plot's left edge.
items.iter().find_map(|it| match it {
GraphicItem::Path { segs, stroke: Some(_), fill: None, .. } if segs.len() == 2 => match segs.first() {
Some(PathSeg::Move(x, _)) => Some(*x),
_ => None,
},
_ => None,
})
};
let rect_of = |items: &[GraphicItem], name: &str| {
items.iter().find_map(|it| if let GraphicItem::Text { text, rect, .. } = it { text.starts_with(name).then_some(*rect) } else { None })
};
let turned = line_with("", &title(r#" rot="-5400000" vert="horz""#, "Revenue"));
let flat = line_with("", &title(r#" rot="0" vert="horz""#, "Revenue"));
let (a, b) = (plot_left(&turned).unwrap(), plot_left(&flat).unwrap());
assert!(b > a + 10.0, "a horizontal title takes more room than a turned one: {a} {b}");
// The title box lies left of the plot and inside the chart.
let r = rect_of(&flat, "Revenue").unwrap();
assert!(r[0] >= 0.0 && r[0] + r[2] <= b + 0.5, "{r:?} {b}");
// A hostile title never takes more than a quarter of the width and stays finite.
let long = "Wibble ".repeat(500);
let items = line_with("", &title(r#" rot="0""#, &long));
let p = plot_left(&items).unwrap();
assert!(p.is_finite() && p <= 300.0 * 0.3, "{p}");
}
/// A one-series chart whose series carries `sp` as its `c:spPr`.
fn chart_with_sp(kind: &str, sp: &str) -> Vec<GraphicItem> {
let s = ser(0, "S", None, &["Q1", "Q2"], &["4", "8"]).replace("<c:cat>", &format!("{sp}<c:cat>"));
let plot = match kind {
"area" => format!(
r#"<c:areaChart><c:grouping val="standard"/>{s}<c:axId val="1"/><c:axId val="2"/></c:areaChart><c:catAx><c:axId val="1"/></c:catAx><c:valAx><c:axId val="2"/></c:valAx>"#
),
dir => format!(
r#"<c:barChart><c:barDir val="{dir}"/><c:grouping val="clustered"/>{s}<c:axId val="1"/><c:axId val="2"/></c:barChart><c:catAx><c:axId val="1"/></c:catAx><c:valAx><c:axId val="2"/></c:valAx>"#
),
};
chart_items(&space(&plot), &theme(), 300.0, 200.0)
}
/// The series' closed shapes: (filled, stroked with the given width).
fn series_shapes(items: &[GraphicItem], width: f32) -> (usize, usize) {
let closed = |segs: &[PathSeg]| segs.iter().any(|s| matches!(s, PathSeg::Close));
let fills = items.iter().filter(|it| matches!(it, GraphicItem::Path { segs, fill: Some(_), .. } if closed(segs))).count();
let strokes = items
.iter()
.filter(|it| matches!(it, GraphicItem::Path { segs, fill: None, stroke: Some(_), stroke_width, .. } if closed(segs) && (*stroke_width - width).abs() < 0.01))
.count();
(fills, strokes)
}
#[test]
fn unfilled_bars_keep_their_explicit_outline() {
let outlined = r#"<c:spPr><a:noFill/><a:ln w="25400"><a:solidFill><a:srgbClr val="FF0000"/></a:solidFill></a:ln></c:spPr>"#;
let no_line = r#"<c:spPr><a:noFill/><a:ln><a:noFill/></a:ln></c:spPr>"#;
for dir in ["col", "bar"] {
// Two points, two outlines, no fill.
assert_eq!(series_shapes(&chart_with_sp(dir, outlined), 2.0), (0, 2), "{dir}");
// Control: no fill and no line draws nothing.
assert_eq!(series_shapes(&chart_with_sp(dir, no_line), 2.0), (0, 0), "{dir}");
}
}
#[test]
fn no_fill_area_series_is_not_painted() {
let filled = r#"<c:spPr><a:solidFill><a:srgbClr val="FF0000"/></a:solidFill></c:spPr>"#;
let none = r#"<c:spPr><a:noFill/><a:ln w="25400"><a:solidFill><a:srgbClr val="FF0000"/></a:solidFill></a:ln></c:spPr>"#;
assert_eq!(series_shapes(&chart_with_sp("area", filled), 2.0).0, 1);
assert_eq!(series_shapes(&chart_with_sp("area", none), 2.0).0, 0);
}
#[test]
fn source_linked_series_labels_do_not_inherit_the_chart_format() {
let dl = |fmt: &str| format!(r#"<c:dLbls><c:numFmt formatCode="{fmt}" sourceLinked="{{}}"/><c:showVal val="1"/></c:dLbls>"#);
let shown = |items: &[GraphicItem]| -> Vec<String> {
items.iter().filter_map(|it| if let GraphicItem::Text { text, .. } = it { Some(text.clone()) } else { None }).collect()
};
// Chart-level 0%; the series' own labels are `linked` (or have no labels element at all).
let run = |series_dl: &str| {
let s = ser(0, "S", Some("FF0000"), &["Q1", "Q2", "Q3"], &["3.2", "4.1", "5"]).replace("<c:cat>", &format!("{series_dl}<c:cat>"));
let chart_dl = dl("0%").replace("{}", "0");
let plot = format!(
r#"<c:lineChart><c:grouping val="standard"/>{s}{chart_dl}<c:axId val="1"/><c:axId val="2"/></c:lineChart><c:catAx><c:axId val="1"/></c:catAx><c:valAx><c:axId val="2"/></c:valAx>"#
);
shown(&chart_items(&space(&plot), &theme(), 300.0, 200.0))
};
for linked in ["1", "true"] {
let own = run(&dl("General").replace("{}", linked));
assert!(own.contains(&"3.2".to_string()) && !own.contains(&"320%".to_string()), "sourceLinked={linked}: {own:?}");
}
// Control: a series without a label format still takes the chart's.
let inherited = run(r#"<c:dLbls><c:showVal val="1"/></c:dLbls>"#);
assert!(inherited.contains(&"320%".to_string()), "{inherited:?}");
}
+10 -1
View File
@@ -463,7 +463,16 @@ fn text(sp: &El, xf: Xf, b: [f64; 4], theme: &[Rgb], filled: bool, fit: f32, out
let mut y = y0 + (th - total) / 2.0;
for l in lines {
let lh = l.size * LINE;
out.push(GraphicItem::Text { rect: [x0, y, tw, lh], text: l.text, size: l.size, color: l.color, bold: l.bold, align: l.align, font: l.font });
out.push(GraphicItem::Text {
rect: [x0, y, tw, lh],
text: l.text,
size: l.size,
color: l.color,
bold: l.bold,
align: l.align,
font: l.font,
angle: 0.0,
});
y += lh;
}
}
+5 -4
View File
@@ -249,10 +249,11 @@ impl Reader<'_> {
.unwrap_or_default();
para.props.fmt_change = PropChange::boxed(self.revision(RevisionKind::Format, ch), old);
}
if para.props.numbering.is_some()
&& let Some(ch) = ppr.child("w:numPr").and_then(|n| n.child("w:numberingChange"))
{
let original = ch.attr("w:original").unwrap_or("").chars().take(256).collect();
if let Some(ch) = ppr.child("w:numPr").and_then(|n| n.child("w:numberingChange")) {
// Multi-level numbering caches (`w:original`) routinely exceed 256 chars while staying
// bounded against hostile attributes.
const MAX_CHANGE_ORIGINAL: usize = 4096;
let original = ch.attr("w:original").unwrap_or("").chars().take(MAX_CHANGE_ORIGINAL).collect();
para.props.num_change = Some(Box::new(NumChange { rev: self.revision(RevisionKind::Format, ch), original }));
}
if let Some(s) = ppr.child("w:sectPr") {
+2 -1
View File
@@ -884,10 +884,11 @@ mod tests {
let spec = ChartSpec {
kind: ChartType::Column,
categories: vec!["a".into(), "b".into()],
series: (0..series).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 2] }).collect(),
series: (0..series).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 2], ..Default::default() }).collect(),
title: None,
legend: None,
data_labels: false,
..Default::default()
};
let graphic = Graphic { kind: GraphicKind::Chart, w: 100.0, h: 100.0, chart: Some(Arc::new(spec)), ..Default::default() };
InlineObject::Graphic { w: 100.0, h: 100.0, alt: String::new(), float: Default::default(), graphic: Arc::new(graphic) }
+5 -3
View File
@@ -261,10 +261,12 @@ pub fn ppr_inner(w: &mut W, p: &ParaProps, framed: bool, num_change: Option<&Cha
toggle(w, "w:pageBreakBefore", p.page_break_before);
write_frame(w, p, framed);
toggle(w, "w:widowControl", p.widow_control);
if let Some(nr) = p.numbering {
if p.numbering.is_some() || num_change.is_some() {
w.open("w:numPr", &[]);
w.val("w:ilvl", &n(nr.level.min(8) as i64));
w.val("w:numId", &n(nr.num as i64));
if let Some(nr) = p.numbering {
w.val("w:ilvl", &n(nr.level.min(8) as i64));
w.val("w:numId", &n(nr.num as i64));
}
if let Some(a) = num_change {
let refs: Vec<(&str, &str)> = a.iter().map(|(k, v)| (*k, v.as_str())).collect();
w.empty("w:numberingChange", &refs);
+44
View File
@@ -2197,6 +2197,50 @@ fn formatting_revisions_round_trip() {
assert_eq!(again.revisions, d.revisions);
}
#[test]
fn numbering_change_without_active_list_roundtrips() {
let body = r#"<w:p><w:pPr><w:numPr><w:numberingChange w:id="1" w:author="Alice" w:date="2026-10-01T00:00:00Z" w:original="%1:1:0:."/></w:numPr></w:pPr><w:r><w:t>Removed</w:t></w:r></w:p>"#;
let d = read_body(body);
let p = paras(&d);
assert_eq!(p.len(), 1);
assert!(p[0].props.numbering.is_none());
let nc = p[0].props.num_change.as_deref().expect("numberingChange read back");
assert_eq!(nc.original, "%1:1:0:.");
let bytes = craft_docx::write(&d).unwrap();
let mut zip = zip::ZipArchive::new(std::io::Cursor::new(&bytes)).unwrap();
let mut xml = String::new();
std::io::Read::read_to_string(&mut zip.by_name("word/document.xml").unwrap(), &mut xml).unwrap();
assert!(xml.contains(r#"<w:numPr><w:numberingChange w:id="#), "{xml}");
assert!(xml.contains(r#"w:original="%1:1:0:."/></w:numPr>"#), "{xml}");
let again = craft_docx::read(&bytes).unwrap();
let ap = paras(&again);
assert!(ap[0].props.numbering.is_none());
let anc = ap[0].props.num_change.as_deref().expect("numberingChange preserved after save");
assert_eq!(anc.original, "%1:1:0:.");
}
#[test]
fn numbering_change_original_preserves_long_values_within_bound() {
let long_orig = "%1:1:0:.".repeat(64); // 512 chars, well past the old 256-char cap
assert!(long_orig.len() > 256 && long_orig.len() < 4096);
let over_cap = "x".repeat(5000);
let body = format!(
r#"<w:p><w:pPr><w:numPr><w:numberingChange w:id="1" w:author="Alice" w:original="{long_orig}"/></w:numPr></w:pPr><w:r><w:t>Long</w:t></w:r></w:p>
<w:p><w:pPr><w:numPr><w:numberingChange w:id="2" w:author="Alice" w:original="{over_cap}"/></w:numPr></w:pPr><w:r><w:t>Bounded</w:t></w:r></w:p>"#
);
let d = read_body(&body);
let p = paras(&d);
assert_eq!(p[0].props.num_change.as_deref().map(|n| n.original.as_str()), Some(long_orig.as_str()));
assert_eq!(p[1].props.num_change.as_deref().map(|n| n.original.chars().count()), Some(4096));
let bytes = craft_docx::write(&d).unwrap();
let again = craft_docx::read(&bytes).unwrap();
let ap = paras(&again);
assert_eq!(ap[0].props.num_change.as_deref().map(|n| n.original.as_str()), Some(long_orig.as_str()));
}
/// A package with `styles`, a theme (accent1 4472C4, accent2 ED7D31, major font "Major Face") and,
/// when given, `settings`.
fn themed_docx(body: &str, styles: &str, settings: Option<&str>) -> Document {
+3
View File
@@ -34,6 +34,9 @@
//! Input is treated as hostile: every header read is bounds-checked, the picture size is capped
//! by [`Limits`] and the image data must be present in full before anything is allocated.
/// RGBA8-to-10-bit DPX encoding.
pub mod rgba8_dpx10;
#[cfg(test)]
mod tests;
+122
View File
@@ -0,0 +1,122 @@
//! Uncompressed RGB(A) output from top-down RGBA8 pixels.
/// Why the supplied frame cannot be encoded.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Rgba8EncodeError {
/// Empty, overflowing, or mismatched dimensions.
FrameSize,
/// A dimension cannot fit the output format.
TooLarge,
/// The output buffer could not be allocated.
OutOfMemory,
}
/// Reserves the DPX output up front; a failed reservation becomes [`Rgba8EncodeError::OutOfMemory`].
fn output_buffer(size: usize) -> std::result::Result<Vec<u8>, Rgba8EncodeError> {
let mut buf = Vec::new();
match buf.try_reserve_exact(size) {
Ok(()) => Ok(buf),
Err(_) => Err(Rgba8EncodeError::OutOfMemory),
}
}
fn checked_len(rgba: &[u8], w: u32, h: u32) -> std::result::Result<usize, Rgba8EncodeError> {
let n = (w as usize).checked_mul(h as usize).and_then(|n| n.checked_mul(4)).ok_or(Rgba8EncodeError::FrameSize)?;
if w == 0 || h == 0 || rgba.len() != n {
return Err(Rgba8EncodeError::FrameSize);
}
Ok(n)
}
/// Header size of the DPX files we write (file + image + orientation + film + TV headers).
const DPX_HEADER: usize = 2048;
/// A DPX file: 10-bit RGB (8-bit values widened by bit replication, so 0 → 0 and 255 → 1023).
pub fn encode_rgba8_dpx10(rgba: &[u8], w: u32, h: u32, creator: &str) -> std::result::Result<Vec<u8>, Rgba8EncodeError> {
let n = checked_len(rgba, w, h)?;
// one 32-bit word per pixel; `n` is already `w * h * 4`
let size = n.checked_add(DPX_HEADER).ok_or(Rgba8EncodeError::FrameSize)?;
let file_size = u32::try_from(size).map_err(|_| Rgba8EncodeError::TooLarge)?;
// SMPTE ST 268: undefined numeric fields are all ones, undefined text fields are NUL
let mut hd = vec![0xFF_u8; DPX_HEADER];
for (a, b) in [
(36, 660), // file name, creation time, creator, project, copyright
(664, 768), // reserved
(1356, 1408), // reserved
(1432, 1620), // source file name, time, input device, serial number
(1644, 1664), // reserved
(1664, 1712), // film manufacturer, type, offset, prefix, count, format
(1732, 1920), // frame id, slate, reserved
(1972, 2048), // reserved
] {
hd[a..b].fill(0);
}
for e in 0..8 {
let d = 780 + e * 72 + 40;
hd[d..d + 32].fill(0); // element descriptions
}
let put32 = |hd: &mut Vec<u8>, at: usize, v: u32| hd[at..at + 4].copy_from_slice(&v.to_be_bytes());
let put16 = |hd: &mut Vec<u8>, at: usize, v: u16| hd[at..at + 2].copy_from_slice(&v.to_be_bytes());
// file information header
hd[0..4].copy_from_slice(b"SDPX");
put32(&mut hd, 4, DPX_HEADER as u32); // offset to the image data
hd[8..16].copy_from_slice(b"V2.0\0\0\0\0");
put32(&mut hd, 16, file_size);
put32(&mut hd, 20, 1); // ditto key: new frame
put32(&mut hd, 24, 1664); // generic header length
put32(&mut hd, 28, 384); // industry header length
put32(&mut hd, 32, 0); // user data length
let creator = creator.as_bytes();
let n = creator.len().min(100);
hd[160..160 + n].copy_from_slice(&creator[..n]);
// image information header
put16(&mut hd, 768, 0); // orientation: left to right, top to bottom
put16(&mut hd, 770, 1); // one image element
put32(&mut hd, 772, w);
put32(&mut hd, 776, h);
let el = 780;
put32(&mut hd, el, 0); // unsigned
put32(&mut hd, el + 4, 0); // reference low data code
put32(&mut hd, el + 12, 1023); // reference high data code
hd[el + 20] = 50; // descriptor: RGB
hd[el + 21] = 6; // transfer: ITU-R BT.709
hd[el + 22] = 6; // colorimetric: ITU-R BT.709
hd[el + 23] = 10; // bit depth
put16(&mut hd, el + 24, 1); // packing: filled to 32-bit words, method A
put16(&mut hd, el + 26, 0); // no encoding (uncompressed)
put32(&mut hd, el + 28, DPX_HEADER as u32); // offset to the element's data
put32(&mut hd, el + 32, 0); // end-of-line padding
put32(&mut hd, el + 36, 0); // end-of-image padding
// image orientation header: no offset, square pixels
put32(&mut hd, 1408, 0);
put32(&mut hd, 1412, 0);
put32(&mut hd, 1424, w);
put32(&mut hd, 1428, h);
put32(&mut hd, 1628, 1);
put32(&mut hd, 1632, 1);
// television header: progressive
hd[1928] = 0;
// The caller already holds the frame; reserve the output fallibly so a large export reports
// an error instead of aborting the process.
let mut out = output_buffer(size)?;
out.extend_from_slice(&hd);
let ten = |v: u8| (u32::from(v) << 2) | (u32::from(v) >> 6);
for &[r, g, b, _] in rgba.as_chunks::<4>().0 {
out.extend_from_slice(&((ten(r) << 22) | (ten(g) << 12) | (ten(b) << 2)).to_be_bytes());
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn packed_rgb_and_creator() {
let bytes = encode_rgba8_dpx10(&[255, 0, 128, 255], 1, 1, "FilmCraft").unwrap();
assert_eq!(&bytes[..4], b"SDPX");
assert_eq!(&bytes[160..169], b"FilmCraft");
assert_eq!(&bytes[2048..], &((1023u32 << 22) | (514u32 << 2)).to_be_bytes());
assert_eq!(encode_rgba8_dpx10(&[], 0, 1, ""), Err(Rgba8EncodeError::FrameSize));
}
#[test]
fn output_reservation_failure_maps_to_out_of_memory() {
assert!(matches!(output_buffer(usize::MAX), Err(Rgba8EncodeError::OutOfMemory)));
assert_eq!(output_buffer(DPX_HEADER).map(|b| b.capacity() >= DPX_HEADER), Ok(true));
}
}
+55 -24
View File
@@ -62,6 +62,10 @@ impl Limits {
/// A byte count in megabytes for messages ("9.9 MB", "1024 MB").
#[cfg_attr(no_dwg_bridge, allow(dead_code))]
#[expect(
clippy::cast_precision_loss,
reason = "Megabyte formatting is rounded for display; exact integer precision beyond 2^53 bytes is not meaningful."
)]
fn mb(n: u64) -> String {
let m = n as f64 / f64::from(1u32 << 20);
if m >= 100.0 { format!("{m:.0} MB") } else { format!("{m:.1} MB") }
@@ -79,12 +83,17 @@ mod proxy_transport;
#[cfg(not(no_dwg_bridge))]
mod native {
use std::fmt::Write as _;
use std::io::{Cursor, Write};
use std::panic::AssertUnwindSafe;
use super::{Limits, mb};
/// Read DWG bytes and return an ASCII DXF rendition, within [`Limits::DEFAULT`].
///
/// # Errors
///
/// Returns an error when the DWG cannot be read or post-processed, or its DXF rendition exceeds the default limits.
pub fn dwg_to_dxf(bytes: &[u8]) -> Result<Vec<u8>, String> {
dwg_to_dxf_with(bytes, Limits::DEFAULT)
}
@@ -92,6 +101,10 @@ mod native {
/// [`dwg_to_dxf`] with explicit limits. Both are checked before the expensive step they
/// guard: the input size before the DWG reader runs, the DXF size while it is written, so a
/// runaway conversion stops at the limit instead of filling memory first.
///
/// # Errors
///
/// Returns an error when the input exceeds `limits`, decoding fails, or the DXF output exceeds `limits`.
pub fn dwg_to_dxf_with(bytes: &[u8], limits: Limits) -> Result<Vec<u8>, String> {
dwg_to_dxf_noted(bytes, limits).map(|(dxf, _)| dxf)
}
@@ -100,6 +113,10 @@ mod native {
/// failsafe mode: an object record it can't decode is skipped instead of failing the whole
/// file, and the notes say how many were skipped and of which types, so a partial read never
/// passes for a complete one.
///
/// # Errors
///
/// Returns an error when the input exceeds `limits`, decoding fails, or the DXF output exceeds `limits`.
pub fn dwg_to_dxf_noted(bytes: &[u8], limits: Limits) -> Result<(Vec<u8>, Vec<String>), String> {
if bytes.len() > limits.max_dwg_bytes {
return Err(format!(
@@ -122,13 +139,13 @@ mod native {
let mut notes = read_notes(&outcome.stats);
let mut doc = outcome.document;
notes.extend(
guarded(AssertUnwindSafe(|| application_objects(&mut doc))).map_err(|_| "the DWG application object conversion failed".to_string())?,
guarded(AssertUnwindSafe(|| application_objects(&mut doc))).map_err(|()| "the DWG application object conversion failed".to_string())?,
);
// After application_objects, so the proxy objects it makes are numbered as the DXF's.
notes.extend(
guarded(AssertUnwindSafe(|| super::proxy_transport::prepare(&mut doc))).map_err(|_| "the DWG proxy conversion failed".to_string())?,
guarded(AssertUnwindSafe(|| super::proxy_transport::prepare(&mut doc))).map_err(|()| "the DWG proxy conversion failed".to_string())?,
);
let orphans = guarded(AssertUnwindSafe(|| post_process(&mut doc))).map_err(|_| "the DWG post-processing failed on this file".to_string())?;
let orphans = guarded(AssertUnwindSafe(|| post_process(&mut doc))).map_err(|()| "the DWG post-processing failed on this file".to_string())?;
if orphans > 0 {
let (s, were) = if orphans == 1 { ("", "was") } else { ("s", "were") };
notes.push(format!("DWG: {orphans} DGN line style component{s} that no line style definition uses {were} left out"));
@@ -632,7 +649,7 @@ mod native {
// A real table has at most one merge per cell; more ranges than that come from a
// damaged file and would only make this rows × cols × ranges.
let max_ranges = rows.saturating_mul(cols).min(1 << 16);
for r in t.merged_ranges.iter().take(max_ranges).cloned().collect::<Vec<_>>() {
for r in t.merged_ranges.iter().take(max_ranges).copied().collect::<Vec<_>>() {
// Clamp to the table: the ranges come from the file.
let (bottom, right) = (r.bottom_row.min(rows.saturating_sub(1)), r.right_col.min(cols.saturating_sub(1)));
if r.top_row > bottom || r.left_col > right || rows == 0 || cols == 0 || (r.top_row, r.left_col) == (bottom, right) {
@@ -715,8 +732,8 @@ mod native {
}
/// Leave out DGN line style components (`LS*COMPONENT`) that no line style definition uses
/// (#1482). A drawing that once came from MicroStation can carry hundreds of thousands of
/// them, orphaned copies piled up by earlier DGN imports and copy-pastes (AutoCAD's PURGE
/// (#1482). A drawing that once came from `MicroStation` can carry hundreds of thousands of
/// them, orphaned copies piled up by earlier DGN imports and copy-pastes (`AutoCAD`'s PURGE
/// removes them as orphaned data): in a real 83 MB site plan they were 888,483 of 889,053
/// components and made its DXF rendition 1.2 GB, over [`Limits::DEFAULT`], so the file
/// didn't open at all. No definition that was read refers to them (a definition the reader
@@ -846,7 +863,7 @@ mod native {
for (name, code, value) in unwritten_header_vars(h) {
let written = section.windows(name.len() + 2).any(|w| w.starts_with(name.as_bytes()) && w.ends_with(b"\r\n"));
if !written && value.parse::<f64>().is_ok_and(f64::is_finite) {
add.push_str(&format!(" 9\r\n{name}\r\n{code:>3}\r\n{value}\r\n"));
let _ = write!(add, " 9\r\n{name}\r\n{code:>3}\r\n{value}\r\n");
}
}
dxf.splice(end..end, add.into_bytes());
@@ -878,14 +895,22 @@ mod native {
/// Oldest DWG version written; older DXF input is written as this version.
const MIN_DWG_VERSION: acadrust::DxfVersion = acadrust::DxfVersion::AC1018;
/// Convert DXF bytes into a DWG file of the DXF's version, and at least AutoCAD 2004
/// Convert DXF bytes into a DWG file of the DXF's version, and at least `AutoCAD` 2004
/// (AC1018).
///
/// # Errors
///
/// Returns an error when the DXF cannot be decoded or the DWG cannot be written, including oversized input.
pub fn dxf_to_dwg(dxf: &[u8]) -> Result<Vec<u8>, String> {
convert(dxf, None, None).map(|(dwg, _)| dwg)
}
/// Convert DXF bytes into a DWG file of `version`, a code from [`super::VERSIONS`] ("AC1032"),
/// whatever version the DXF declares.
///
/// # Errors
///
/// Returns an error for an unsupported version, invalid DXF, oversized input, or a DWG write failure.
pub fn dxf_to_dwg_as(dxf: &[u8], version: &str) -> Result<Vec<u8>, String> {
dxf_to_dwg_noted(dxf, version).map(|(dwg, _)| dwg)
}
@@ -1011,7 +1036,7 @@ mod native {
/// The current dimension style (`$DIMSTYLE`) by its name: the DXF gives the name, and the
/// handle read with it needn't be the style's in the document, in which case the DWG writer
/// fell back to Standard and AutoCAD opened the drawing with Standard current (#1051).
/// fell back to Standard and `AutoCAD` opened the drawing with Standard current (#1051).
fn current_dimstyle(doc: &mut acadrust::CadDocument) {
let name = doc.header.current_dimstyle_name.clone();
if let Some(h) = doc.dim_styles.iter().find(|ds| ds.name.eq_ignore_ascii_case(&name)).map(|ds| ds.handle) {
@@ -1021,7 +1046,7 @@ mod native {
/// The current text style (`$TEXTSTYLE`) by its name, the same way as [`current_dimstyle`]:
/// the handle read with the DXF needn't be the style's in the document, in which case the DWG
/// writer fell back to Standard and AutoCAD opened the drawing with Standard current (#1650).
/// writer fell back to Standard and `AutoCAD` opened the drawing with Standard current (#1650).
fn current_text_style(doc: &mut acadrust::CadDocument) {
let name = doc.header.current_text_style_name.clone();
if let Some(h) = doc.text_styles.iter().find(|ts| ts.name.eq_ignore_ascii_case(&name)).map(|ts| ts.handle) {
@@ -1211,7 +1236,7 @@ mod native {
};
let (classes, source) = guarded(AssertUnwindSafe(|| (dxf_class_names(&data), super::proxy_transport::source_classes(&data))))
.map_err(|_| "the DXF proxy class conversion failed".to_string())?;
let mut doc = std::panic::catch_unwind(move || acadrust::DxfReader::from_reader(Cursor::new(data)).and_then(|r| r.read()))
let mut doc = std::panic::catch_unwind(move || acadrust::DxfReader::from_reader(Cursor::new(data)).and_then(acadrust::DxfReader::read))
.map_err(|_| "the DXF→DWG conversion failed".to_string())?
.map_err(|e| format!("DXF: {e}"))?;
match version {
@@ -1250,14 +1275,14 @@ mod native {
current_layer(&mut doc);
unwritten
}))
.map_err(|_| "the DXF to DWG conversion failed on this file".to_string())?;
.map_err(|()| "the DXF to DWG conversion failed on this file".to_string())?;
let dwg = std::panic::catch_unwind(AssertUnwindSafe(|| acadrust::DwgWriter::write_to_vec(&doc)))
.map_err(|_| "the DWG writer failed on this drawing".to_string())?
.map_err(|e| format!("DWG write: {e}"))?;
Ok((dwg, unwritten))
}
/// AutoCAD's entry name for the drawing variables dictionary (the DXF Reference calls it the
/// `AutoCAD`'s entry name for the drawing variables dictionary (the DXF Reference calls it the
/// ACDBVARIABLEDICTIONARY; dictionary keys are case-insensitive).
const VARIABLE_DICTIONARY: &str = "AcDbVariableDictionary";
@@ -1273,11 +1298,11 @@ mod native {
if valid.contains(&value) { value.to_string() } else { default.to_string() }
}
/// Add the drawing variables dictionary AutoCAD always writes into the named object
/// Add the drawing variables dictionary `AutoCAD` always writes into the named object
/// dictionary, with the variables it keeps there, when the drawing has none (#1813).
/// Vectric Aspire 10.5 reads a 2004 (AC1018) DWG without it as empty ("No data was imported
/// from the DXF / DWG file"), though it reads the same drawing as DXF — which has no such
/// dictionary either, as CADCraft's and ezdxf's writers don't make one. AutoCAD's AC1032 DWGs
/// dictionary either, as CADCraft's and ezdxf's writers don't make one. `AutoCAD`'s AC1032 DWGs
/// read there without it, so newer output is left as it was, and a dictionary the drawing
/// does have keeps its own entries and values.
fn add_variable_dictionary(doc: &mut acadrust::CadDocument) {
@@ -1326,7 +1351,7 @@ mod native {
/// Entities of a layout other than the one in `*Paper_Space` are written owned by their block
/// (entity mode 0, with its handle). acadrust writes every paper-space entity with mode 1,
/// "paper space", which AutoCAD reads as the `*Paper_Space` block's: everything drawn on a
/// "paper space", which `AutoCAD` reads as the `*Paper_Space` block's: everything drawn on a
/// second layout landed on the first one, and the second stayed empty (#1057).
fn other_layout_owners(doc: &mut acadrust::CadDocument) {
let others: std::collections::HashSet<acadrust::Handle> = doc
@@ -1549,11 +1574,11 @@ thread_local! {
/// Panics when a test asked for `stage` to fail (see [`PANIC_IN`]); does nothing otherwise.
#[cfg(not(no_dwg_bridge))]
#[allow(clippy::panic)]
fn fail_in_test(_stage: &str) {
fn fail_in_test(stage: &str) {
#[cfg(not(test))]
let _ = stage;
#[cfg(test)]
if PANIC_IN.with(|p| p.get()) == Some(_stage) {
panic!("test-forced panic in {_stage}");
}
assert!(PANIC_IN.with(std::cell::Cell::get) != Some(stage), "test-forced panic in {stage}");
}
/// The error for a DWG version this writer can't create.
@@ -1633,6 +1658,8 @@ pub fn dxf_to_dwg_kept_noted(_dxf: &[u8], _version: &str, _kept: &str) -> Result
#[cfg(test)]
mod tests {
use std::fmt::Write as _;
use super::*;
/// The DXF of an empty drawing, enough for the bridge to make a small DWG.
@@ -1726,9 +1753,9 @@ mod tests {
let mut s = String::from(" 0\nSECTION\n 2\nHEADER\n 9\n$ACADVER\n 1\nAC1032\n 0\nENDSEC\n 0\nSECTION\n 2\nENTITIES\n");
for (text, flags, sub, pts) in dims {
s += " 0\nDIMENSION\n 8\n0\n100\nAcDbEntity\n100\nAcDbDimension\n 10\n0\n 20\n0\n 30\n0\n";
s += &format!(" 11\n50\n 21\n-20\n 31\n0\n 70\n{flags}\n 1\n{text}\n 3\nStandard\n100\n{sub}\n");
for (code, [x, y]) in pts.iter() {
s += &format!("{code:>3}\n{x}\n{:>3}\n{y}\n{:>3}\n0\n", code + 10, code + 20);
let _ = write!(s, " 11\n50\n 21\n-20\n 31\n0\n 70\n{flags}\n 1\n{text}\n 3\nStandard\n100\n{sub}\n");
for (code, [x, y]) in *pts {
let _ = write!(s, "{code:>3}\n{x}\n{:>3}\n{y}\n{:>3}\n0\n", code + 10, code + 20);
}
}
s + " 0\nENDSEC\n 0\nEOF\n"
@@ -1797,13 +1824,17 @@ mod tests {
.collect()
}
/// Issue #1482: a DWG that once came from MicroStation can carry hundreds of thousands of DGN
/// Issue #1482: a DWG that once came from `MicroStation` can carry hundreds of thousands of DGN
/// line style components no definition uses (a real 83 MB site plan: 888,483 of 889,053, a
/// 1.2 GB DXF rendition over the limit, so it didn't open). They're left out with a note, and
/// their dictionary entries with them; the definition and every component it reaches —
/// through compound and point components, around a cycle — are kept.
#[cfg(not(no_dwg_bridge))]
#[test]
#[expect(
clippy::too_many_lines,
reason = "This integration test builds and verifies one complete DGN component graph; splitting the fixture would obscure reachability."
)]
fn orphaned_dgn_line_style_components_are_left_out() {
use acadrust::objects::*;
let mut doc = acadrust::CadDocument::new();
+774
View File
@@ -0,0 +1,774 @@
//! Incremental ASCII DXF file writer used by VectorCraft.
//!
//! This is the file skeleton around a pre-written `ENTITIES` section: group codes, handles, and
//! the HEADER / TABLES / BLOCKS / OBJECTS sections CAD apps expect. It is a different API from
//! [`crate::model::write`], which serializes a parsed [`crate::model`] drawing as tags.
//!
//! Moved from `vectorcraft-cad` (`crates/cad/src/writer.rs`). The bytes a [`Drawing`] writes must
//! stay the same: VectorCraft's export tests treat this as the format oracle.
//!
//! DXF R13 and later give every object a handle and subclass markers; R2000 and later add layouts
//! and plot styles.
use std::fmt::Write as _;
pub use crate::model::write::Format;
/// An object handle (written in hex).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Handle(pub u32);
/// The handles of the fixed objects every drawing has. Entities and table entries count up from
/// [`FIRST_FREE`].
pub mod fixed {
use super::Handle;
pub const LAYER_TABLE: Handle = Handle(0x1);
pub const LTYPE_TABLE: Handle = Handle(0x2);
pub const APPID_TABLE: Handle = Handle(0x3);
pub const DIMSTYLE_TABLE: Handle = Handle(0x4);
pub const STYLE_TABLE: Handle = Handle(0x5);
pub const UCS_TABLE: Handle = Handle(0x6);
pub const VIEW_TABLE: Handle = Handle(0x7);
pub const VPORT_TABLE: Handle = Handle(0x8);
pub const BLOCK_RECORD_TABLE: Handle = Handle(0x9);
pub const ROOT: Handle = Handle(0xA);
pub const GROUPS: Handle = Handle(0xB);
pub const LAYOUTS: Handle = Handle(0xC);
pub const MODEL_RECORD: Handle = Handle(0xD);
pub const PAPER_RECORD: Handle = Handle(0xE);
pub const MODEL_BLOCK: Handle = Handle(0xF);
pub const MODEL_END: Handle = Handle(0x10);
pub const PAPER_BLOCK: Handle = Handle(0x11);
pub const PAPER_END: Handle = Handle(0x12);
pub const MODEL_LAYOUT: Handle = Handle(0x13);
pub const PAPER_LAYOUT: Handle = Handle(0x14);
pub const ACTIVE_VPORT: Handle = Handle(0x15);
pub const ACAD_APPID: Handle = Handle(0x16);
pub const STANDARD_DIMSTYLE: Handle = Handle(0x17);
pub const IMAGE_DICT: Handle = Handle(0x18);
pub const IMAGE_VARS: Handle = Handle(0x19);
pub const PLOT_STYLES: Handle = Handle(0x1A);
pub const PLOT_NORMAL: Handle = Handle(0x1B);
pub const FIRST_FREE: u32 = 0x30;
}
/// A number as DXF writes it: up to 6 decimals, no trailing zeros, 0 for anything not finite.
pub fn num(v: f64) -> String {
if !v.is_finite() {
return "0".into();
}
let s = format!("{v:.6}");
let s = s.trim_end_matches('0').trim_end_matches('.');
if s == "-0" || s.is_empty() { "0".into() } else { s.into() }
}
/// Text as a DXF string value: one line, `^` escaped, and before DXF 2007 (which reads UTF-8)
/// every character outside ASCII as `\U+XXXX`.
pub fn text(s: &str, utf8: bool) -> String {
let mut out = String::with_capacity(s.len());
for c in s.chars() {
match c {
'\r' | '\n' | '\t' => out.push(' '),
c if c.is_control() => {}
'^' => out.push_str("^ "),
c if !utf8 && !c.is_ascii() => match u16::try_from(u32::from(c)) {
Ok(u) => _ = write!(out, "\\U+{u:04X}"),
Err(_) => out.push('?'),
},
c => out.push(c),
}
}
out
}
/// Group-code/value pairs in the order they are written.
pub struct Codes {
pub v: Format,
out: String,
}
impl Codes {
pub fn new(v: Format) -> Self {
Self { v, out: String::new() }
}
pub fn into_string(self) -> String {
self.out
}
pub fn append(&mut self, other: &str) {
self.out.push_str(other);
}
/// A value written as it is (names and keywords the writer itself chose).
pub fn raw(&mut self, code: u16, value: &str) {
_ = writeln!(self.out, "{code:>3}\n{value}");
}
/// Text from the document, encoded for the version (see [`text`]).
pub fn str(&mut self, code: u16, value: &str) {
let t = text(value, self.v.utf8());
self.raw(code, &t);
}
pub fn int(&mut self, code: u16, value: i64) {
_ = writeln!(self.out, "{code:>3}\n{value}");
}
pub fn num(&mut self, code: u16, value: f64) {
_ = writeln!(self.out, "{code:>3}\n{}", num(value));
}
pub fn handle(&mut self, code: u16, h: Handle) {
_ = writeln!(self.out, "{code:>3}\n{:X}", h.0);
}
/// A 2D point at `code` (x), `code + 10` (y).
pub fn xy(&mut self, code: u16, x: f64, y: f64) {
self.num(code, x);
self.num(code + 10, y);
}
/// A 3D point at `code`, `code + 10`, `code + 20` (z = 0).
pub fn xyz(&mut self, code: u16, x: f64, y: f64) {
self.xy(code, x, y);
self.num(code + 20, 0.0);
}
/// The extrusion direction of planar entities: the z axis.
pub fn normal(&mut self) {
self.num(210, 0.0);
self.num(220, 0.0);
self.num(230, 1.0);
}
/// The start of an object: its type, and from R13 its handle, owner and first subclass.
fn object(&mut self, kind: &str, h: Handle, owner: Handle, subclass: &str) {
self.raw(0, kind);
if self.v.handles() {
self.handle(if kind == "DIMSTYLE" { 105 } else { 5 }, h);
self.handle(330, owner);
self.raw(100, subclass);
}
}
/// A table entry: the shared record subclass, then the table's own (from R13).
fn record(&mut self, kind: &str, h: Handle, table: Handle, subclass: &str) {
self.object(kind, h, table, "AcDbSymbolTableRecord");
if self.v.handles() {
self.raw(100, subclass);
}
}
fn section(&mut self, name: &str) {
self.raw(0, "SECTION");
self.raw(2, name);
}
fn end_section(&mut self) {
self.raw(0, "ENDSEC");
}
fn table(&mut self, name: &str, h: Handle, count: usize) {
self.raw(0, "TABLE");
self.raw(2, name);
if self.v.handles() {
self.handle(5, h);
self.raw(330, "0");
self.raw(100, "AcDbSymbolTable");
}
self.int(70, count as i64);
if name == "DIMSTYLE" && self.v.r2000() {
self.raw(100, "AcDbDimStyleTable");
}
}
fn end_table(&mut self) {
self.raw(0, "ENDTAB");
}
}
/// How an entity is drawn: its colour index, true colour, lineweight, linetype and transparency.
#[derive(Clone, Debug, Default)]
pub struct EntityStyle {
pub aci: u8,
/// `0xRRGGBB` (DXF 2004 and later).
pub rgb: Option<u32>,
/// Hundredths of a millimetre (DXF 2000 and later).
pub lineweight: Option<i32>,
pub linetype: Option<String>,
/// Opacity 0–255 (DXF 2004 and later; opaque is left out).
pub alpha: Option<u8>,
}
impl Codes {
/// The start of an entity of `kind` owned by `owner` (model space, or a polyline for its
/// vertices) on `layer` with its style, ending with `subclass` (none when empty).
pub fn entity(&mut self, kind: &str, h: Handle, owner: Handle, layer: &str, style: &EntityStyle, subclass: &str) {
self.object(kind, h, owner, "AcDbEntity");
self.str(8, layer);
if let Some(lt) = &style.linetype {
self.str(6, lt);
}
self.int(62, i64::from(style.aci));
if self.v.r2000()
&& let Some(lw) = style.lineweight
{
self.int(370, i64::from(lw));
}
if self.v.true_color() {
if let Some(rgb) = style.rgb {
self.int(420, i64::from(rgb));
}
if let Some(a) = style.alpha {
self.int(440, 0x0200_0000 | i64::from(a));
}
}
if self.v.handles() && !subclass.is_empty() {
self.raw(100, subclass);
}
}
}
/// A DXF layer.
#[derive(Clone, Debug)]
pub struct Layer {
pub name: String,
pub aci: u8,
/// Switched off (a hidden layer).
pub off: bool,
pub locked: bool,
/// Plotted (printable).
pub plot: bool,
}
/// A dash pattern: dash, gap, dash… lengths in drawing units.
#[derive(Clone, Debug)]
pub struct LineType {
pub name: String,
pub pattern: Vec<f64>,
}
/// A placed image file and the image entities that show it.
#[derive(Clone, Debug)]
pub struct ImageDef {
pub name: String,
pub size: (u32, u32),
pub handle: Handle,
/// `(reactor, image entity)` per image entity.
pub reactors: Vec<(Handle, Handle)>,
}
/// Everything around the entities, collected while they were written.
pub struct Drawing {
pub version: Format,
pub layers: Vec<Layer>,
pub linetypes: Vec<LineType>,
/// Text styles: (style name, font family).
pub styles: Vec<(String, String)>,
pub images: Vec<ImageDef>,
/// The entities' group codes.
pub entities: String,
/// `[x0, y0, x1, y1]` of everything written (drawing units).
pub extents: [f64; 4],
/// The export region (drawing units).
pub limits: [f64; 4],
/// `$INSUNITS`.
pub insunits: u8,
pub metric: bool,
/// The next free handle.
pub next: u32,
}
/// The text style every drawing has.
pub const STANDARD_STYLE: &str = "Standard";
impl Drawing {
/// The whole file.
pub fn write(self) -> String {
let v = self.version;
let mut c = Codes::new(v);
// Table entries get the handles after the entities'.
let mut next = self.next;
let mut take = |n: usize| {
let start = next;
next = next.saturating_add(u32::try_from(n).unwrap_or(u32::MAX));
start
};
let ltype_base = take(3 + self.linetypes.len());
let layer_base = take(1 + self.layers.len());
let style_base = take(1 + self.styles.len());
self.header(&mut c, next);
if v.handles() {
self.classes(&mut c);
}
c.section("TABLES");
self.vport(&mut c);
self.ltypes(&mut c, ltype_base);
self.layer_table(&mut c, layer_base);
self.style_table(&mut c, style_base);
if v.handles() {
for (name, h) in [("VIEW", fixed::VIEW_TABLE), ("UCS", fixed::UCS_TABLE)] {
c.table(name, h, 0);
c.end_table();
}
c.table("APPID", fixed::APPID_TABLE, 1);
c.record("APPID", fixed::ACAD_APPID, fixed::APPID_TABLE, "AcDbRegAppTableRecord");
c.raw(2, "ACAD");
c.int(70, 0);
c.end_table();
c.table("DIMSTYLE", fixed::DIMSTYLE_TABLE, 1);
c.record("DIMSTYLE", fixed::STANDARD_DIMSTYLE, fixed::DIMSTYLE_TABLE, "AcDbDimStyleTableRecord");
c.raw(2, "Standard");
c.int(70, 0);
c.end_table();
c.table("BLOCK_RECORD", fixed::BLOCK_RECORD_TABLE, 2);
for (name, h, layout) in
[("*Model_Space", fixed::MODEL_RECORD, fixed::MODEL_LAYOUT), ("*Paper_Space", fixed::PAPER_RECORD, fixed::PAPER_LAYOUT)]
{
c.record("BLOCK_RECORD", h, fixed::BLOCK_RECORD_TABLE, "AcDbBlockTableRecord");
c.raw(2, name);
if v.r2000() {
c.handle(340, layout);
c.int(70, 0);
c.int(280, 1);
c.int(281, 0);
}
}
c.end_table();
}
c.end_section();
if v.handles() {
c.section("BLOCKS");
for (name, record, begin, end) in [
("*Model_Space", fixed::MODEL_RECORD, fixed::MODEL_BLOCK, fixed::MODEL_END),
("*Paper_Space", fixed::PAPER_RECORD, fixed::PAPER_BLOCK, fixed::PAPER_END),
] {
c.object("BLOCK", begin, record, "AcDbEntity");
c.raw(8, "0");
c.raw(100, "AcDbBlockBegin");
c.raw(2, name);
c.int(70, 0);
c.xyz(10, 0.0, 0.0);
c.raw(3, name);
c.raw(1, "");
c.object("ENDBLK", end, record, "AcDbEntity");
c.raw(8, "0");
c.raw(100, "AcDbBlockEnd");
}
c.end_section();
}
c.section("ENTITIES");
c.append(&self.entities);
c.end_section();
if v.handles() {
self.objects(&mut c);
}
c.raw(0, "EOF");
c.into_string()
}
fn header(&self, c: &mut Codes, handseed: u32) {
let v = self.version;
let var = |c: &mut Codes, name: &str| c.raw(9, name);
c.section("HEADER");
var(c, "$ACADVER");
c.raw(1, v.acadver());
var(c, "$DWGCODEPAGE");
c.raw(3, "ANSI_1252");
var(c, "$INSBASE");
c.xyz(10, 0.0, 0.0);
let [x0, y0, x1, y1] = self.extents;
var(c, "$EXTMIN");
c.xyz(10, x0, y0);
var(c, "$EXTMAX");
c.xyz(10, x1, y1);
let [lx0, ly0, lx1, ly1] = self.limits;
var(c, "$LIMMIN");
c.xy(10, lx0, ly0);
var(c, "$LIMMAX");
c.xy(10, lx1, ly1);
if v.r2000() {
var(c, "$LWDISPLAY");
c.int(290, 1);
var(c, "$INSUNITS");
c.int(70, i64::from(self.insunits));
var(c, "$MEASUREMENT");
c.int(70, i64::from(self.metric));
}
if v.handles() {
var(c, "$HANDSEED");
c.handle(5, Handle(handseed));
}
c.end_section();
}
fn classes(&self, c: &mut Codes) {
let mut classes: Vec<(&str, &str, &str, i64, bool)> = vec![];
if self.version.r2000() {
classes.extend([
("ACDBDICTIONARYWDFLT", "AcDbDictionaryWithDefault", "ObjectDBX Classes", 0, false),
("ACDBPLACEHOLDER", "AcDbPlaceHolder", "ObjectDBX Classes", 0, false),
("LAYOUT", "AcDbLayout", "ObjectDBX Classes", 0, false),
]);
}
if !self.images.is_empty() {
classes.extend([
("RASTERVARIABLES", "AcDbRasterVariables", "ISM", 0, false),
("IMAGE", "AcDbRasterImage", "ISM", 2175, true),
("IMAGEDEF", "AcDbRasterImageDef", "ISM", 0, false),
("IMAGEDEF_REACTOR", "AcDbRasterImageDefReactor", "ISM", 1, false),
]);
}
c.section("CLASSES");
for (name, cpp, app, flags, entity) in classes {
c.raw(0, "CLASS");
c.raw(1, name);
c.raw(2, cpp);
c.raw(3, app);
c.int(90, flags);
if self.version.true_color() {
c.int(91, 0);
}
c.int(280, 0);
c.int(281, i64::from(entity));
}
c.end_section();
}
/// The active viewport, zoomed to the drawing's extents.
fn vport(&self, c: &mut Codes) {
let [x0, y0, x1, y1] = self.extents;
c.table("VPORT", fixed::VPORT_TABLE, 1);
c.record("VPORT", fixed::ACTIVE_VPORT, fixed::VPORT_TABLE, "AcDbViewportTableRecord");
c.raw(2, if self.version.handles() { "*Active" } else { "*ACTIVE" });
c.int(70, 0);
c.xy(10, 0.0, 0.0);
c.xy(11, 1.0, 1.0);
c.xy(12, (x0 + x1) / 2.0, (y0 + y1) / 2.0);
c.xy(13, 0.0, 0.0);
c.xy(14, 1.0, 1.0);
c.xy(15, 10.0, 10.0);
// The view direction looks down the z axis: (0, 0, 1), each code once.
c.xy(16, 0.0, 0.0);
c.num(36, 1.0);
c.xyz(17, 0.0, 0.0);
let (w, h) = ((x1 - x0).abs(), (y1 - y0).abs());
// The view's height, with a margin; its width follows from the aspect ratio.
c.num(40, (h.max(w / 1.5) * 1.1).max(1.0));
c.num(41, 1.5);
c.num(42, 50.0);
c.num(43, 0.0);
c.num(44, 0.0);
c.num(50, 0.0);
c.num(51, 0.0);
for (code, value) in [(71, 0), (72, 1000), (73, 1), (74, 3), (75, 0), (76, 0), (77, 0), (78, 0)] {
c.int(code, value);
}
if self.version.r2000() {
c.int(281, 0);
c.int(65, 0);
c.num(146, 0.0);
}
c.end_table();
}
fn ltypes(&self, c: &mut Codes, base: u32) {
c.table("LTYPE", fixed::LTYPE_TABLE, 3 + self.linetypes.len());
let solid = [("ByBlock", ""), ("ByLayer", ""), ("Continuous", "Solid line")];
let rows = solid.iter().map(|(n, d)| (*n, *d, &[][..])).chain(self.linetypes.iter().map(|l| (l.name.as_str(), "", l.pattern.as_slice())));
for (i, (name, desc, pattern)) in rows.enumerate() {
c.record("LTYPE", Handle(base.saturating_add(i as u32)), fixed::LTYPE_TABLE, "AcDbLinetypeTableRecord");
c.str(2, name);
c.int(70, 0);
c.str(3, if pattern.is_empty() { desc } else { "Dashed" });
c.int(72, 65);
c.int(73, pattern.len() as i64);
c.num(40, pattern.iter().map(|v| v.abs()).sum());
for v in pattern {
c.num(49, *v);
if self.version.handles() {
c.int(74, 0);
}
}
}
c.end_table();
}
fn layer_table(&self, c: &mut Codes, base: u32) {
let zero = Layer { name: "0".into(), aci: 7, off: false, locked: false, plot: true };
c.table("LAYER", fixed::LAYER_TABLE, 1 + self.layers.len());
for (i, l) in std::iter::once(&zero).chain(&self.layers).enumerate() {
c.record("LAYER", Handle(base.saturating_add(i as u32)), fixed::LAYER_TABLE, "AcDbLayerTableRecord");
c.str(2, &l.name);
c.int(70, if l.locked { 4 } else { 0 });
// A negative colour switches the layer off.
let aci = i64::from(l.aci.max(1));
c.int(62, if l.off { -aci } else { aci });
c.raw(6, "Continuous");
if self.version.r2000() {
c.int(290, i64::from(l.plot));
c.int(370, -3);
c.handle(390, fixed::PLOT_NORMAL);
}
}
c.end_table();
}
fn style_table(&self, c: &mut Codes, base: u32) {
c.table("STYLE", fixed::STYLE_TABLE, 1 + self.styles.len());
let standard = (STANDARD_STYLE.to_string(), String::new());
for (i, (name, family)) in std::iter::once(&standard).chain(&self.styles).enumerate() {
c.record("STYLE", Handle(base.saturating_add(i as u32)), fixed::STYLE_TABLE, "AcDbTextStyleTableRecord");
c.str(2, name);
c.int(70, 0);
c.num(40, 0.0);
c.num(41, 1.0);
c.num(50, 0.0);
c.int(71, 0);
c.num(42, 2.5);
// The font file; from 2000 the family name too, which is how CAD apps find a TrueType font.
if family.is_empty() {
c.raw(3, "txt");
} else {
c.str(3, &format!("{family}.ttf"));
}
c.raw(4, "");
if self.version.r2000() && !family.is_empty() {
c.raw(1001, "ACAD");
c.str(1000, family);
c.int(1071, 0);
}
}
c.end_table();
}
fn objects(&self, c: &mut Codes) {
let v = self.version;
let images = !self.images.is_empty();
c.section("OBJECTS");
let dict = |c: &mut Codes, h: Handle, owner: Handle, entries: &[(&str, Handle)]| {
c.object("DICTIONARY", h, owner, "AcDbDictionary");
c.int(281, 1);
for (name, e) in entries {
c.str(3, name);
c.handle(350, *e);
}
};
let mut root = vec![("ACAD_GROUP", fixed::GROUPS)];
if images {
root.extend([("ACAD_IMAGE_DICT", fixed::IMAGE_DICT), ("ACAD_IMAGE_VARS", fixed::IMAGE_VARS)]);
}
if v.r2000() {
root.extend([("ACAD_LAYOUT", fixed::LAYOUTS), ("ACAD_PLOTSTYLENAME", fixed::PLOT_STYLES)]);
}
dict(c, fixed::ROOT, Handle(0), &root);
dict(c, fixed::GROUPS, fixed::ROOT, &[]);
if v.r2000() {
dict(c, fixed::LAYOUTS, fixed::ROOT, &[("Layout1", fixed::PAPER_LAYOUT), ("Model", fixed::MODEL_LAYOUT)]);
c.object("ACDBDICTIONARYWDFLT", fixed::PLOT_STYLES, fixed::ROOT, "AcDbDictionary");
c.int(281, 1);
c.raw(3, "Normal");
c.handle(350, fixed::PLOT_NORMAL);
c.raw(100, "AcDbDictionaryWithDefault");
c.handle(340, fixed::PLOT_NORMAL);
// A placeholder has no data and no subclass marker of its own.
c.raw(0, "ACDBPLACEHOLDER");
c.handle(5, fixed::PLOT_NORMAL);
c.handle(330, fixed::PLOT_STYLES);
self.layout(c, fixed::MODEL_LAYOUT, "Model", 0, fixed::MODEL_RECORD);
self.layout(c, fixed::PAPER_LAYOUT, "Layout1", 1, fixed::PAPER_RECORD);
}
if images {
c.object("RASTERVARIABLES", fixed::IMAGE_VARS, fixed::ROOT, "AcDbRasterVariables");
c.int(90, 0);
c.int(70, 0);
c.int(71, 1);
c.int(72, 0);
let keys: Vec<String> = self.images.iter().map(|i| format!("IMAGE_{:X}", i.handle.0)).collect();
let entries: Vec<(&str, Handle)> = keys.iter().zip(&self.images).map(|(key, image)| (key.as_str(), image.handle)).collect();
dict(c, fixed::IMAGE_DICT, fixed::ROOT, &entries);
for img in &self.images {
c.raw(0, "IMAGEDEF");
c.handle(5, img.handle);
c.raw(102, "{ACAD_REACTORS");
c.handle(330, fixed::IMAGE_DICT);
for (reactor, _) in &img.reactors {
c.handle(330, *reactor);
}
c.raw(102, "}");
c.handle(330, fixed::IMAGE_DICT);
c.raw(100, "AcDbRasterImageDef");
c.int(90, 0);
c.str(1, &img.name);
c.xy(10, f64::from(img.size.0), f64::from(img.size.1));
c.xy(11, 1.0, 1.0);
c.int(280, 1);
c.int(281, 0);
for (reactor, image) in &img.reactors {
c.object("IMAGEDEF_REACTOR", *reactor, *image, "AcDbRasterImageDefReactor");
c.int(90, 2);
c.handle(330, *image);
}
}
}
c.end_section();
}
/// A layout (DXF 2000 and later): its plot settings, then the layout itself.
fn layout(&self, c: &mut Codes, h: Handle, name: &str, tab: i64, record: Handle) {
let [x0, y0, x1, y1] = self.extents;
c.object("LAYOUT", h, fixed::LAYOUTS, "AcDbPlotSettings");
c.raw(1, "");
c.raw(2, "none_device");
c.raw(4, "");
c.raw(6, "");
for code in 40..=49 {
c.num(code, 0.0);
}
c.num(140, 0.0);
c.num(141, 0.0);
c.num(142, 1.0);
c.num(143, 1.0);
// Plot flags; 1024 marks the model space layout.
c.int(70, if record == fixed::MODEL_RECORD { 688 | 1024 } else { 688 });
c.int(72, 1);
c.int(73, 0);
c.int(74, 5);
c.raw(7, "");
c.int(75, 16);
if self.version.true_color() {
// Shade plot mode (as displayed), resolution level (normal) and custom DPI.
c.int(76, 0);
c.int(77, 2);
c.int(78, 300);
}
c.num(147, 1.0);
// The paper image origin is 148 (x) and 149 (y), not a point at 148 and 158.
c.num(148, 0.0);
c.num(149, 0.0);
c.raw(100, "AcDbLayout");
c.raw(1, name);
c.int(70, 1);
c.int(71, tab);
let [lx0, ly0, lx1, ly1] = self.limits;
c.xy(10, lx0, ly0);
c.xy(11, lx1, ly1);
c.xyz(12, 0.0, 0.0);
c.xyz(14, x0, y0);
c.xyz(15, x1, y1);
c.num(146, 0.0);
c.xyz(13, 0.0, 0.0);
c.xyz(16, 1.0, 0.0);
c.xyz(17, 0.0, 1.0);
c.int(76, 1);
c.handle(330, record);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn numbers_and_text_are_written_plainly() {
assert_eq!(num(1.5), "1.5");
assert_eq!(num(-0.0000001), "0");
assert_eq!(num(2.0), "2");
assert_eq!(num(f64::NAN), "0");
assert_eq!(text("a^b\nc", true), "a^ b c");
assert_eq!(text("Größe 😀", false), "Gr\\U+00F6\\U+00DFe ?");
assert_eq!(text("Größe", true), "Größe");
}
/// A line from (0, 0) to (10, 20) on layer `Layer1`, in the style scene.rs writes entities.
fn line_entities(v: Format) -> String {
let mut e = Codes::new(v);
let style = EntityStyle { aci: 1, rgb: Some(0xFF0000), lineweight: Some(25), linetype: None, alpha: None };
e.entity("LINE", Handle(fixed::FIRST_FREE), fixed::MODEL_RECORD, "Layer1", &style, "AcDbLine");
e.xyz(10, 0.0, 0.0);
e.xyz(11, 10.0, 20.0);
e.into_string()
}
fn sample(v: Format, with_line: bool) -> Drawing {
Drawing {
version: v,
layers: vec![Layer { name: "Layer1".into(), aci: 1, off: false, locked: false, plot: true }],
linetypes: vec![],
styles: vec![],
images: vec![],
entities: if with_line { line_entities(v) } else { String::new() },
extents: [0.0, 0.0, 200.0, 100.0],
limits: [0.0, 0.0, 200.0, 100.0],
insunits: 4,
metric: true,
next: fixed::FIRST_FREE + if with_line { 1 } else { 0 },
}
}
fn assert_golden(name: &str, got: &str) {
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data").join(name);
if std::env::var_os("WRITE_DXF_GOLDENS").is_some() {
std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir");
std::fs::write(&path, got).expect("write golden");
return;
}
let expected = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display()));
assert_eq!(got, expected, "bytes changed versus {}", path.display());
}
use std::path::PathBuf;
#[test]
fn empty_and_line_drawings_match_goldens() {
for (name, v, line) in [
("drawing_empty_r12.dxf", Format::R12, false),
("drawing_empty_r2018.dxf", Format::R2018, false),
("drawing_line_r12.dxf", Format::R12, true),
("drawing_line_r2000.dxf", Format::R2000, true),
("drawing_line_r2018.dxf", Format::R2018, true),
] {
assert_golden(name, &sample(v, line).write());
}
}
#[test]
fn entity_preserves_byblock_colour() {
for v in Format::ALL {
let mut c = Codes::new(v);
c.entity("LINE", Handle(fixed::FIRST_FREE), fixed::MODEL_RECORD, "0", &EntityStyle::default(), "AcDbLine");
assert!(c.into_string().contains(" 62\n0\n"), "{v:?}");
}
}
#[test]
fn image_dictionary_keys_are_independent_of_filenames() {
let mut drawing = sample(Format::R2018, false);
for (index, name) in [r"C:\images\scan,1.png", "scan,1.png"].into_iter().enumerate() {
drawing.images.push(ImageDef { name: name.into(), size: (10, 20), handle: Handle(fixed::FIRST_FREE + index as u32), reactors: vec![] });
}
drawing.next = fixed::FIRST_FREE + 2;
let images = drawing.images.clone();
let output = drawing.write();
for image in &images {
assert!(output.contains(&format!(" 3\nIMAGE_{:X}\n350\n{:X}\n", image.handle.0, image.handle.0)));
assert!(output.contains(&format!(" 1\n{}\n", image.name)));
assert!(!output.contains(&format!(" 3\n{}\n", image.name)));
}
}
#[test]
fn every_format_writes_its_acadver_and_ends_at_eof() {
for v in Format::ALL {
let s = sample(v, true).write();
assert!(s.contains(&format!(" 1\n{}\n", v.acadver())), "{v:?}");
assert!(s.ends_with(" 0\nEOF\n"), "{v:?}");
if v.handles() {
assert!(s.contains(" 0\nSECTION\n 2\nCLASSES\n"), "{v:?}");
} else {
assert!(!s.contains("\nCLASSES\n"), "{v:?}");
}
}
}
}
+8 -3
View File
@@ -3,14 +3,18 @@
//! DXF is a sequence of (group code, value) pairs. This crate parses ASCII and binary DXF into a
//! flat list of [`Tag`]s, splits them into sections and entities, and writes tags back. [`model`]
//! reads the tags as a drawing (header variables, tables, blocks, entities, objects); mapping
//! that onto an application's own document is left to the caller. Written from Autodesk's
//! that onto an application's own document is left to the caller. [`drawing`] writes an ASCII
//! file around a pre-built `ENTITIES` section (VectorCraft's writer). Written from Autodesk's
//! published DXF Reference; depends on no other Craft crate. Moved from CADCraft's `cadcraft-dxf`.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
use std::io::Write as _;
pub mod drawing;
pub mod model;
pub mod mtext;
pub mod mtext_tokens;
pub mod paths;
mod version;
pub use version::Version;
@@ -195,8 +199,9 @@ pub use codepage::decode_code_page;
/// # Errors
///
/// Returns an error for empty input, an invalid group code, an incomplete group-code/value pair,
/// truncated binary data, or more than [`MAX_TAGS`] group codes. For ASCII input, invalid numeric
/// text becomes `0.0` for real values or `0` for integer values, and an EOF tag is not required.
/// truncated binary data, or more group codes than the limit returned by [`max_tags()`], which is
/// [`MAX_TAGS`] by default. For ASCII input, invalid numeric text becomes `0.0` for real values or
/// `0` for integer values, and an EOF tag is not required.
pub fn parse(bytes: &[u8]) -> Result<Vec<Tag>> {
parse_with_limit(bytes, max_tags())
}

Some files were not shown because too many files have changed in this diff Show More