From 16936716aa960dda9a7f1532bb04a53caa3a5c75 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:24:06 -0500 Subject: [PATCH] Add craft-stream-codecs: the codec modules of PdfCraft's stream filters (#953) LZW, ASCII85, ASCIIHex, RunLength, the TIFF and PNG predictors, bounded CCITT fax decoding (through hayro-ccitt) with Group 4 encoding (through fax), and the lossless JBIG2 encoder with its MQ coder, from pdfcraft-filters at PdfCraft revision 80e26e674d1aa33078e83c8139875047f3289eb6. The crate sits below craft-pdf and craft-tiff: it knows no container. The PDF filter chain, Flate and the bounded JBIG2 decoding entry are not part of it. --- Cargo.lock | 22 +- Cargo.toml | 1 + README.md | 5 + crates/stream-codecs/Cargo.toml | 24 + crates/stream-codecs/README.md | 78 ++ crates/stream-codecs/src/ascii85.rs | 156 +++ crates/stream-codecs/src/asciihex.rs | 106 ++ crates/stream-codecs/src/ccitt.rs | 546 +++++++++ crates/stream-codecs/src/jbig2.rs | 468 ++++++++ crates/stream-codecs/src/jbig2/symbols.rs | 1329 +++++++++++++++++++++ crates/stream-codecs/src/lib.rs | 117 ++ crates/stream-codecs/src/lzw.rs | 314 +++++ crates/stream-codecs/src/mq.rs | 209 ++++ crates/stream-codecs/src/predictor.rs | 356 ++++++ crates/stream-codecs/src/runlength.rs | 117 ++ 15 files changed, 3846 insertions(+), 2 deletions(-) create mode 100644 crates/stream-codecs/Cargo.toml create mode 100644 crates/stream-codecs/README.md create mode 100644 crates/stream-codecs/src/ascii85.rs create mode 100644 crates/stream-codecs/src/asciihex.rs create mode 100644 crates/stream-codecs/src/ccitt.rs create mode 100644 crates/stream-codecs/src/jbig2.rs create mode 100644 crates/stream-codecs/src/jbig2/symbols.rs create mode 100644 crates/stream-codecs/src/lib.rs create mode 100644 crates/stream-codecs/src/lzw.rs create mode 100644 crates/stream-codecs/src/mq.rs create mode 100644 crates/stream-codecs/src/predictor.rs create mode 100644 crates/stream-codecs/src/runlength.rs diff --git a/Cargo.lock b/Cargo.lock index c7fe9c680..2fc2f8292 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1588,6 +1588,18 @@ dependencies = [ "zip", ] +[[package]] +name = "craft-stream-codecs" +version = "0.1.0" +dependencies = [ + "fax", + "hayro-ccitt", + "hayro-jbig2", + "proptest", + "thiserror 2.0.21", + "weezl 0.2.1", +] + [[package]] name = "craft-svg" version = "0.1.0" @@ -2777,7 +2789,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" dependencies = [ "color_quant", - "weezl", + "weezl 0.1.12", ] [[package]] @@ -6731,7 +6743,7 @@ dependencies = [ "flate2", "half", "quick-error 2.0.1", - "weezl", + "weezl 0.1.12", "zune-jpeg", ] @@ -7590,6 +7602,12 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "weezl" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ca08e5ef825b65b056d9efbd95c8750683f0a6d0466d02e96dc2e4e360f3d2" + [[package]] name = "wgpu" version = "30.0.1" diff --git a/Cargo.toml b/Cargo.toml index 7059505ca..3634d3b8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,6 +91,7 @@ craft-settings = { path = "crates/settings" } craft-silhouette-studio = { path = "crates/silhouette-studio" } craft-svg = { path = "crates/svg" } craft-svg-paths = { path = "crates/svg-paths" } +craft-stream-codecs = { path = "crates/stream-codecs" } craft-text = { path = "crates/text" } craft-tga = { path = "crates/tga" } craft-timeline = { path = "crates/timeline" } diff --git a/README.md b/README.md index 3776d0b56..b48364a81 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,11 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = "` form on craft-xml elements, value codecs and the package writer; and InDesign Tagged Text, read and written. +- [`craft-stream-codecs`](crates/stream-codecs/README.md): byte-stream and bi-level image codecs that + PDF, PostScript and TIFF share: LZW, ASCII85, ASCIIHex, RunLength, TIFF and PNG predictors, + CCITT Group 3 and 4 decoding and Group 4 encoding, lossless JBIG2 encoding. Every + decoder is bounded and returns its partial output on damage. From PdfCraft; a PDF filter chain + is built on it in `craft-pdf`. - [`craft-timeline`](crates/timeline/README.md): editorial timeline interchange: exact media time and SMPTE timecode, import/export reports, media path and `file://` URL handling, Avid Log Exchange (ALE) and CMX 3600 EDL documents, format detection. From FilmCraft. - [`craft-metadata`](crates/metadata/README.md): photo metadata, from LightCraft: EXIF (read and write, maker-note lens and focus point, every tag by name), IPTC-IIM, XMP packets (read, write, and a merge diff --git a/crates/stream-codecs/Cargo.toml b/crates/stream-codecs/Cargo.toml new file mode 100644 index 000000000..7624050e2 --- /dev/null +++ b/crates/stream-codecs/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "craft-stream-codecs" +description = "Byte-stream and bi-level image codecs that PDF, PostScript and TIFF share: LZW, ASCII85, ASCIIHex, RunLength, TIFF and PNG predictors, CCITT Group 3 and 4 decoding and Group 4 encoding, lossless JBIG2 encoding" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +# CCITT Group 4 encoding (ITU-T T.6). MIT. +fax = "0.2.7" +# CCITT fax decoding, the decoder the vendored PDF reader uses. MIT OR Apache-2.0. +hayro-ccitt = { version = "0.3", default-features = false } +thiserror = { workspace = true } + +[dev-dependencies] +# The JBIG2 decoder the vendored PDF reader uses: reads back what the encoder writes. +hayro-jbig2 = { version = "=0.3.0", default-features = false, features = ["std"] } +proptest = { workspace = true } +weezl = "0.2" diff --git a/crates/stream-codecs/README.md b/crates/stream-codecs/README.md new file mode 100644 index 000000000..3a6c73208 --- /dev/null +++ b/crates/stream-codecs/README.md @@ -0,0 +1,78 @@ +# craft-stream-codecs + +Byte-stream and bi-level image codecs that PDF, PostScript and TIFF share. The crate knows no +container: a caller hands it bytes and parameters. A PDF `/Filter` chain (stage order, limits +across stages, Flate, accounting) belongs to `craft-pdf`, which builds on this crate; a TIFF +reader can use the same LZW, predictor, CCITT and JBIG2 code. + +- **Licence:** MIT OR Apache-2.0. Written from ISO 32000-2 §7.4, TIFF 6.0 §14 (predictor 2) and + RFC 2083 §6 (PNG filters). CCITT Group 4 encoding (ITU-T T.6) is the `fax` crate's (MIT). CCITT + decoding (T.4 and T.6) is `hayro-ccitt`'s (MIT OR Apache-2.0), bounded here: this crate has + no CCITT decoder of its own, and no JBIG2 decoder (that is `hayro-jbig2`, which the tests + read the encoder's output back with). JBIG2 encoding + is written from the public ISO/IEC 14492 Final Committee Draft (WG1 N1359, 1999); Annex H.2 + checks the arithmetic coder byte for byte. +- No `unsafe`. Never panics on input. +- From PdfCraft's `pdfcraft-filters` crate (revision `80e26e674d1aa33078e83c8139875047f3289eb6`): + its codec modules, moved whole. Its filter chain and Flate stay with the PDF code. + +## API + +```rust +pub struct Params { predictor, colors, bits_per_component, columns, early_change } // i64s; Default = 1, 1, 8, 1, 1 +pub enum CodecError { Corrupt { filter, detail }, LimitExceeded(usize), Unsupported(String), Spent } +pub struct Failure { error, partial, discarded } // what a decoder decoded before it failed +pub type Step = Result, Failure>; +pay(work: &mut u64, steps) -> Result<(), CodecError> // an allowance shared across a run of images + +lzw::decode(data, early_change, max) -> Step; lzw::encode(data, early_change) -> Vec +ascii85::decode(data, max) -> Step; ascii85::encode(data) -> Vec +asciihex::decode(data, max) -> Step; asciihex::encode(data) -> Vec +runlength::decode(data, max) -> Step; runlength::encode(data) -> Vec +predictor::decode(¶ms, data, name, rows) -> Step; predictor::encode(¶ms, data) -> Result, CodecError> + +pub struct CcittParams { k, columns, rows, end_of_line, encoded_byte_align, end_of_block, black_is_1 } +decode_ccitt_within(data, ¶ms, max_rows, work: &mut u64) -> Result, CodecError> +encode_g4(data, columns, rows) -> Result, CodecError> + +encode_jbig2_generic(data, columns, rows) -> Result, CodecError> +encode_jbig2(data, columns, rows) -> Result, CodecError> +encode_jbig2_shared(&[(data, columns, rows)]) -> Result, CodecError> +``` + +`max` bounds a decoder's output and gives `LimitExceeded`. `Corrupt` names the codec in `filter` +by its PDF filter name. Bi-level images are rows of 1-bit samples, most significant bit first, +each row on a new byte. + +## Behaviour + +| Codec | Decode | Encode | +|---|---|---| +| LZW | 9–12-bit MSB-first codes, clear (256), end (257), early change on or off; a missing end code or initial clear is tolerated; a full table stops growing | greedy, leading clear, clear when the table fills, end code | +| Predictors | 2 = TIFF (1/2/4/8/16 bits per component), 10–15 = PNG with per-row tags 0–4; partial final rows decoded; bad parameters or tags are `Corrupt`; with `rows`, data that is exactly that many rows without tag bytes is taken as the samples and reported as damage | 2, 10 None, 11 Sub, 12 Up, 13 Average, 14 Paeth, 15 per-row optimum | +| ASCIIHex | white space ignored, odd digit padded with 0, `>` ends (optional), other characters are `Corrupt` | upper case, newline every 64 digits, `>` | +| ASCII85 | `<~` prefix, `z`, white space, `~>` (or bare `~`, or none), final partial group; `z` inside a group, a 1-character final group, overflow or bad characters are `Corrupt` | `z` for zero groups, newline every 75 characters, `~>` | +| RunLength | 0–127 literal, 129–255 repeat, 128 ends (optional); a truncated run is `Corrupt` | runs of 2 or more repeat, literals up to 128, 128 | +| CCITT fax | Group 3 (1-D and 2-D) and Group 4 through `hayro-ccitt`, as rows of 1-bit samples, black 0 unless `black_is_1`, at most `max_rows` rows (fewer with `rows`), the rows before any damage, up to `MAX_CCITT_COLUMNS` (1,048,576) samples a row; the rows are paid for from `work` before they are decoded | `encode_g4`: Group 4 with an end-of-block mark, 0 bits coded black; up to `MAX_G4_COLUMNS` (32,767) samples a row | +| JBIG2 | not here: `hayro-jbig2` | `encode_jbig2_generic`: one immediate lossless generic region (template 0, typical prediction, MQ coder). `encode_jbig2`: the smaller of that and symbol coding (exact copies only, so the stream decodes to the same bits). `encode_jbig2_shared`: a group of images sharing one symbol dictionary (the PDF `/JBIG2Globals` data) | + +The limits of the JBIG2 encoder (symbols per plan, instances per text region, marks, runs) are +those `hayro-jbig2` reads back; they are documented on the constants in `src/jbig2/symbols.rs`. + +## Tests + +- Unit tests per codec with hand-computed vectors (the ISO 32000-2 LZW example, PNG and TIFF rows). +- LZW is cross-checked against `weezl` (dev-dependency) in both directions for both early-change settings. +- `ccitt.rs`: decoding gives back the rows `encode_g4` was given; Group 3 layouts over seeded + pictures; Group 4 output decoded by `hayro-ccitt` matches the input bits, with property tests + over bitmaps of noise and of runs; any input returns rather than panics. +- `mq.rs`: the arithmetic coder reproduces Annex H.2's 30 bytes from its 256 decisions. +- `jbig2.rs`, `jbig2/symbols.rs`: output decoded by `hayro-jbig2` matches the input bits for + every coding, plan and strip size, with property tests; size and limit refusals. + +## Not here + +A bounded JBIG2 decoding entry (`decode_jbig2_within` in PdfCraft): it needs the caller-owned +work allowance PdfCraft added to its `hayro-jbig2` (`Image::decode_within`, `charge_reading`), +which the copy vendored here does not have yet. Flate (a container's choice of zlib settings and tolerances), DCT and JPEG 2000, CCITT Group 3 +encoding. diff --git a/crates/stream-codecs/src/ascii85.rs b/crates/stream-codecs/src/ascii85.rs new file mode 100644 index 000000000..beb0098fd --- /dev/null +++ b/crates/stream-codecs/src/ascii85.rs @@ -0,0 +1,156 @@ +//! ASCII85Decode (ISO 32000-2 §7.4.3): groups of five characters `!`..`u` encode four +//! bytes base-85, `z` stands for four zero bytes, `~>` is EOD, white space is ignored and +//! a final partial group of n characters (2–5) yields n − 1 bytes. A leading `<~` and a +//! missing EOD are tolerated. + +use crate::{Failure, Step, is_pdf_whitespace}; + +const NAME: &str = "ASCII85Decode"; + +fn push(out: &mut Vec, bytes: &[u8], max: usize) -> Result<(), Failure> { + if out.len() + bytes.len() > max { + return Err(Failure::limit(max)); + } + out.extend_from_slice(bytes); + Ok(()) +} + +pub fn decode(data: &[u8], max: usize) -> Step { + let start = data.iter().position(|&b| !is_pdf_whitespace(b)).unwrap_or(data.len()); + let body = data[start..].strip_prefix(b"<~").unwrap_or(&data[start..]); + let mut out = Vec::with_capacity((body.len() / 5 * 4 + 4).min(max)); + let mut acc: u64 = 0; + let mut n = 0usize; + for (i, &b) in body.iter().enumerate() { + match b { + b'!'..=b'u' => { + acc = acc * 85 + u64::from(b - b'!'); + n += 1; + if n == 5 { + let Ok(v) = u32::try_from(acc) else { + return Err(Failure::corrupt(NAME, format!("group ending at {i} exceeds 2^32"), out)); + }; + push(&mut out, &v.to_be_bytes(), max)?; + acc = 0; + n = 0; + } + } + b'z' if n == 0 => push(&mut out, &[0; 4], max)?, + b'z' => return Err(Failure::corrupt(NAME, format!("'z' inside a group at {i}"), out)), + // `~` starts the EOD marker; tolerate it without the following `>`. + b'~' => break, + b if is_pdf_whitespace(b) => {} + b => return Err(Failure::corrupt(NAME, format!("invalid character 0x{b:02x} at {i}"), out)), + } + } + match n { + 0 => {} + 1 => return Err(Failure::corrupt(NAME, "final group has a single character", out)), + _ => { + // Pad with `u` (84) to a full group and keep n − 1 bytes. + for _ in n..5 { + acc = acc * 85 + 84; + } + let Ok(v) = u32::try_from(acc) else { + return Err(Failure::corrupt(NAME, "final group exceeds 2^32", out)); + }; + push(&mut out, &v.to_be_bytes()[..n - 1], max)?; + } + } + Ok(out) +} + +/// Encodes with `z` for zero groups, a newline every 75 characters, and a `~>` EOD. +pub fn encode(data: &[u8]) -> Vec { + let mut out = Vec::with_capacity(data.len() * 5 / 4 + data.len() / 60 + 8); + let mut col = 0; + let mut put = |out: &mut Vec, s: &[u8]| { + for &c in s { + if col == 75 { + out.push(b'\n'); + col = 0; + } + out.push(c); + col += 1; + } + }; + for chunk in data.chunks(4) { + let mut word = [0u8; 4]; + word[..chunk.len()].copy_from_slice(chunk); + let mut v = u32::from_be_bytes(word); + if v == 0 && chunk.len() == 4 { + put(&mut out, b"z"); + continue; + } + let mut digits = [0u8; 5]; + for d in digits.iter_mut().rev() { + *d = (v % 85) as u8 + b'!'; + v /= 85; + } + put(&mut out, &digits[..chunk.len() + 1]); + } + out.extend_from_slice(b"~>"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::CodecError; + + fn dec(d: &[u8]) -> Result, CodecError> { + decode(d, 1000).map_err(|f| f.error) + } + + #[test] + fn vectors() { + // "Man " = 0x4D616E20 = 24·85⁴ + 73·85³ + 80·85² + 78·85 + 61 -> "9jqo^". + assert_eq!(dec(b"9jqo^~>").unwrap(), b"Man "); + assert_eq!(encode(b"Man "), b"9jqo^~>"); + // "M" padded to 0x4D000000 -> "9`" (24, 63); decoding pads with 'u'. + assert_eq!(encode(b"M"), b"9`~>"); + assert_eq!(dec(b"9`~>").unwrap(), b"M"); + assert_eq!(dec(b"z~>").unwrap(), [0; 4]); + assert_eq!(encode(&[0; 4]), b"z~>"); + // A partial zero group is not `z`. + assert_eq!(encode(&[0; 3]), b"!!!!~>"); + assert_eq!(dec(b"!!!!~>").unwrap(), [0; 3]); + assert_eq!(dec(b"s8W-!~>").unwrap(), [0xff; 4]); + } + + #[test] + fn syntax_tolerance() { + assert_eq!(dec(b"<~9jqo^~>").unwrap(), b"Man "); + assert_eq!(dec(b" \n<~9j\r\nq o\t^z~>").unwrap(), b"Man \0\0\0\0"); + assert_eq!(dec(b"9jqo^").unwrap(), b"Man "); // missing EOD + assert_eq!(dec(b"9jqo^~").unwrap(), b"Man "); // bare ~ + assert_eq!(dec(b"9jqo^~>garbage{").unwrap(), b"Man "); + assert_eq!(dec(b"~>").unwrap(), b""); + assert_eq!(dec(b"").unwrap(), b""); + } + + #[test] + fn corrupt() { + assert!(matches!(dec(b"9jz~>"), Err(CodecError::Corrupt { .. }))); + assert!(matches!(dec(b"9jqo^9~>"), Err(CodecError::Corrupt { .. }))); + assert!(matches!(dec(b"uuuuu~>"), Err(CodecError::Corrupt { .. }))); // > 2^32 - 1 + assert!(matches!(dec(b"9jqo^v~>"), Err(CodecError::Corrupt { .. }))); + let f = decode(b"9jqo^{", 100).err().unwrap(); + assert_eq!(f.partial, b"Man "); + } + + #[test] + fn limit() { + assert!(matches!(decode(b"zz", 7).map_err(|f| f.error), Err(CodecError::LimitExceeded(7)))); + assert!(matches!(decode(b"z9`", 4).map_err(|f| f.error), Err(CodecError::LimitExceeded(4)))); + assert_eq!(decode(b"zz", 8).ok().unwrap().len(), 8); + } + + #[test] + fn line_wrapping() { + let data: Vec = (0..=255u8).collect(); + let e = encode(&data); + assert!(e.split(|&b| b == b'\n').all(|l| l.len() <= 77)); + assert_eq!(dec(&e).unwrap_or_default().len(), 256); + } +} diff --git a/crates/stream-codecs/src/asciihex.rs b/crates/stream-codecs/src/asciihex.rs new file mode 100644 index 000000000..60e962216 --- /dev/null +++ b/crates/stream-codecs/src/asciihex.rs @@ -0,0 +1,106 @@ +//! ASCIIHexDecode (ISO 32000-2 §7.4.2): pairs of hex digits, white space ignored, +//! `>` is EOD, an odd final digit is padded with 0. A missing EOD is tolerated. + +use crate::{Failure, Step, is_pdf_whitespace}; + +const NAME: &str = "ASCIIHexDecode"; + +fn nibble(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +pub fn decode(data: &[u8], max: usize) -> Step { + let mut out = Vec::with_capacity((data.len() / 2).min(max)); + let mut hi: Option = None; + for (i, &b) in data.iter().enumerate() { + if is_pdf_whitespace(b) { + continue; + } + if b == b'>' { + break; + } + let Some(n) = nibble(b) else { + return Err(Failure::corrupt(NAME, format!("invalid character 0x{b:02x} at offset {i}"), out)); + }; + match hi.take() { + None => hi = Some(n), + Some(h) => { + if out.len() == max { + return Err(Failure::limit(max)); + } + out.push(h << 4 | n); + } + } + } + if let Some(h) = hi { + if out.len() == max { + return Err(Failure::limit(max)); + } + out.push(h << 4); + } + Ok(out) +} + +/// Upper-case hex, a newline every 64 digits, terminated by `>`. +pub fn encode(data: &[u8]) -> Vec { + const HEX: &[u8; 16] = b"0123456789ABCDEF"; + let mut out = Vec::with_capacity(data.len() * 2 + data.len() / 32 + 1); + for (i, &b) in data.iter().enumerate() { + if i > 0 && i % 32 == 0 { + out.push(b'\n'); + } + out.push(HEX[usize::from(b >> 4)]); + out.push(HEX[usize::from(b & 15)]); + } + out.push(b'>'); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::CodecError; + + fn dec(d: &[u8]) -> Result, CodecError> { + decode(d, 1000).map_err(|f| f.error) + } + + #[test] + fn vectors() { + assert_eq!(dec(b"48656c6C6F>").unwrap(), b"Hello"); + assert_eq!(dec(b" 4 8\n6\r5\t6\x0c c\x006c 6f >").unwrap(), b"Hello"); + assert_eq!(dec(b"901FA>").unwrap(), [0x90, 0x1f, 0xa0]); + assert_eq!(dec(b"901FA").unwrap(), [0x90, 0x1f, 0xa0]); // missing EOD + assert_eq!(dec(b"41>zz garbage after EOD").unwrap(), b"A"); + assert_eq!(dec(b">").unwrap(), b""); + assert_eq!(dec(b"").unwrap(), b""); + } + + #[test] + fn invalid_char() { + let f = decode(b"4142G43>", 1000).err().unwrap(); + assert!(matches!(f.error, CodecError::Corrupt { .. })); + assert_eq!(f.partial, b"AB"); + } + + #[test] + fn limit() { + assert!(matches!(decode(b"414243", 2).map_err(|f| f.error), Err(CodecError::LimitExceeded(2)))); + assert!(matches!(decode(b"41424", 2).map_err(|f| f.error), Err(CodecError::LimitExceeded(2)))); + assert_eq!(decode(b"4142", 2).ok().unwrap(), b"AB"); + } + + #[test] + fn encoding() { + assert_eq!(encode(b"Hello"), b"48656C6C6F>"); + assert_eq!(encode(b""), b">"); + let long = encode(&[0xab; 40]); + assert_eq!(long[64], b'\n'); + assert_eq!(dec(&long).unwrap(), [0xab; 40]); + } +} diff --git a/crates/stream-codecs/src/ccitt.rs b/crates/stream-codecs/src/ccitt.rs new file mode 100644 index 000000000..2733cf98b --- /dev/null +++ b/crates/stream-codecs/src/ccitt.rs @@ -0,0 +1,546 @@ +//! CCITT fax for `CCITTFaxDecode` (ISO 32000-2 §7.4.6): Group 4 (ITU-T T.6) encoding, with `/K -1`, +//! and decoding of Group 3 (T.4) and Group 4. The coding is the `fax` crate's and the decoding the +//! `hayro-ccitt` crate's; this module checks what they are given and bounds what they give. + +use fax::encoder::Encoder; +use fax::{Color, VecWriter}; + +use crate::CodecError; + +/// The widest row [`encode_g4`] encodes. The encoder counts the positions in a row in 16 bits +/// and adds two of them, so a row stays within half that range. +pub const MAX_G4_COLUMNS: u32 = 32_767; + +/// `rows` rows of `columns` 1-bit samples, most significant bit first, each row starting on a new +/// byte (as a PDF image's samples are packed), encoded as CCITT Group 4 with an end-of-block mark. +/// +/// A 0 bit is coded black and a 1 bit white, so the stream decodes to the same bits with +/// `/BlackIs1 false`, the default: an image keeps its colour space and decode array. Bytes after +/// the last row are ignored, as are the bits that pad a row to a whole byte. +/// +/// Errors: no rows or columns, or rows wider than [`MAX_G4_COLUMNS`] ([`CodecError::Unsupported`]); +/// fewer bytes than the rows need ([`CodecError::Corrupt`]). +pub fn encode_g4(data: &[u8], columns: u32, rows: u32) -> Result, CodecError> { + let width = match u16::try_from(columns) { + Ok(w) if w > 0 && columns <= MAX_G4_COLUMNS => w, + _ => return Err(CodecError::Unsupported(format!("CCITT Group 4 encoding of rows {columns} samples wide"))), + }; + if rows == 0 { + return Err(CodecError::Unsupported("CCITT Group 4 encoding of an image without rows".into())); + } + let stride = usize::from(width).div_ceil(8); + let samples = usize::try_from(rows).ok().and_then(|r| r.checked_mul(stride)).and_then(|n| data.get(..n)).ok_or_else(|| CodecError::Corrupt { + filter: "CCITTFaxDecode", + detail: format!("{} bytes for {rows} rows of {columns} samples", data.len()), + })?; + let mut encoder = Encoder::new(VecWriter::new()); + for row in samples.chunks_exact(stride) { + let pels = (0..usize::from(width)).map(|x| { + let white = row.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0); + if white { Color::White } else { Color::Black } + }); + // `VecWriter` writes to memory: its error type has no values. + if let Err(never) = encoder.encode_line(pels, width) { + match never {} + } + } + match encoder.finish() { + Ok(writer) => Ok(writer.finish()), + Err(never) => match never {}, + } +} + +/// The widest row [`decode_ccitt_within`] decodes: its row buffer stays within 128 KiB. +pub const MAX_CCITT_COLUMNS: u32 = 1 << 20; + +/// A `CCITTFaxDecode` stream's `/DecodeParms` (ISO 32000-2 Table 11). [`Default`] gives the +/// table's defaults. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CcittParams { + /// `/K`: negative for Group 4, 0 for Group 3 one-dimensional, positive for Group 3 mixed. + pub k: i64, + pub columns: i64, + /// `/Rows`: 0 when not given. + pub rows: i64, + pub end_of_line: bool, + pub encoded_byte_align: bool, + pub end_of_block: bool, + pub black_is_1: bool, +} + +impl Default for CcittParams { + fn default() -> Self { + CcittParams { k: 0, columns: 1728, rows: 0, end_of_line: false, encoded_byte_align: false, end_of_block: true, black_is_1: false } + } +} + +/// What [`decode_ccitt_within`] decodes once it is paid for. +fn decode_ccitt(data: &[u8], p: &CcittParams, max_rows: u32) -> Result, CodecError> { + let columns = u32::try_from(p.columns) + .ok() + .filter(|c| (1..=MAX_CCITT_COLUMNS).contains(c)) + .ok_or_else(|| CodecError::Unsupported(format!("CCITT fax rows {} samples wide", p.columns)))?; + let rows = u32::try_from(p.rows).ok().filter(|r| *r > 0).map_or(max_rows, |r| r.min(max_rows)); + if rows == 0 { + return Ok(Vec::new()); + } + let settings = hayro_ccitt::DecodeSettings { + columns, + rows, + end_of_block: p.end_of_block, + end_of_line: p.end_of_line, + // Group 3 rows on bytes may have end-of-line codes whatever `end_of_line` says (the filter + // always accepts them), with the fill starting each code on a byte or ending it on one (T.4's + // fill, as TIFF writes it). Read unaligned, the codes mark the rows either way; without + // codes, the rows are read on bytes. A reading that comes up short isn't tried the other + // way: data cut short would then read as other rows. + rows_are_byte_aligned: p.encoded_byte_align && (p.k < 0 || !starts_with_end_of_line(data)), + encoding: match p.k { + ..0 => hayro_ccitt::EncodingMode::Group4, + 0 => hayro_ccitt::EncodingMode::Group3_1D, + k => hayro_ccitt::EncodingMode::Group3_2D { k: u32::try_from(k).unwrap_or(u32::MAX) }, + }, + invert_black: false, + }; + // u32 to usize: wasm32 and up have at least 32-bit usize. + let mut out = Rows::new(columns as usize, rows as usize, p.black_is_1); + // The rows decoded before damaged data are kept. + let _ = hayro_ccitt::decode(data, &mut out, &mut hayro_ccitt::DecoderContext::new(settings)); + Ok(out.out) +} + +/// The steps of reading a byte of fax code ([`decode_ccitt_within`]): noise takes 114 ns a byte of +/// rows and about two bytes of code to a byte of rows, so that and a step a byte of rows is about +/// what a step of JBIG2 decoding takes. +const CODE_WORK: u64 = 4; + +/// CCITT fax `data` decoded as `CCITTFaxDecode` gives it: rows of `columns` 1-bit samples, most +/// significant bit first, each starting on a new byte, with black as 0 (as 1 with `black_is_1`). +/// +/// At most `max_rows` rows are decoded (fewer when `rows` says so), so the output is bounded by +/// the caller. Damaged data gives the rows decoded before the damage, as viewers show them: a +/// caller that needs a number of rows checks it got them. +/// +/// Group 3 rows on bytes (`k` from 0, `encoded_byte_align`) are read by their end-of-line codes, +/// whether the fill starts each code on a byte or ends it on one, when `data` starts with one (T.4 +/// sends one before the first row when it sends them at all), and on bytes otherwise, whatever +/// `end_of_line` says. Group 4 is read as its parameters say. Data with codes but none before the +/// first row is read on bytes, as hayro's renderer draws it; with `k` 0 and the fill ending each +/// code on a byte, that may be other rows than were coded. +/// +/// Errors: `columns` that aren't from 1 to [`MAX_CCITT_COLUMNS`] ([`CodecError::Unsupported`]). +/// +/// Decoding spends from `work`, what decoding may still cost across a run of images, and there is +/// no way in without it: a +/// byte of rows a step, whether or not the data holds them, and [`CODE_WORK`] a byte of `data`. +/// A blank row takes a bit of code, so the rows are what a few bytes can ask for. It is all paid +/// before anything is decoded, and what `work` can't pay for spends all of it. +pub fn decode_ccitt_within(data: &[u8], p: &CcittParams, max_rows: u32, work: &mut u64) -> Result, CodecError> { + let rows = u32::try_from(p.rows).ok().filter(|r| *r > 0).map_or(max_rows, |r| r.min(max_rows)); + let stride = u64::try_from(p.columns).unwrap_or(0).div_ceil(8); + crate::pay(work, stride.saturating_mul(u64::from(rows)).saturating_add((data.len() as u64).saturating_mul(CODE_WORK)))?; + decode_ccitt(data, p, max_rows) +} + +/// Whether CCITT fax `data` starts with an end-of-line code: a 1 after 11 zeros or more, fill +/// included. T.4 sends one before the first row when it sends them at all, and no other code starts +/// with more than eight zeros. +fn starts_with_end_of_line(data: &[u8]) -> bool { + let Some((i, b)) = data.iter().enumerate().find(|(_, b)| **b != 0) else { return false }; + i.saturating_mul(8).saturating_add(b.leading_zeros() as usize) >= 11 +} + +/// Rows of 1-bit samples filled from a decoder's pixels: `columns` a row, each row on a new byte, at +/// most `rows` rows. Pixels past a row's end and rows past the last are dropped. +pub(crate) struct Rows { + pub(crate) out: Vec, + row: Vec, + x: usize, + columns: usize, + rows: usize, + done: usize, + black_is_1: bool, +} + +impl Rows { + pub(crate) fn new(columns: usize, rows: usize, black_is_1: bool) -> Self { + Rows { out: Vec::new(), row: vec![0; columns.div_ceil(8)], x: 0, columns, rows, done: 0, black_is_1 } + } + + /// `count` pixels, white or black. + pub(crate) fn push(&mut self, white: bool, count: usize) { + let end = self.x.saturating_add(count); + if white != self.black_is_1 { + for x in self.x..end.min(self.columns) { + if let Some(b) = self.row.get_mut(x / 8) { + *b |= 0x80 >> (x % 8); + } + } + } + self.x = end; + } + + /// The row filled so far kept, if fewer than `rows` are, and the next one begun. + pub(crate) fn end_row(&mut self) { + if self.done < self.rows { + self.out.extend_from_slice(&self.row); + self.done += 1; + } + self.row.fill(0); + self.x = 0; + } +} + +impl hayro_ccitt::Decoder for Rows { + fn push_pixel(&mut self, white: bool) { + self.push(white, 1); + } + fn push_pixel_chunk(&mut self, white: bool, chunks: u32) { + self.push(white, (chunks as usize).saturating_mul(8)); + } + fn next_line(&mut self) { + self.end_row(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A run pays for the rows an image asks for and the code it reads, before any of it is + /// decoded: rows the data doesn't hold are paid for too, as a few bytes can ask for any number. + #[test] + fn a_run_of_images_pays_for_the_rows_it_asks_for() { + let (w, h) = (4096u32, 64u32); + let stride = (w as usize).div_ceil(8); + let data = encode_g4(&vec![0xFF; stride * h as usize], w, h).unwrap(); + let p = CcittParams { k: -1, columns: i64::from(w), ..CcittParams::default() }; + let whole = decode_ccitt(&data, &p, h).unwrap(); + let cost = (whole.len() + 4 * data.len()) as u64; + let mut work = cost; + assert_eq!(decode_ccitt_within(&data, &p, h, &mut work).unwrap(), whole, "exactly enough"); + assert_eq!(work, 0); + let mut work = cost + 7; + decode_ccitt_within(&data, &p, h, &mut work).unwrap(); + assert_eq!(work, 7, "only what it took"); + let mut work = cost - 1; + let refused = decode_ccitt_within(&data, &p, h, &mut work).unwrap_err(); + assert!(matches!(refused, CodecError::Spent), "a step short: {refused}"); + assert_eq!(work, 0, "spends all that was left"); + // Nothing left: not read at all, so bad columns aren't reported as bad columns. + let bad = CcittParams { columns: 0, ..p }; + assert!(decode_ccitt_within(&data, &bad, h, &mut 0).is_err_and(|e| matches!(e, CodecError::Spent))); + // No code at all, for the same rows. + let mut work = u64::MAX; + decode_ccitt_within(&[], &p, h, &mut work).unwrap(); + assert_eq!(u64::MAX - work, whole.len() as u64); + // `/Rows` below what is asked for, and no rows. + let mut work = u64::MAX; + decode_ccitt_within(&data, &CcittParams { rows: 3, ..p }, h, &mut work).unwrap(); + assert_eq!(u64::MAX - work, (3 * stride + 4 * data.len()) as u64); + let mut work = u64::MAX; + decode_ccitt_within(&data, &p, 0, &mut work).unwrap(); + assert_eq!(u64::MAX - work, 4 * data.len() as u64); + } + + /// The rows decoded by an independent decoder (`hayro-ccitt`), as 1 for white and 0 for black. + pub(crate) fn decode(data: &[u8], columns: u32, rows: u32) -> Vec> { + struct Rows(Vec>, Vec, usize); + impl hayro_ccitt::Decoder for Rows { + fn push_pixel(&mut self, white: bool) { + self.1.push(white); + } + fn push_pixel_chunk(&mut self, white: bool, chunks: u32) { + self.1.extend(std::iter::repeat_n(white, chunks as usize * 8)); + } + fn next_line(&mut self) { + let mut row = std::mem::take(&mut self.1); + row.truncate(self.2); + self.0.push(row); + } + } + let settings = hayro_ccitt::DecodeSettings { + columns, + rows, + end_of_block: true, + end_of_line: false, + rows_are_byte_aligned: false, + encoding: hayro_ccitt::EncodingMode::Group4, + invert_black: false, + }; + let mut out = Rows(Vec::new(), Vec::new(), columns as usize); + hayro_ccitt::decode(data, &mut out, &mut hayro_ccitt::DecoderContext::new(settings)).unwrap(); + out.0 + } + + /// The rows of packed samples `data`, as 1 for a 1 bit. + pub(crate) fn unpack(data: &[u8], columns: u32, rows: u32) -> Vec> { + let stride = (columns as usize).div_ceil(8); + (0..rows as usize).map(|y| (0..columns as usize).map(|x| data[y * stride + x / 8] & (0x80 >> (x % 8)) != 0).collect()).collect() + } + + #[test] + fn a_page_of_text_like_marks_round_trips_and_shrinks() { + let (w, h) = (1700u32, 220u32); + let stride = (w as usize).div_ceil(8); + let mut data = vec![0xFF; stride * h as usize]; + // Short black strokes in lines, like glyphs on a white page. + for y in (20..h as usize).step_by(40) { + for dy in 0..12 { + for x in (100..1600).step_by(9) { + if (x / 9 + dy) % 5 != 0 { + data[(y + dy) * stride + x / 8] &= !(0x80 >> (x % 8)); + } + } + } + } + let g4 = encode_g4(&data, w, h).unwrap(); + assert_eq!(decode(&g4, w, h), unpack(&data, w, h)); + assert!(g4.len() * 4 < data.len(), "{} bytes from {}", g4.len(), data.len()); + } + + #[test] + fn padding_bits_and_trailing_bytes_are_ignored() { + // 3 columns: the last 5 bits of each byte pad the row; set them to anything. + let data = [0b1010_1111, 0b0101_0000, 0b1110_1010, 0xAB, 0xCD]; + let g4 = encode_g4(&data, 3, 3).unwrap(); + assert_eq!(decode(&g4, 3, 3), [[true, false, true], [false, true, false], [true, true, true]]); + } + + #[test] + fn what_it_cannot_encode_is_an_error() { + assert!(matches!(encode_g4(&[0; 8], 0, 1), Err(CodecError::Unsupported(_)))); + assert!(matches!(encode_g4(&[0; 8], 8, 0), Err(CodecError::Unsupported(_)))); + assert!(matches!(encode_g4(&vec![0; 4097], MAX_G4_COLUMNS + 1, 1), Err(CodecError::Unsupported(_)))); + assert!(matches!(encode_g4(&[0; 3], 16, 2), Err(CodecError::Corrupt { .. })), "two rows of 16 need 4 bytes"); + assert!(matches!(encode_g4(&[], 1, u32::MAX), Err(CodecError::Corrupt { .. }))); + let widest = vec![0x55; (MAX_G4_COLUMNS as usize).div_ceil(8) * 2]; + let g4 = encode_g4(&widest, MAX_G4_COLUMNS, 2).unwrap(); + assert_eq!(decode(&g4, MAX_G4_COLUMNS, 2), unpack(&widest, MAX_G4_COLUMNS, 2)); + } + + /// Rows of `columns` samples built from `runs`: alternating white and black runs one longer + /// than those values (so never empty, even when shrinking makes them all 0), repeated until the + /// rows are full; or, with `noise`, the bytes of `runs` as they are. + fn bitmap(columns: u32, rows: u32, runs: &[u8], noise: bool) -> Vec { + let stride = (columns as usize).div_ceil(8); + let mut data = vec![0u8; stride * rows as usize]; + if runs.is_empty() { + return data; + } + if noise { + data.iter_mut().zip(runs.iter().cycle()).for_each(|(d, r)| *d = *r); + return data; + } + let (mut white, mut left, mut next) = (true, 1 + usize::from(runs[0]), 1); + for y in 0..rows as usize { + for x in 0..columns as usize { + if left == 0 { + white = !white; + left = 1 + usize::from(runs[next % runs.len()]); + next += 1; + } + if white { + data[y * stride + x / 8] |= 0x80 >> (x % 8); + } + left -= 1; + } + } + data + } + + fn g4(columns: u32) -> CcittParams { + CcittParams { k: -1, columns: i64::from(columns), ..CcittParams::default() } + } + + #[test] + fn decoding_gives_the_rows_back() { + // 3 columns, padded with anything: the decoded padding is 0. + let data = [0b1010_1111, 0b0101_0000, 0b1110_1010]; + let coded = encode_g4(&data, 3, 3).unwrap(); + assert_eq!(decode_ccitt(&coded, &g4(3), 3).unwrap(), [0b1010_0000, 0b0100_0000, 0b1110_0000]); + // /BlackIs1 gives black as 1. + let p = CcittParams { black_is_1: true, ..g4(3) }; + assert_eq!(decode_ccitt(&coded, &p, 3).unwrap(), [0b0100_0000, 0b1010_0000, 0b0000_0000]); + // At most /Rows rows, and at most max_rows. + assert_eq!(decode_ccitt(&coded, &CcittParams { rows: 2, ..g4(3) }, 3).unwrap().len(), 2); + assert_eq!(decode_ccitt(&coded, &g4(3), 1).unwrap().len(), 1); + assert_eq!(decode_ccitt(&coded, &CcittParams { rows: 3, ..g4(3) }, 1).unwrap().len(), 1, "max_rows bounds /Rows"); + assert_eq!(decode_ccitt(&coded, &g4(3), 0).unwrap(), [0u8; 0]); + // Damaged data gives the rows before the damage; none here. + assert!(decode_ccitt(&coded[..1], &g4(3), 3).unwrap().len() < 3); + } + + #[test] + fn what_it_cannot_decode_is_an_error() { + for columns in [0, -1, i64::from(MAX_CCITT_COLUMNS) + 1, i64::MAX] { + let p = CcittParams { columns, ..CcittParams::default() }; + assert!(matches!(decode_ccitt(&[0; 8], &p, 4), Err(CodecError::Unsupported(_))), "{columns}"); + } + assert_eq!(CcittParams::default().columns, 1728, "Table 11"); + } + + /// Group 3 data: rows of run lengths below 64, white first, coded one-dimensionally with T.4's + /// terminating codes (ITU-T T.4 Tables 2 and 3). Each row starts with an end-of-line code with + /// `eol`, then with `tag` a 1 bit (the next row one-dimensional, as `k` above 0 has it). Fill + /// bits (0s) go where `fill` says. + fn group_3(rows: &[Vec], eol: bool, tag: bool, fill: Fill) -> Vec { + // The codes for runs 0 to 63, in order. + const WHITE: &str = concat!( + "00110101 000111 0111 1000 1011 1100 1110 1111 10011 10100 00111 01000 001000 000011 110100 110101 101010 101011 ", + "0100111 0001100 0001000 0010111 0000011 0000100 0101000 0101011 0010011 0100100 0011000 00000010 00000011 00011010 ", + "00011011 00010010 00010011 00010100 00010101 00010110 00010111 00101000 00101001 00101010 00101011 00101100 00101101 ", + "00000100 00000101 00001010 00001011 01010010 01010011 01010100 01010101 00100100 00100101 01011000 01011001 01011010 ", + "01011011 01001010 01001011 00110010 00110011 00110100", + ); + const BLACK: &str = concat!( + "0000110111 010 11 10 011 0011 0010 00011 000101 000100 0000100 0000101 0000111 00000100 00000111 000011000 0000010111 ", + "0000011000 0000001000 00001100111 00001101000 00001101100 00000110111 00000101000 00000010111 00000011000 ", + "000011001010 000011001011 000011001100 000011001101 000001101000 000001101001 000001101010 000001101011 000011010010 ", + "000011010011 000011010100 000011010101 000011010110 000011010111 000001101100 000001101101 000011011010 000011011011 ", + "000001010100 000001010101 000001010110 000001010111 000001100100 000001100101 000001010010 000001010011 000000100100 ", + "000000110111 000000111000 000000100111 000000101000 000001011000 000001011001 000000101011 000000101100 000001011010 ", + "000001100110 000001100111", + ); + let code = |white: bool, run: u32| (if white { WHITE } else { BLACK }).split(' ').nth(run as usize).expect("a run below 64"); + let mut bits = String::new(); + for row in rows { + // The end-of-line code is 12 bits. + let ends_code = if fill == Fill::CodeEnd && eol { 12 } else { 0 }; + while fill != Fill::None && !(bits.len() + ends_code).is_multiple_of(8) { + bits.push('0'); + } + if eol { + bits.push_str("000000000001"); + } + if tag { + bits.push('1'); + } + row.iter().enumerate().for_each(|(i, run)| bits.push_str(code(i % 2 == 0, *run))); + } + bits.as_bytes().chunks(8).map(|c| c.iter().enumerate().fold(0u8, |b, (i, d)| b | (u8::from(*d == b'1') << (7 - i)))).collect() + } + + /// Where [`group_3`] puts fill bits: none; so each row, its end-of-line code first, starts on a + /// byte; or so each end-of-line code ends on one (T.4's fill, as TIFF writes it). + #[derive(Clone, Copy, Debug, PartialEq)] + enum Fill { + None, + RowStart, + CodeEnd, + } + + /// `rows` text-like rows of `columns` samples as runs, white first, from `seed`: long white runs + /// and short black ones; a row may start black. + fn text_like(seed: u64, columns: u32, rows: u32) -> Vec> { + let mut seed = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; + let mut random = |n: u64| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + (seed % n) as u32 + }; + (0..rows) + .map(|_| { + let (mut runs, mut left) = (Vec::new(), columns); + while left > 0 { + let run = match runs.len() % 2 { + 0 if runs.is_empty() => random(31), + 0 => 1 + random(30), + _ => 1 + random(5), + }; + runs.push(run.min(left)); + left -= run.min(left); + } + runs + }) + .collect() + } + + /// Rows of runs, white first, packed as `decode_ccitt` gives them (white 1). + fn pack(runs: &[Vec], columns: u32) -> Vec { + let stride = (columns as usize).div_ceil(8); + let mut out = vec![0u8; stride * runs.len()]; + for (y, row) in runs.iter().enumerate() { + let mut x = 0; + for (i, run) in row.iter().enumerate() { + if i % 2 == 0 { + (x..x + *run as usize).for_each(|x| out[y * stride + x / 8] |= 0x80 >> (x % 8)); + } + x += *run as usize; + } + } + out + } + + /// Group 3 rows on bytes in each layout, over many pictures, as how a misreading comes out + /// depends on the picture: no codes (as TIFF's Modified Huffman has it), and `end_of_line` + /// without them; codes, each starting on a byte or ending on one, `end_of_line` or not. Each + /// decodes to its picture, and a byte short to fewer rows, not to other rows read another way. + /// Group 4 rows on bytes keep their aligned read, `end_of_line` or not. + #[test] + fn group_3_rows_on_bytes_read_by_the_codes_in_the_data() { + for seed in 0..32u64 { + let (columns, rows) = (8 + (seed as u32 * 37) % 200, 2 + seed as u32 % 20); + let runs = text_like(seed, columns, rows); + let picture = pack(&runs, columns); + for k in [0, 1] { + for (codes, fill, end_of_line) in [ + (false, Fill::RowStart, false), + (false, Fill::RowStart, true), + (true, Fill::RowStart, false), + (true, Fill::RowStart, true), + (true, Fill::CodeEnd, false), + (true, Fill::CodeEnd, true), + ] { + let p = CcittParams { k, columns: i64::from(columns), end_of_line, encoded_byte_align: true, ..CcittParams::default() }; + let mut data = group_3(&runs, codes, k > 0, fill); + let case = format!("seed {seed}, {columns} × {rows}, k {k}, codes {codes}, fill {fill:?}, end_of_line {end_of_line}"); + assert_eq!(decode_ccitt(&data, &p, rows).unwrap(), picture, "{case}"); + data.pop(); + assert!(decode_ccitt(&data, &p, rows).unwrap().len() < picture.len(), "{case}, a byte short"); + } + } + } + // White rows, each the one code 1 (vertical, no change) and its fill. + for end_of_line in [false, true] { + let p = CcittParams { k: -1, columns: 64, end_of_line, encoded_byte_align: true, ..CcittParams::default() }; + assert_eq!(decode_ccitt(&[0x80; 16], &p, 16).unwrap(), [0xFF; 8 * 16], "Group 4, end_of_line {end_of_line}"); + } + } + + proptest::proptest! { + /// Any bitmap, of noise or of runs, decodes to its own bits, through an independent decoder + /// and through `decode_ccitt`. + #[test] + fn bitmaps_round_trip( + columns in 1u32..400, + rows in 1u32..24, + runs in proptest::collection::vec(0u8..=255, 1..48), + noise in proptest::bool::ANY, + ) { + let data = bitmap(columns, rows, &runs, noise); + let coded = encode_g4(&data, columns, rows).unwrap(); + proptest::prop_assert_eq!(decode(&coded, columns, rows), unpack(&data, columns, rows)); + let decoded = decode_ccitt(&coded, &g4(columns), rows).unwrap(); + proptest::prop_assert_eq!(unpack(&decoded, columns, rows), unpack(&data, columns, rows)); + } + + /// Whatever it is given, it returns. + #[test] + fn any_input_returns( + data in proptest::collection::vec(0u8..=255, 0..600), + columns in 0u32..40_000, + rows in 0u32..80, + k in -2i64..3, + end_of_line in proptest::bool::ANY, + encoded_byte_align in proptest::bool::ANY, + ) { + let _ = encode_g4(&data, columns, rows); + let p = CcittParams { k, columns: i64::from(columns), end_of_line, encoded_byte_align, ..CcittParams::default() }; + if let Ok(out) = decode_ccitt(&data, &p, rows) { + proptest::prop_assert!(out.len() <= (columns as usize).div_ceil(8) * rows as usize); + } + } + } +} diff --git a/crates/stream-codecs/src/jbig2.rs b/crates/stream-codecs/src/jbig2.rs new file mode 100644 index 000000000..0b8b76720 --- /dev/null +++ b/crates/stream-codecs/src/jbig2.rs @@ -0,0 +1,468 @@ +//! JBIG2 encoding (ITU-T T.88 | ISO/IEC 14492), for `JBIG2Decode` (ISO 32000-2 §7.4.7), lossless: +//! a generic region, template 0 with its nominal adaptive pixels, typical prediction on, MQ +//! arithmetic coding ([`crate::mq`]); or the image's marks as symbols ([`symbols`]), which a group +//! of images may share through `/JBIG2Globals`. Written from the standard's 6.2.5 (coding a generic +//! region), 7.2 (segment headers), 7.4.1, 7.4.6 and 7.4.8 (the segments) and D.3 (the embedded +//! organisation PDF uses). Decoding is in [`decode`]. + +mod symbols; + +/// The largest image the encoders take, in pixels: the largest JBIG2 page `hayro-jbig2` decodes +/// (it decodes a page whole, and takes 20,000 × 10,000), so what they store reads back. +pub const MAX_JBIG2_PIXELS: u64 = 1 << 28; + +use crate::CodecError; +use crate::mq::{Context, MqEncoder}; + +/// The widest row, and the most rows, [`encode_jbig2_generic`] encodes: what `hayro-jbig2`, the +/// decoder behind `JBIG2Decode` here, takes on a side. JBIG2 allows 32-bit sizes, but past this +/// nothing here could read the stream, and the optimizer, which doesn't read back what it stores, +/// replaced such an image with one that draws nothing. +pub const MAX_JBIG2_COLUMNS: u32 = 65_535; + +/// Template 0's pixels (Figure 3), as (x, y) offsets from the pixel being coded, each a bit of the +/// context in this order: the four on the current row, the five fixed ones on the row above, the +/// three on the row above that, then the adaptive pixels A1–A4 at their nominal places (Table 5). +const TEMPLATE: [(i32, i32); 16] = [ + (-1, 0), + (-2, 0), + (-3, 0), + (-4, 0), + (-2, -1), + (-1, -1), + (0, -1), + (1, -1), + (2, -1), + (-1, -2), + (0, -2), + (1, -2), + (3, -1), + (-3, -1), + (2, -2), + (-2, -2), +]; + +/// The context that codes SLTP when template 0 is used (Figure 8): those pixel values, in +/// [`TEMPLATE`]'s order. Figure 8 reads, by row from x = -4: `0 1 0 1` on the current row, +/// `0 1 1 0 0 1 0` (x = -3 to 3) above it, `1 0 0 1 1` (x = -2 to 2) above that. +const SLTP: [u8; 16] = [1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1]; + +/// Columns of padding either side of a row, so every template pixel of a row's pixels is in it. +const PAD: usize = 4; + +/// `rows` rows of `columns` 1-bit PDF samples as lossless JBIG2, the smaller of two codings: +/// [`encode_jbig2_generic`]'s, or one that stores each distinct mark of the image once, as a +/// symbol, and places its copies (see [`symbols`]). Either way the stream is in the embedded +/// organisation and decodes to the same bits without `/JBIG2Globals`. +/// +/// Errors: as [`encode_jbig2_generic`]. +pub fn encode_jbig2(data: &[u8], columns: u32, rows: u32) -> Result, CodecError> { + let generic = encode_jbig2_generic(data, columns, rows)?; + let samples = rows_to_encode(data, columns, rows).ok(); + Ok(match samples.and_then(|s| symbols::encode(s, columns, rows)) { + Some(coded) if coded.len() < generic.len() => coded, + _ => generic, + }) +} + +/// `rows` rows of `columns` 1-bit PDF samples (most significant bit first, each row starting on +/// a new byte), as a JBIG2 stream in the embedded organisation: a page information segment and an +/// immediate lossless generic region segment, which `JBIG2Decode` decodes to the same bits +/// without `/JBIG2Globals`. +/// +/// A 0 sample is coded as a JBIG2 1 (black), since `JBIG2Decode` gives black as 0, as a grey image's +/// samples have it. Bytes after the last row are ignored, as are the bits that pad a row to a whole +/// byte. +/// +/// Errors: no rows or columns, more of either than [`MAX_JBIG2_COLUMNS`], or more samples than +/// [`MAX_JBIG2_PIXELS`], what the decoder takes ([`CodecError::Unsupported`]); fewer bytes than the +/// rows need ([`CodecError::Corrupt`]). +pub fn encode_jbig2_generic(data: &[u8], columns: u32, rows: u32) -> Result, CodecError> { + let samples = rows_to_encode(data, columns, rows)?; + let width = columns as usize; + let stride = width.div_ceil(8); + let mut contexts: Vec = vec![0; 1 << 16]; + let mut mq = MqEncoder::new(); + let height = samples.len() / stride; + code_generic(&mut mq, &mut contexts, width, height, true, |y, row| { + let packed = samples.get(y * stride..).unwrap_or_default(); + for (x, pixel) in row.iter_mut().enumerate() { + let sample = packed.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0); + *pixel = u8::from(!sample); + } + }); + let region = mq.finish(); + let mut out = Vec::with_capacity(region.len() + 64); + segment(&mut out, 0, 48, &[], false, &page_information(columns, rows)); + segment(&mut out, 1, 39, &[], false, &generic_region(columns, rows, 0, 0, ®ion)); + Ok(out) +} + +/// The samples of `rows` rows of `columns` 1-bit samples in `data`, when the encoders here take +/// that size; the errors are [`encode_jbig2_generic`]'s. +fn rows_to_encode(data: &[u8], columns: u32, rows: u32) -> Result<&[u8], CodecError> { + let too_large = columns > MAX_JBIG2_COLUMNS || rows > MAX_JBIG2_COLUMNS || u64::from(columns) * u64::from(rows) > MAX_JBIG2_PIXELS; + if columns == 0 || rows == 0 || too_large { + return Err(CodecError::Unsupported(format!("JBIG2 encoding of {columns} × {rows} samples"))); + } + let stride = (columns as usize).div_ceil(8); + usize::try_from(rows) + .ok() + .and_then(|r| r.checked_mul(stride)) + .and_then(|n| data.get(..n)) + .ok_or_else(|| CodecError::Corrupt { filter: "JBIG2Decode", detail: format!("{} bytes for {rows} rows of {columns} samples", data.len()) }) +} + +/// A group of images coded by [`encode_jbig2_shared`]: the data of the `/JBIG2Globals` stream +/// they share, and each image's own stream, in the order given, or `None` for one that shares +/// nothing with the others. +#[derive(Debug)] +pub struct SharedJbig2 { + pub globals: Vec, + pub images: Vec>>, +} + +/// Images, each `rows` rows of `columns` 1-bit PDF samples, as lossless JBIG2 sharing one symbol +/// dictionary: the marks found in more than one of them are stored once, in a symbol dictionary +/// segment not associated with a page, which is the `/JBIG2Globals` stream's data, and each image's +/// stream, in the embedded organisation, places them from there, with its other marks coded as +/// [`encode_jbig2`]'s symbol coding codes them (see [`symbols`]). With the globals, each decodes to +/// its own bits. An image that holds none of those marks, or that symbol coding doesn't take, has +/// no stream here; it is stored alone. `None` when the images share fewer than two marks. +/// +/// What each image's stream saves is left for the caller to weigh against what the globals take, +/// which the images that use them share. +/// +/// Errors: as [`encode_jbig2_generic`], for the first image it would refuse. +pub fn encode_jbig2_shared(images: &[(&[u8], u32, u32)]) -> Result, CodecError> { + let images = + images.iter().map(|(data, columns, rows)| Ok((rows_to_encode(data, *columns, *rows)?, *columns, *rows))).collect::, _>>()?; + Ok(symbols::encode_shared(&images).map(|(globals, images)| SharedJbig2 { globals, images })) +} + +/// A page information segment's data (7.4.8): the page's size, resolution unknown, eventually +/// lossless, default pixel 0 (white), OR combination, not striped. +fn page_information(columns: u32, rows: u32) -> Vec { + let mut page = Vec::with_capacity(19); + page.extend_from_slice(&columns.to_be_bytes()); + page.extend_from_slice(&rows.to_be_bytes()); + page.extend_from_slice(&[0; 8]); + page.push(0x01); + page.extend_from_slice(&[0, 0]); + page +} + +/// A region segment information field (7.4.1): a `width` × `height` region at (`x`, `y`) on the +/// page, combined with it by OR. +fn region_information(out: &mut Vec, width: u32, height: u32, x: u32, y: u32) { + for v in [width, height, x, y] { + out.extend_from_slice(&v.to_be_bytes()); + } + out.push(0); +} + +/// An immediate lossless generic region segment's data (7.4.6): its region information, flags +/// (MMR 0, template 0, TPGDON 1), the AT pixels at their nominal places, then `coded`. +fn generic_region(width: u32, height: u32, x: u32, y: u32, coded: &[u8]) -> Vec { + let mut data = Vec::with_capacity(26 + coded.len()); + region_information(&mut data, width, height, x, y); + data.push(0x08); + at_pixels(&mut data); + data.extend_from_slice(coded); + data +} + +/// Template 0's four AT pixels at their nominal places, as AT flags (7.4.6.3, 7.4.2.1.2). +fn at_pixels(out: &mut Vec) { + for (x, y) in &TEMPLATE[12..] { + out.push(*x as i8 as u8); + out.push(*y as i8 as u8); + } +} + +/// A segment (7.2) of page 1: its header (number, type, the segments it refers to, the page, data +/// length), then its data. `retained`: a later segment refers to this one. `refers_to` holds each +/// referred-to segment's number and whether a segment after this one refers to it too (its +/// retain bit, 7.2.4). Segment numbers here stay below 256, so each referred-to one takes a byte +/// (7.2.5); a segment refers to at most four, so their count and retain bits take one byte too. +fn segment(out: &mut Vec, number: u32, kind: u8, refers_to: &[(u8, bool)], retained: bool, data: &[u8]) { + segment_of(out, 1, number, kind, refers_to, retained, data); +} + +/// [`segment`], of page `page`: 0 for a segment associated with no page (7.2.6), such as the +/// global ones `/JBIG2Globals` holds, which a page's segments may refer to. +fn segment_of(out: &mut Vec, page: u8, number: u32, kind: u8, refers_to: &[(u8, bool)], retained: bool, data: &[u8]) { + out.extend_from_slice(&number.to_be_bytes()); + out.push(kind); + // At most four, so the count fits its three bits. + let refers_to = refers_to.get(..refers_to.len().min(4)).unwrap_or_default(); + let kept = refers_to.iter().enumerate().fold(0u8, |bits, (k, (_, kept))| bits | u8::from(*kept) << (k + 1)); + out.push((refers_to.len() as u8) << 5 | kept | u8::from(retained)); + out.extend(refers_to.iter().map(|(n, _)| *n)); + out.push(page); + // A segment's data never comes near 4 GiB here: an image is at most 2^28 samples. + out.extend_from_slice(&u32::try_from(data.len()).unwrap_or(u32::MAX).to_be_bytes()); + out.extend_from_slice(data); +} + +/// Codes a `width` × `height` bitmap as a generic region (6.2.5.7) with template 0 and the nominal +/// AT pixels, its decisions in `contexts` (2^16 of them), with typical prediction when `typical` +/// (TPGDON). `fill(y, row)` sets row `y`'s `width` pixels, 1 for black. Pixels outside the bitmap +/// count as 0 (white), as the decoder takes them. +fn code_generic(mq: &mut MqEncoder, contexts: &mut [Context], width: usize, height: usize, typical: bool, mut fill: impl FnMut(usize, &mut [u8])) { + let sltp = SLTP.iter().enumerate().fold(0usize, |cx, (k, v)| cx | usize::from(*v) << k); + // The two rows above the current one and the current one, as JBIG2 pixels (1 black), padded. + let blank = vec![0u8; width + 2 * PAD]; + let (mut above2, mut above1) = (blank.clone(), blank.clone()); + let mut row = blank; + let mut ltp = false; + for y in 0..height { + if let Some(pixels) = row.get_mut(PAD..PAD + width) { + fill(y, pixels); + } + if typical { + // Typical prediction (6.2.5.5): a row the same as the one above is coded as one decision. + let same = row == above1; + if let Some(cx) = contexts.get_mut(sltp) { + mq.encode(cx, same != ltp); + } + ltp = same; + } + if !ltp { + for x in 0..width { + let at = |dx: i32, dy: i32| { + let line = match dy { + 0 => &row, + -1 => &above1, + _ => &above2, + }; + let i = (PAD + x).checked_add_signed(dx as isize); + i.and_then(|i| line.get(i)).copied().unwrap_or(0) + }; + let cx = TEMPLATE.iter().enumerate().fold(0usize, |cx, (k, (dx, dy))| cx | usize::from(at(*dx, *dy)) << k); + if let Some(state) = contexts.get_mut(cx) { + mq.encode(state, row.get(PAD + x) == Some(&1)); + } + } + } + std::mem::swap(&mut above2, &mut above1); + std::mem::swap(&mut above1, &mut row); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The rows an independent decoder (`hayro-jbig2`) reads from `data`, as PDF samples: true + /// for white (a JBIG2 0). + pub(super) fn decode(data: &[u8]) -> Vec> { + decode_with(data, None) + } + + /// [`decode`], with `/JBIG2Globals`. + pub(super) fn decode_with(data: &[u8], globals: Option<&[u8]>) -> Vec> { + struct Rows(Vec>, Vec, usize); + impl hayro_jbig2::Decoder for Rows { + fn push_pixel(&mut self, black: bool) { + self.1.push(!black); + } + fn push_pixel_chunk(&mut self, black: bool, chunks: u32) { + self.1.extend(std::iter::repeat_n(!black, chunks as usize * 8)); + } + fn next_line(&mut self) { + let mut row = std::mem::take(&mut self.1); + row.truncate(self.2); + self.0.push(row); + } + } + let image = hayro_jbig2::Image::new_embedded(data, globals).unwrap(); + let mut rows = Rows(Vec::new(), Vec::new(), image.width() as usize); + image.decode(&mut rows).unwrap(); + rows.0 + } + + /// The rows of packed samples `data`, true for a 1 bit. + pub(super) fn unpack(data: &[u8], columns: u32, rows: u32) -> Vec> { + let stride = (columns as usize).div_ceil(8); + (0..rows as usize).map(|y| (0..columns as usize).map(|x| data[y * stride + x / 8] & (0x80 >> (x % 8)) != 0).collect()).collect() + } + + /// Rows of `columns` samples from `runs`: alternating white and black runs one longer than + /// those values (never empty, even when shrinking makes them all 0), until the rows are full; + /// or, with `noise`, the bytes of `runs` as they are. + fn bitmap(columns: u32, rows: u32, runs: &[u8], noise: bool) -> Vec { + let stride = (columns as usize).div_ceil(8); + let mut data = vec![0u8; stride * rows as usize]; + if runs.is_empty() { + return data; + } + if noise { + data.iter_mut().zip(runs.iter().cycle()).for_each(|(d, r)| *d = *r); + return data; + } + let (mut white, mut left, mut next) = (true, 1 + usize::from(runs[0]), 1); + for y in 0..rows as usize { + for x in 0..columns as usize { + if left == 0 { + white = !white; + left = 1 + usize::from(runs[next % runs.len()]); + next += 1; + } + if white { + data[y * stride + x / 8] |= 0x80 >> (x % 8); + } + left -= 1; + } + } + data + } + + #[test] + fn a_page_of_text_like_marks_round_trips_and_shrinks() { + let (w, h) = (1700u32, 220u32); + let stride = (w as usize).div_ceil(8); + let mut data = vec![0xFF; stride * h as usize]; + for y in (20..h as usize).step_by(40) { + for dy in 0..12 { + for x in (100..1600).step_by(9) { + if (x / 9 + dy) % 5 != 0 { + data[(y + dy) * stride + x / 8] &= !(0x80 >> (x % 8)); + } + } + } + } + let jbig2 = encode_jbig2_generic(&data, w, h).unwrap(); + assert_eq!(decode(&jbig2), unpack(&data, w, h)); + let g4 = crate::encode_g4(&data, w, h).unwrap(); + assert!(jbig2.len() < g4.len(), "JBIG2 {} bytes, Group 4 {}", jbig2.len(), g4.len()); + } + + #[test] + fn edges_padding_and_typical_rows_round_trip() { + // 3 columns: the last 5 bits of each byte pad the row; set them to anything. + let data = [0b1010_1111, 0b0101_0000, 0b1110_1010, 0xAB, 0xCD]; + assert_eq!(decode(&encode_jbig2_generic(&data, 3, 3).unwrap()), [[true, false, true], [false, true, false], [true, true, true]]); + for (w, h, fill) in [(1, 1, 0x00), (1, 1, 0xFF), (13, 9, 0x00), (13, 9, 0xFF), (64, 40, 0x5A)] { + let data = vec![fill; (w as usize).div_ceil(8) * h as usize]; + assert_eq!(decode(&encode_jbig2_generic(&data, w, h).unwrap()), unpack(&data, w, h), "{w} × {h} of {fill:#04x}"); + } + } + + /// Packed samples from JBIG2 pixels (1 black): a 0 sample for each 1. + pub(super) fn samples(pixels: &[Vec]) -> Vec { + let width = pixels.first().map_or(0, Vec::len); + let stride = width.div_ceil(8); + let mut data = vec![0xFF; stride * pixels.len()]; + for (y, row) in pixels.iter().enumerate() { + for (x, p) in row.iter().enumerate() { + if *p == 1 { + data[y * stride + x / 8] &= !(0x80 >> (x % 8)); + } + } + } + data + } + + /// A row the same as the one above is typical (6.2.5.5): one decision, not a row of pixels. + #[test] + fn a_repeated_row_is_one_decision() { + let row: Vec = (0..800u32).map(|x| u8::from((x * 7919 + x / 3) % 5 < 2)).collect(); + let pixels = vec![row; 1000]; + let data = samples(&pixels); + crate::mq::DECISIONS.with(|n| n.set(0)); + let jbig2 = encode_jbig2_generic(&data, 800, 1000).unwrap(); + // The first row's 800 pixels, and one typical-row decision for each of the 1000 rows. + assert_eq!(crate::mq::DECISIONS.with(|n| n.get()), 800 + 1000); + assert_eq!(decode(&jbig2), unpack(&data, 800, 1000)); + } + + /// SLTP is coded in the context Figure 8 shows, which ordinary pixels can share: an image + /// tiled with that very neighbourhood decodes only if both coders use the same context. + #[test] + fn the_typical_row_flag_shares_figure_8s_context() { + // Figure 8, around a pixel at x = 4 of an 8-column tile: two rows up x = 2..6, one row up + // x = 1..7, the pixel's own row x = 0..3. + let tile = [[0, 0, 1, 0, 0, 1, 1, 0], [0, 0, 1, 1, 0, 0, 1, 0], [0, 1, 0, 1, 1, 0, 0, 0]]; + let pixels: Vec> = (0..90).map(|y| (0..400).map(|x| tile[y % 3][x % 8]).collect()).collect(); + let data = samples(&pixels); + assert_eq!(decode(&encode_jbig2_generic(&data, 400, 90).unwrap()), unpack(&data, 400, 90)); + } + + #[test] + fn what_it_cannot_encode_is_an_error() { + assert!(matches!(encode_jbig2_generic(&[0; 8], 0, 1), Err(CodecError::Unsupported(_)))); + assert!(matches!(encode_jbig2_generic(&[0; 8], 8, 0), Err(CodecError::Unsupported(_)))); + assert!(matches!(encode_jbig2_generic(&[0; 3], 16, 2), Err(CodecError::Corrupt { .. })), "two rows of 16 need 4 bytes"); + assert!(matches!(encode_jbig2_generic(&[], 1, 2), Err(CodecError::Corrupt { .. }))); + } + + /// The encoder stops where the decoder does, on each side and in total: past that it would + /// write a stream nothing here can read. 16,384 squared is exactly the bound on samples. + /// `encode_jbig2_shared` refuses what `encode_jbig2_generic` refuses: an image taller than + /// the decoder takes is `Unsupported`, one at the bound isn't refused. (Its one tall mark is + /// too large to be a symbol, so nothing is shared: `Ok(None)`.) + #[test] + fn shared_coding_takes_only_what_the_decoder_takes() { + let past = vec![0x7F; MAX_JBIG2_COLUMNS as usize + 1]; + let tall = (&past[..], 1, MAX_JBIG2_COLUMNS + 1); + assert!(matches!(encode_jbig2_shared(&[tall, tall]), Err(CodecError::Unsupported(_))), "too tall"); + let at = (&past[..MAX_JBIG2_COLUMNS as usize], 1, MAX_JBIG2_COLUMNS); + assert!(matches!(encode_jbig2_shared(&[at, at]), Ok(None)), "at the bound"); + } + + #[test] + fn it_encodes_only_what_the_decoder_takes() { + let unsupported = |columns, rows| matches!(encode_jbig2_generic(&[], columns, rows), Err(CodecError::Unsupported(_))); + assert!(unsupported(MAX_JBIG2_COLUMNS + 1, 1), "too wide"); + assert!(unsupported(1, MAX_JBIG2_COLUMNS + 1), "too tall"); + assert!(unsupported(16_385, 16_384), "too many samples, each side within the bound"); + assert!(unsupported(u32::MAX, u32::MAX), "the product doesn't overflow"); + // At the bounds what it writes reads back, through the decoder that has to read it. + let wide = vec![0x0F; (MAX_JBIG2_COLUMNS as usize).div_ceil(8) * 4]; + assert_eq!(decode(&encode_jbig2_generic(&wide, MAX_JBIG2_COLUMNS, 4).unwrap()), unpack(&wide, MAX_JBIG2_COLUMNS, 4)); + let tall = vec![0x7F; MAX_JBIG2_COLUMNS as usize]; + assert_eq!(decode(&encode_jbig2_generic(&tall, 1, MAX_JBIG2_COLUMNS).unwrap()), unpack(&tall, 1, MAX_JBIG2_COLUMNS)); + struct Rows(u32, u64); + impl hayro_jbig2::Decoder for Rows { + fn push_pixel(&mut self, black: bool) { + self.1 += u64::from(black); + } + fn push_pixel_chunk(&mut self, black: bool, chunks: u32) { + self.1 += u64::from(black) * u64::from(chunks) * 8; + } + fn next_line(&mut self) { + self.0 += 1; + } + } + // One black pixel, so a page that came back blank wouldn't pass. + let mut largest = vec![0xFF; 2_048 * 16_384]; + largest[0] = 0x7F; + let largest = encode_jbig2_generic(&largest, 16_384, 16_384).unwrap(); + let mut rows = Rows(0, 0); + hayro_jbig2::Image::new_embedded(&largest, None).unwrap().decode(&mut rows).unwrap(); + assert_eq!(rows.0, 16_384); + assert_eq!(rows.1, 1); + } + + proptest::proptest! { + /// Any bitmap, of noise or of runs, decodes to its own bits through an independent decoder. + #[test] + fn bitmaps_round_trip( + columns in 1u32..300, + rows in 1u32..24, + runs in proptest::collection::vec(0u8..=255, 1..48), + noise in proptest::bool::ANY, + ) { + let data = bitmap(columns, rows, &runs, noise); + let jbig2 = encode_jbig2_generic(&data, columns, rows).unwrap(); + proptest::prop_assert_eq!(decode(&jbig2), unpack(&data, columns, rows)); + } + + /// Whatever it is given, it returns. + #[test] + fn any_input_returns(data in proptest::collection::vec(0u8..=255, 0..600), columns in 0u32..5000, rows in 0u32..80) { + let _ = encode_jbig2_generic(&data, columns, rows); + } + } +} diff --git a/crates/stream-codecs/src/jbig2/symbols.rs b/crates/stream-codecs/src/jbig2/symbols.rs new file mode 100644 index 000000000..f56503a84 --- /dev/null +++ b/crates/stream-codecs/src/jbig2/symbols.rs @@ -0,0 +1,1329 @@ +//! JBIG2 symbol coding, lossless: each distinct mark of an image (a group of black pixels that +//! touch, side by side or at a corner) is stored once, as a symbol in a symbol dictionary, and +//! text regions place a copy of it wherever it occurs. Marks too large to be symbols (rules, +//! frames, pictures) are left to a generic region beside them, and so, in [`encode`]'s second +//! plan, are marks that occur only once. Only exact copies share a symbol, so the image decodes to the +//! same bits. Written from the committee draft's 6.4 (the text region decoding procedure), 6.5 +//! (the symbol dictionary decoding procedure), 7.4.2 and 7.4.3 (their segments) and Annex A (the +//! arithmetic integer coders). + +use std::collections::HashMap; +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::ops::Range; + +use super::{at_pixels, code_generic, generic_region, page_information, region_information, segment, segment_of}; +use crate::mq::{Context, MqEncoder}; + +/// The longest side of a mark stored as a symbol. A larger one is left to the generic region. +const MAX_SYMBOL_SIDE: u32 = 256; + +/// The most black runs looked at. Past it (halftones, noise) the image is left to generic coding, +/// which suits it better anyway; this bounds the memory the marks take. +const MAX_RUNS: usize = 1 << 20; + +/// The most distinct marks sorted into symbols, and the most pixels their bitmaps may hold +/// together: past either the image is left to generic coding. These bound the memory they take. +const MAX_SYMBOLS: usize = 1 << 16; +const MAX_SYMBOL_PIXELS: usize = 1 << 24; + +/// The most symbols one plan codes. `hayro-jbig2`, the decoder the Craft apps render JBIG2 with, +/// refuses a dictionary whose new and exported symbols pass 65,535 together, and a plan exports +/// every symbol it codes, so it refuses 32,768 or more; others may too, so a plan with more isn't +/// made. +const MAX_CODED_SYMBOLS: usize = (1 << 15) - 1; + +/// The most symbol instances one text region places. `hayro-jbig2`, the decoder the Craft apps render JBIG2 with, +/// refuses a text region of more than 10,000 (a page it then draws blank), so a page with more is +/// placed by several regions, each over a band of rows, all referring to the one dictionary. +const MAX_REGION_INSTANCES: usize = 10_000; + +/// The most symbols a group's shared dictionary holds ([`encode_shared`]): half of what a page may +/// place, so each page keeps room for its own. +const MAX_SHARED_SYMBOLS: usize = MAX_CODED_SYMBOLS / 2; + +// Segment numbers stay at most 256, so a referred-to one takes a byte (7.2.5): the shared +// dictionary, the page, its own dictionary, a text region for each MAX_REGION_INSTANCES of at most +// MAX_RUNS marks, and the generic region. +const _: () = assert!(MAX_RUNS.div_ceil(MAX_REGION_INSTANCES) + 4 <= 256); + +/// The integer coders' ranges (Table A.1): how many 1 bits pick the range, how many bits give the +/// offset into it, and its first value. The last range has no 0 after its prefix. +const RANGES: [(u32, u32, u32); 6] = [(0, 2, 0), (1, 4, 4), (2, 6, 20), (3, 8, 84), (4, 12, 340), (5, 32, 4436)]; + +/// A run of black pixels on row `y`, from `x0` up to but not including `x1`. +#[derive(Clone, Copy, Debug)] +struct Run { + y: u32, + x0: u32, + x1: u32, +} + +/// A mark: its box (`x1` and `y1` just past it) and its runs, as a range of the run order +/// [`marks`] gives. +#[derive(Debug)] +struct Mark { + x0: u32, + x1: u32, + y0: u32, + y1: u32, + runs: Range, +} + +/// A distinct mark's bitmap, a byte a pixel (1 black), and how many marks are exact copies of it. +#[derive(Debug)] +struct Symbol { + width: u32, + height: u32, + pixels: Vec, + uses: u32, +} + +/// An image's marks, sorted out: its black runs, its marks with the run order they index, the +/// distinct marks of at most [`MAX_SYMBOL_SIDE`] a side as symbols, each such mark as (its +/// symbol, the mark), and the marks too large to be symbols. +struct Page { + runs: Vec, + marks: Vec, + order: Vec, + symbols: Vec, + copies: Vec<(u32, u32)>, + large: Vec, +} + +/// A symbol instance (6.4.5): the symbol, and where its bottom-left pixel goes on the page, as S +/// (the column) and T (the row). `width` and `height` are the symbol's. +#[derive(Clone, Copy, Debug)] +struct Instance { + id: u32, + s: u32, + t: u32, + width: u32, + height: u32, +} + +/// A group of images coded by [`encode_shared`]: the globals' data, and each image's stream. +type SharedCoding = (Vec, Vec>>); + +/// The symbols of a group's shared dictionary, as one image's [`plan`] places them: the ID each of +/// the image's symbols has there, if it is there, and how many symbols that dictionary exports. +struct Shared { + ids: Vec>, + count: u32, +} + +/// `rows` rows of `columns` 1-bit PDF samples (black 0), symbol coded: the smaller of two plans, +/// every mark of at most [`MAX_SYMBOL_SIDE`] a side as a symbol, or only those with a copy. +/// `None` when neither plan has from two to [`MAX_CODED_SYMBOLS`] symbols, or the image is too +/// large or too busy to look at. +pub(super) fn encode(samples: &[u8], columns: u32, rows: u32) -> Option> { + let page = page(samples, columns)?; + [1, 2].into_iter().filter_map(|copies| plan(&page, columns, rows, copies, None, None)).min_by_key(Vec::len) +} + +/// Images, each as [`encode`] takes one, symbol coded sharing one dictionary: the globals' data, a +/// symbol dictionary segment associated with no page (number 0) holding the distinct marks found +/// in more than one image, and each image's stream placing them from there, the smaller of +/// [`encode`]'s two plans for its other marks. The group's marks are held within the limits one +/// image's are ([`MAX_SYMBOLS`], [`MAX_SYMBOL_PIXELS`]): past them, later marks aren't shared. Past +/// [`MAX_SHARED_SYMBOLS`] shared marks, those in the most images are kept. +/// +/// An image too large or too busy to look at, or that places none of the shared marks, gets `None`; +/// `None` overall when fewer than two marks are shared. +pub(super) fn encode_shared(images: &[(&[u8], u32, u32)]) -> Option { + // The group's distinct marks, each with how many images hold it (as its uses), looked at an + // image at a time. + let mut table: Vec = Vec::new(); + let mut by_hash: HashMap> = HashMap::new(); + let mut held = 0usize; + for (samples, columns, _) in images { + let Some(page) = page(samples, *columns) else { continue }; + // An image's symbols are distinct, so each counts an image once. + for symbol in page.symbols { + match find(&table, &by_hash, &symbol) { + Some(t) => { + if let Some(s) = table.get_mut(t as usize) { + s.uses = s.uses.saturating_add(1); + } + } + None => { + let Some(more) = held.checked_add(symbol.pixels.len()).filter(|h| *h <= MAX_SYMBOL_PIXELS && table.len() < MAX_SYMBOLS) else { + continue; + }; + held = more; + let t = u32::try_from(table.len()).ok()?; + by_hash.entry(fingerprint(&symbol)).or_default().push(t); + table.push(Symbol { uses: 1, ..symbol }); + } + } + } + } + let uses = |t: &u32| table.get(*t as usize).map_or(0, |s| s.uses); + let mut shared: Vec = (0..u32::try_from(table.len()).ok()?).filter(|t| uses(t) >= 2).collect(); + if shared.len() < 2 { + return None; + } + shared.sort_by_key(|t| std::cmp::Reverse(uses(t))); + shared.truncate(MAX_SHARED_SYMBOLS); + // The dictionary's order, as `plan` keeps it. + shared.sort_by_key(|t| table.get(*t as usize).map(|s| (s.height, s.width))); + let mut global = vec![None; table.len()]; + for (k, t) in shared.iter().enumerate() { + if let Some(slot) = global.get_mut(*t as usize) { + *slot = u32::try_from(k).ok(); + } + } + let symbols: Vec<&Symbol> = shared.iter().filter_map(|t| table.get(*t as usize)).collect(); + let mut globals = Vec::new(); + segment_of(&mut globals, 0, 0, 0, &[], true, &symbol_dictionary(&symbols)?); + let count = u32::try_from(symbols.len()).ok()?; + let streams = images + .iter() + .map(|(samples, columns, rows)| { + let page = page(samples, *columns)?; + let ids = page.symbols.iter().map(|s| find(&table, &by_hash, s).and_then(|t| global.get(t as usize).copied().flatten())).collect(); + let shared = Shared { ids, count }; + [1, 2].into_iter().filter_map(|copies| plan(&page, *columns, *rows, copies, None, Some(&shared))).min_by_key(Vec::len) + }) + .collect(); + Some((globals, streams)) +} + +/// What sorts a symbol's bitmap ([`symbols`] and [`encode_shared`] look exact copies up by it). +fn fingerprint(s: &Symbol) -> u64 { + let mut hasher = DefaultHasher::new(); + (s.width, s.height, &s.pixels).hash(&mut hasher); + hasher.finish() +} + +/// The index in `table` of the symbol with `s`'s bitmap, looked up by its [`fingerprint`]. +fn find(table: &[Symbol], by_hash: &HashMap>, s: &Symbol) -> Option { + by_hash + .get(&fingerprint(s))? + .iter() + .copied() + .find(|t| table.get(*t as usize).is_some_and(|t| (t.width, t.height) == (s.width, s.height) && t.pixels == s.pixels)) +} + +/// The image's marks and symbols; `None` past the limits above. +fn page(samples: &[u8], columns: u32) -> Option { + let width = usize::try_from(columns).ok()?; + let stride = width.div_ceil(8); + if stride == 0 { + return None; + } + let runs = black_runs(samples, width, stride)?; + let (marks, order) = marks(&runs); + let mut page = Page { runs, marks, order, symbols: Vec::new(), copies: Vec::new(), large: Vec::new() }; + symbols(&mut page)?; + Some(page) +} + +/// The black runs of each row (0 samples), row by row and left to right; `None` past [`MAX_RUNS`]. +fn black_runs(samples: &[u8], width: usize, stride: usize) -> Option> { + let black = |row: &[u8], x: usize| row.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) == 0); + let mut runs = Vec::new(); + for (y, row) in samples.chunks_exact(stride).enumerate() { + let y = u32::try_from(y).ok()?; + let mut x = 0; + while x < width { + // Whole white bytes at once. + if x % 8 == 0 && row.get(x / 8) == Some(&0xFF) { + x += 8; + continue; + } + if !black(row, x) { + x += 1; + continue; + } + let start = x; + while x < width && black(row, x) { + x += 1; + } + if runs.len() == MAX_RUNS { + return None; + } + runs.push(Run { y, x0: u32::try_from(start).ok()?, x1: u32::try_from(x).ok()? }); + } + } + Some(runs) +} + +/// The marks the runs make, in the order their first runs come (so from the top), and the run +/// order: the runs' indexes, mark by mark, each mark's in row order. +fn marks(runs: &[Run]) -> (Vec, Vec) { + // Union–find over the runs; a set's root is its first run, as the larger root always joins + // the smaller. Runs on adjacent rows join when they overlap, counting a corner. + let mut parent: Vec = (0..runs.len()).map(|i| i as u32).collect(); + let (mut above, mut above_at, mut above_y): (&[Run], usize, Option) = (&[], 0, None); + let mut at = 0; + for row in runs.chunk_by(|a, b| a.y == b.y) { + let y = row.first().map(|r| r.y); + if above_y.and_then(|a| a.checked_add(1)) == y { + let mut i = 0; + for (j, b) in row.iter().enumerate() { + while above.get(i).is_some_and(|a| a.x1 < b.x0) { + i += 1; + } + let mut k = i; + while above.get(k).is_some_and(|a| a.x0 <= b.x1) { + union(&mut parent, above_at + k, at + j); + k += 1; + } + } + } + (above, above_at, above_y) = (row, at, y); + at += row.len(); + } + let mut mark_of = vec![0u32; runs.len()]; + let mut marks: Vec = Vec::new(); + let mut counts: Vec = Vec::new(); + for (i, r) in runs.iter().enumerate() { + let root = find_root(&mut parent, i); + let m = if root == i { + marks.push(Mark { x0: r.x0, x1: r.x1, y0: r.y, y1: r.y.saturating_add(1), runs: 0..0 }); + counts.push(0); + marks.len() - 1 + } else { + mark_of.get(root).map_or(0, |m| *m as usize) + }; + if let Some(slot) = mark_of.get_mut(i) { + *slot = m as u32; + } + if let (Some(mark), Some(count)) = (marks.get_mut(m), counts.get_mut(m)) { + mark.x0 = mark.x0.min(r.x0); + mark.x1 = mark.x1.max(r.x1); + mark.y1 = mark.y1.max(r.y.saturating_add(1)); + *count += 1; + } + } + let mut next = 0; + for (mark, count) in marks.iter_mut().zip(&counts) { + mark.runs = next..next + count; + next += count; + } + let mut order = vec![0u32; runs.len()]; + let mut fill: Vec = marks.iter().map(|m| m.runs.start).collect(); + for (i, m) in mark_of.iter().enumerate() { + if let Some(n) = fill.get_mut(*m as usize) { + if let Some(slot) = order.get_mut(*n) { + *slot = i as u32; + } + *n += 1; + } + } + (marks, order) +} + +/// The root of `i`'s set, halving the path to it as it goes. Parents are never above their +/// children, so the walk ends. +fn find_root(parent: &mut [u32], mut i: usize) -> usize { + loop { + let Some(&p) = parent.get(i) else { return i }; + let p = p as usize; + if p == i { + return i; + } + let grandparent = parent.get(p).copied().unwrap_or(p as u32); + if let Some(slot) = parent.get_mut(i) { + *slot = grandparent; + } + i = grandparent as usize; + } +} + +fn union(parent: &mut [u32], a: usize, b: usize) { + let (a, b) = (find_root(parent, a), find_root(parent, b)); + if let Some(slot) = parent.get_mut(a.max(b)) { + *slot = a.min(b) as u32; + } +} + +/// Each mark's own pixels in its box: those of its runs, not of other marks the box overlaps. +fn pixels(mark: &Mark, runs: &[Run], order: &[u32]) -> Vec { + // A mark's box holds its runs, so these don't go below 0; a symbol is at most 256 a side. + let width = mark.x1.saturating_sub(mark.x0) as usize; + let height = mark.y1.saturating_sub(mark.y0) as usize; + let mut pixels = vec![0u8; width * height]; + for r in order.get(mark.runs.clone()).unwrap_or_default().iter().filter_map(|i| runs.get(*i as usize)) { + let row = r.y.saturating_sub(mark.y0) as usize * width; + let from = row + r.x0.saturating_sub(mark.x0) as usize; + let to = row + r.x1.saturating_sub(mark.x0) as usize; + pixels.get_mut(from..to).unwrap_or_default().fill(1); + } + pixels +} + +/// Sorts the marks into symbols: exact copies share one. `None` past [`MAX_SYMBOLS`] or +/// [`MAX_SYMBOL_PIXELS`]. +fn symbols(page: &mut Page) -> Option<()> { + let mut by_hash: HashMap> = HashMap::new(); + let mut kept = 0usize; + for (m, mark) in page.marks.iter().enumerate() { + let m = u32::try_from(m).ok()?; + let (width, height) = (mark.x1.saturating_sub(mark.x0), mark.y1.saturating_sub(mark.y0)); + if width > MAX_SYMBOL_SIDE || height > MAX_SYMBOL_SIDE { + page.large.push(m); + continue; + } + let symbol = Symbol { width, height, pixels: pixels(mark, &page.runs, &page.order), uses: 0 }; + let symbols = &mut page.symbols; + let id = match find(symbols, &by_hash, &symbol) { + Some(id) => id, + None => { + kept = kept.checked_add(symbol.pixels.len())?; + if kept > MAX_SYMBOL_PIXELS || symbols.len() == MAX_SYMBOLS { + return None; + } + let id = u32::try_from(symbols.len()).ok()?; + by_hash.entry(fingerprint(&symbol)).or_default().push(id); + symbols.push(symbol); + id + } + }; + if let Some(s) = symbols.get_mut(id as usize) { + s.uses = s.uses.saturating_add(1); + } + page.copies.push((id, m)); + } + Some(()) +} + +/// One plan's stream: the symbols with at least `copies` uses in a symbol dictionary, text +/// regions placing them (in strips of 2^`log_strips` rows, or whichever size codes smallest), and +/// the other marks in a generic region. `None` with fewer than two such symbols (their IDs would +/// take no bits at all, SBSYMCODELEN 0, a corner of the standard left alone here) or more than +/// [`MAX_CODED_SYMBOLS`]. +/// +/// With `shared`, the symbols a group's dictionary holds are placed from there, whatever their uses +/// here, and the page's own dictionary holds only the others (none, when it has none), as +/// [`encode_shared`] writes them; `None` too for a page that places none of the shared ones. +fn plan(page: &Page, columns: u32, rows: u32, copies: u32, log_strips: Option, shared: Option<&Shared>) -> Option> { + let symbol = |s: u32| page.symbols.get(s as usize); + let global = |s: u32| shared.and_then(|g| g.ids.get(s as usize).copied().flatten()); + let globals = shared.map_or(0, |g| g.count); + // The dictionary's order (6.5.5): height classes from the shortest, each narrowest first. + let mut kept: Vec = + (0..page.symbols.len() as u32).filter(|s| global(*s).is_none() && symbol(*s).is_some_and(|s| s.uses >= copies)).collect(); + let count = usize::try_from(globals).ok()?.checked_add(kept.len())?; + if !(2..=MAX_CODED_SYMBOLS).contains(&count) { + return None; + } + kept.sort_by_key(|s| symbol(*s).map(|s| (s.height, s.width))); + // The shared symbols' IDs come first, as their dictionary is referred to first (Table 31's + // SBSYMS). + let mut id: Vec> = (0..page.symbols.len() as u32).map(global).collect(); + for (k, s) in kept.iter().enumerate() { + if let Some(slot) = id.get_mut(*s as usize) { + *slot = u32::try_from(k).ok().and_then(|k| k.checked_add(globals)); + } + } + let mut instances = Vec::new(); + let mut left = page.large.clone(); + for (s, m) in &page.copies { + match (id.get(*s as usize).copied().flatten(), page.marks.get(*m as usize), symbol(*s)) { + (Some(id), Some(mark), Some(symbol)) => { + instances.push(Instance { id, s: mark.x0, t: mark.y1.saturating_sub(1), width: symbol.width, height: symbol.height }) + } + _ => left.push(*m), + } + } + if shared.is_some() && !instances.iter().any(|i| i.id < globals) { + return None; + } + let kept: Vec<&Symbol> = kept.iter().filter_map(|s| symbol(*s)).collect(); + let dictionary = if kept.is_empty() { None } else { Some(symbol_dictionary(&kept)?) }; + let count = u32::try_from(count).ok()?; + // Bands of at most MAX_REGION_INSTANCES, from the top. + instances.sort_by_key(|i| (i.t, i.s)); + let mut regions = Vec::new(); + for band in instances.chunks_mut(MAX_REGION_INSTANCES) { + let top = band.iter().map(|i| i.t.saturating_add(1).saturating_sub(i.height)).min()?; + let height = band.iter().map(|i| i.t).max()?.checked_sub(top)?.checked_add(1)?; + regions.push(match log_strips { + Some(l) => text_region(band, count, columns, top, height, l)?, + None => (0..4).filter_map(|l| text_region(band, count, columns, top, height, l)).min_by_key(Vec::len)?, + }); + } + let mut out = Vec::new(); + // The shared dictionary is segment 0, in the globals, and a segment refers only to lower + // numbers (7.2.5): the page's own follow it. + let mut number = u32::from(shared.is_some()); + segment(&mut out, number, 48, &[], false, &page_information(columns, rows)); + number += 1; + let mut dictionaries: Vec = shared.map(|_| 0).into_iter().collect(); + if let Some(dictionary) = &dictionary { + segment(&mut out, number, 0, &[], true, dictionary); + dictionaries.push(u8::try_from(number).ok()?); + number += 1; + } + for (k, text) in regions.iter().enumerate() { + // The dictionaries stay retained for the next region, until the last (7.2.4). + let refers_to: Vec<(u8, bool)> = dictionaries.iter().map(|n| (*n, k + 1 < regions.len())).collect(); + segment(&mut out, number, 7, &refers_to, false, text); + number += 1; + } + if !left.is_empty() { + segment(&mut out, number, 39, &[], false, &leftover_region(&left, page)?); + } + Some(out) +} + +/// A symbol dictionary segment's data (7.4.2) defining `symbols` (in dictionary order) and +/// exporting them all: arithmetic coded, no refinement or aggregation, each bitmap a generic +/// region with template 0, its nominal AT pixels and no typical prediction (Table 16), the +/// bitmaps sharing one set of contexts. +fn symbol_dictionary(symbols: &[&Symbol]) -> Option> { + let count = u32::try_from(symbols.len()).ok()?; + let mut data = vec![0, 0]; + at_pixels(&mut data); + data.extend_from_slice(&count.to_be_bytes()); + data.extend_from_slice(&count.to_be_bytes()); + let mut mq = MqEncoder::new(); + let mut contexts: Vec = vec![0; 1 << 16]; + let (mut iadh, mut iadw, mut iaex) = (IntCoder::new(), IntCoder::new(), IntCoder::new()); + let mut height = 0; + for class in symbols.chunk_by(|a, b| a.height == b.height) { + let h = class.first()?.height; + iadh.encode(&mut mq, Some(int(i64::from(h) - i64::from(height))?)); + height = h; + let mut width = 0; + for s in class { + iadw.encode(&mut mq, Some(int(i64::from(s.width) - i64::from(width))?)); + width = s.width; + let w = s.width as usize; + code_generic(&mut mq, &mut contexts, w, s.height as usize, false, |y, row| { + let line = s.pixels.get(y * w..).unwrap_or_default(); + row.iter_mut().zip(line).for_each(|(p, v)| *p = *v); + }); + } + iadw.encode(&mut mq, None); + } + // The export flags (6.5.10): a run of no symbols not exported, then a run of all of them. + iaex.encode(&mut mq, Some(0)); + iaex.encode(&mut mq, Some(int(i64::from(count))?)); + data.extend_from_slice(&mq.finish()); + Some(data) +} + +/// An immediate lossless text region segment's data (7.4.3) placing `instances` in the band of +/// `height` rows from row `top` (which holds them): arithmetic coded, no refinement, strips of +/// 2^`log_strips` rows, each symbol's bottom-left pixel at (S, T) (REFCORNER BOTTOMLEFT, so a +/// line's letters share their T), not transposed, OR, default pixel 0, SBDSOFFSET 0. Sorts +/// `instances` into strips, each left to right. +fn text_region(instances: &mut [Instance], symbols: u32, columns: u32, top: u32, height: u32, log_strips: u32) -> Option> { + // LOGSBSTRIPS has two bits: strips of 1, 2, 4 or 8 rows. + if log_strips > 3 { + return None; + } + let strips = 1i64 << log_strips; + // T counts from the band's top row. + let t = |i: &Instance| i.t.saturating_sub(top); + instances.sort_by_key(|i| (t(i) >> log_strips, i.s)); + let mut data = Vec::new(); + region_information(&mut data, columns, height, 0, top); + data.extend_from_slice(&((log_strips as u16) << 2).to_be_bytes()); + data.extend_from_slice(&u32::try_from(instances.len()).ok()?.to_be_bytes()); + // SBSYMCODELEN (Table 31): ⌈log2 SBNUMSYMS⌉. + let length = symbols.checked_next_power_of_two()?.trailing_zeros(); + let mut ids: Vec = vec![0; 1 << length]; + let (mut iadt, mut iafs, mut iads, mut iait) = (IntCoder::new(), IntCoder::new(), IntCoder::new(), IntCoder::new()); + let mut mq = MqEncoder::new(); + // The initial STRIPT (6.4.5 step 2) is 0, so each strip's delta T is its first row over the + // strip size, from the strip before. + iadt.encode(&mut mq, Some(0)); + let (mut strip_t, mut first_s) = (0i64, 0i64); + for strip in instances.chunk_by(|a, b| t(a) >> log_strips == t(b) >> log_strips) { + let first = i64::from(t(strip.first()?) >> log_strips << log_strips); + iadt.encode(&mut mq, Some(int((first - strip_t) / strips)?)); + strip_t = first; + let mut cur_s = 0; + for (k, instance) in strip.iter().enumerate() { + let s = i64::from(instance.s); + if k == 0 { + iafs.encode(&mut mq, Some(int(s - first_s)?)); + first_s = s; + } else { + iads.encode(&mut mq, Some(int(s - cur_s)?)); + } + if strips > 1 { + iait.encode(&mut mq, Some(int(i64::from(t(instance)) - strip_t)?)); + } + encode_id(&mut mq, &mut ids, length, instance.id); + // The next S counts from this symbol's last column (6.4.5 step 3c x). + cur_s = s + i64::from(instance.width) - 1; + } + iads.encode(&mut mq, None); + } + data.extend_from_slice(&mq.finish()); + Some(data) +} + +/// A generic region segment's data holding the marks `left`, over their box. +fn leftover_region(left: &[u32], page: &Page) -> Option> { + let left: Vec<&Mark> = left.iter().filter_map(|m| page.marks.get(*m as usize)).collect(); + let x0 = left.iter().map(|m| m.x0).min()?; + let y0 = left.iter().map(|m| m.y0).min()?; + let x1 = left.iter().map(|m| m.x1).max()?; + let y1 = left.iter().map(|m| m.y1).max()?; + let (width, height) = (x1.saturating_sub(x0) as usize, y1.saturating_sub(y0) as usize); + let stride = width.div_ceil(8); + // Packed, 1 for black: no larger than the image's own samples. + let mut packed = vec![0u8; stride.checked_mul(height)?]; + for mark in &left { + for r in page.order.get(mark.runs.clone()).unwrap_or_default().iter().filter_map(|i| page.runs.get(*i as usize)) { + let row = r.y.saturating_sub(y0) as usize * stride; + for x in r.x0.saturating_sub(x0) as usize..r.x1.saturating_sub(x0) as usize { + if let Some(b) = packed.get_mut(row + x / 8) { + *b |= 0x80 >> (x % 8); + } + } + } + } + let mut mq = MqEncoder::new(); + let mut contexts: Vec = vec![0; 1 << 16]; + code_generic(&mut mq, &mut contexts, width, height, true, |y, row| { + let line = packed.get(y * stride..).unwrap_or_default(); + for (x, p) in row.iter_mut().enumerate() { + *p = u8::from(line.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0)); + } + }); + Some(generic_region(u32::try_from(width).ok()?, u32::try_from(height).ok()?, x0, y0, &mq.finish())) +} + +fn int(v: i64) -> Option { + i32::try_from(v).ok() +} + +/// An arithmetic integer coder other than IAID (A.2): its 512 contexts, indexed by PREV. +struct IntCoder([Context; 512]); + +impl IntCoder { + fn new() -> Self { + IntCoder([0; 512]) + } + + /// Codes `value`, or OOB for `None`. + fn encode(&mut self, mq: &mut MqEncoder, value: Option) { + for (cx, d) in decisions(value) { + if let Some(state) = self.0.get_mut(cx) { + mq.encode(state, d); + } + } + } +} + +/// The decisions that code `value`, or OOB for `None`, with their contexts (A.2, Table A.1): the +/// sign, the prefix that picks the range, then the offset into it, most significant bit first; +/// OOB is a negative 0. Each decision's context is PREV: a leading 1 and the decisions before it, +/// at most the last eight once there are more. +fn decisions(value: Option) -> Vec<(usize, bool)> { + let (negative, magnitude) = value.map_or((true, 0), |v| (v < 0, v.unsigned_abs())); + let (ones, bits, first) = + RANGES.into_iter().find(|(_, bits, first)| *bits == 32 || u64::from(magnitude) < u64::from(*first) + (1 << bits)).unwrap_or(RANGES[5]); + let offset = magnitude - first; + let mut d = vec![negative]; + d.extend(std::iter::repeat_n(true, ones as usize)); + if ones < 5 { + d.push(false); + } + d.extend((0..bits).rev().map(|i| offset >> i & 1 == 1)); + let mut prev = 1usize; + d.into_iter() + .map(|bit| { + let cx = prev; + prev = if prev < 256 { prev << 1 | usize::from(bit) } else { ((prev << 1 | usize::from(bit)) & 511) | 256 }; + (cx, bit) + }) + .collect() +} + +/// Codes symbol ID `id` in `length` bits, most significant first, through IAID (A.3): each +/// decision's context is a leading 1 and all the bits before it. +fn encode_id(mq: &mut MqEncoder, contexts: &mut [Context], length: u32, id: u32) { + let mut prev = 1usize; + for i in (0..length).rev() { + let d = id >> i & 1 == 1; + if let Some(state) = contexts.get_mut(prev) { + mq.encode(state, d); + } + prev = prev << 1 | usize::from(d); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::CodecError; + use crate::jbig2::tests::{decode, decode_with, samples, unpack}; + use crate::jbig2::{encode_jbig2, encode_jbig2_generic, encode_jbig2_shared}; + + type Bitmap = Vec>; + + /// A small linear congruential generator, so the pictures are the same every run. + struct Lcg(u64); + + impl Lcg { + fn below(&mut self, n: usize) -> usize { + self.0 = self.0.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); + (self.0 >> 33) as usize % n.max(1) + } + } + + /// `count` different letter-like shapes: three strokes two pixels thick in a box of 6 to 18 + /// pixels a side, each after the first starting on one before it, so a shape is one mark; + /// trimmed to its pixels. + fn glyphs(count: usize, seed: u64) -> Vec { + let mut r = Lcg(seed); + let mut out: Vec = Vec::new(); + while out.len() < count { + let (w, h) = (6 + r.below(13), 6 + r.below(13)); + let mut g = vec![vec![0u8; w]; h]; + let (mut x, mut y) = (r.below(w), r.below(h)); + for _ in 0..3 { + let (dx, dy) = [(1, 0), (0, 1), (1, 1)][r.below(3)]; + let mut stroke = Vec::new(); + while x < w && y < h { + g[y][x] = 1; + g[y][(x + 1).min(w - 1)] = 1; + stroke.push((x, y)); + x += dx; + y += dy; + } + (x, y) = stroke[r.below(stroke.len())]; + } + let rows: Vec = (0..h).filter(|y| g[*y].contains(&1)).collect(); + let columns: Vec = (0..w).filter(|x| g.iter().any(|row| row[*x] == 1)).collect(); + let g: Bitmap = g[rows[0]..=rows[rows.len() - 1]].iter().map(|row| row[columns[0]..=columns[columns.len() - 1]].to_vec()).collect(); + if !out.contains(&g) { + out.push(g); + } + } + out + } + + /// Draws `glyph` with its top-left pixel at (`x`, `y`), clipped to the page, by OR. + fn draw(page: &mut Bitmap, glyph: &Bitmap, x: i64, y: i64) { + for (gy, row) in glyph.iter().enumerate() { + for (gx, p) in row.iter().enumerate() { + let (px, py) = (x + gx as i64, y + gy as i64); + if *p == 1 + && px >= 0 + && py >= 0 + && let Some(cell) = page.get_mut(py as usize).and_then(|r| r.get_mut(px as usize)) + { + *cell = 1; + } + } + } + } + + /// Lines of text: glyphs side by side, one to four pixels apart, each line's glyphs on its + /// baseline, the lines 30 rows apart; the first glyphs are the most common, as letters are. + fn text(columns: usize, rows: usize, glyphs: &[Bitmap], seed: u64) -> Bitmap { + let mut page = vec![vec![0u8; columns]; rows]; + let mut r = Lcg(seed); + for baseline in (24..rows).step_by(30) { + let mut x = 4 + r.below(8); + loop { + let common = r.below(glyphs.len()) + 1; + let g = &glyphs[r.below(common)]; + if x + g[0].len() > columns { + break; + } + draw(&mut page, g, x as i64, (baseline + 1 - g.len()) as i64); + x += g[0].len() + 1 + r.below(4); + } + } + page + } + + /// `n` glyphs anywhere, some over the edges, some over each other. + fn scatter(columns: usize, rows: usize, glyphs: &[Bitmap], seed: u64, n: usize) -> Bitmap { + let mut page = vec![vec![0u8; columns]; rows]; + let mut r = Lcg(seed); + for _ in 0..n { + let g = &glyphs[r.below(glyphs.len())]; + draw(&mut page, g, r.below(columns + 12) as i64 - 6, r.below(rows + 12) as i64 - 6); + } + page + } + + /// A segment's number and type, whether it is retained, and the segments it refers to, each + /// with its retain bit. + type Segment = (u32, u8, bool, Vec<(u8, bool)>); + + /// The segments of a stream in the embedded organisation, from their headers (7.2). + fn segments(data: &[u8]) -> Vec { + let (mut segments, mut at) = (Vec::new(), 0); + while at < data.len() { + let flags = data[at + 5]; + let refers = usize::from(flags >> 5); + let length_at = at + 6 + refers + 1; + let length = u32::from_be_bytes(data[length_at..length_at + 4].try_into().unwrap()) as usize; + let referred = data[at + 6..at + 6 + refers].iter().enumerate().map(|(k, n)| (*n, flags >> (k + 1) & 1 == 1)).collect(); + let number = u32::from_be_bytes(data[at..at + 4].try_into().unwrap()); + segments.push((number, data[at + 4] & 0x3F, flags & 1 == 1, referred)); + at = length_at + 4 + length; + } + segments + } + + fn segment_types(data: &[u8]) -> Vec { + segments(data).into_iter().map(|(_, kind, ..)| kind).collect() + } + + /// Every plan, at every strip size, decodes to the picture; returns how many plans there were. + fn every_plan_round_trips(picture: &Bitmap) -> usize { + let (columns, rows) = (picture[0].len() as u32, picture.len() as u32); + let data = samples(picture); + let page = page(&data, columns).unwrap(); + let mut plans = 0; + for copies in [1, 2] { + for log_strips in 0..4 { + if let Some(coded) = plan(&page, columns, rows, copies, Some(log_strips), None) { + assert_eq!(decode(&coded), unpack(&data, columns, rows), "{copies} copies, strips of {}", 1 << log_strips); + plans += 1; + } + } + } + plans + } + + /// Annex A.2's example (12 through IADW: the decisions 0101000, in contexts 1, 10, 101, …) + /// and Table A.1's codes. + #[test] + fn integers_are_coded_as_annex_a_says() { + assert_eq!(decisions(Some(12)), [(1, false), (2, true), (5, false), (10, true), (21, false), (42, false), (84, false)]); + let bits = |v| decisions(v).iter().map(|(_, d)| u8::from(*d)).collect::>(); + assert_eq!(bits(None), [1, 0, 0, 0], "OOB"); + assert_eq!(bits(Some(0)), [0, 0, 0, 0]); + assert_eq!(bits(Some(-1)), [1, 0, 0, 1]); + assert_eq!(bits(Some(-3)), [1, 0, 1, 1]); + assert_eq!(bits(Some(4)), [0, 1, 0, 0, 0, 0, 0]); + assert_eq!(bits(Some(-19)), [1, 1, 0, 1, 1, 1, 1]); + assert_eq!(bits(Some(83)), [0, 1, 1, 0, 1, 1, 1, 1, 1, 1]); + assert_eq!(bits(Some(4435))[..6], [0, 1, 1, 1, 1, 0]); + let big = bits(Some(4436)); + assert_eq!((big.len(), &big[..6]), (6 + 32, &[0, 1, 1, 1, 1, 1][..])); + assert!(big[6..].iter().all(|b| *b == 0)); + // PREV keeps a leading 1 and the last eight decisions, so it stays below 512, also when + // those eight are all 0 (PREV exactly 256), as in 4436's offset of 0. + for value in [Some(i32::MIN), Some(4436), Some(-4436), Some(4436 + 0x100), Some(i32::MAX), None] { + let contexts: Vec = decisions(value).iter().map(|(cx, _)| *cx).collect(); + assert!(contexts.iter().take(8).all(|cx| *cx < 256) && contexts.iter().skip(8).all(|cx| (256..512).contains(cx)), "{value:?}"); + } + assert!(decisions(Some(4436)).iter().any(|(cx, _)| *cx == 256)); + } + + /// Strips and first columns thousands of pixels apart need Annex A's 32-bit range: a mark at + /// the far right of a 12,000-pixel page, then one at the left below it (S back by 11,000), and + /// lines 5,000 rows apart. Each shape occurs twice, so both plans are made. + #[test] + fn coordinates_past_4435_round_trip() { + let shapes = glyphs(3, 4); + let mut picture = vec![vec![0u8; 12_000]; 40]; + for (k, x) in [11_000, 0, 5_000, 11_980].into_iter().enumerate() { + draw(&mut picture, &shapes[k % 2], x, 2 + 9 * k as i64); + } + assert_eq!(every_plan_round_trips(&picture), 8); + let mut tall = vec![vec![0u8; 30]; 10_050]; + for (k, y) in [0, 5_000, 10_030].into_iter().enumerate() { + draw(&mut tall, &shapes[k % 3], 3, y); + draw(&mut tall, &shapes[(k + 1) % 3], 15, y); + } + assert_eq!(every_plan_round_trips(&tall), 8); + } + + /// The padding bits of a row whose width isn't a multiple of 8 are ignored whatever they are: + /// black (0) padding, as many scans have, neither joins the marks nor stops symbol coding. + #[test] + fn black_padding_bits_are_ignored() { + for columns in [13, 241, 901] { + let picture = text(columns, 120, &glyphs(12, 6), 8); + let mut data = samples(&picture); + let stride = columns.div_ceil(8); + for row in data.chunks_mut(stride) { + row[stride - 1] &= 0xFF << (8 * stride - columns); + } + let (c, r) = (columns as u32, 120); + let page = page(&data, c).unwrap(); + assert!(page.marks.iter().all(|m| m.x1 as usize <= columns), "{columns} columns"); + for copies in [1, 2] { + if let Some(coded) = plan(&page, c, r, copies, None, None) { + assert_eq!(decode(&coded), unpack(&data, c, r), "{columns} columns"); + } + } + if columns > 13 { + assert_eq!(segment_types(&encode_jbig2(&data, c, r).unwrap())[1], 0, "{columns} columns: symbol coded"); + } + } + } + + /// Pixels touching only at a corner, down to the right or down to the left, are one mark + /// (8-connected); side by side with a white column between, two. A mark of 256 pixels a side + /// is a symbol; one 257 wide, or 257 tall, is left to the generic region. The diagonal and the + /// dot occur twice, so both plans are made. + #[test] + fn marks_join_at_corners_and_large_ones_are_left_out() { + let mut picture = vec![vec![0u8; 300]; 300]; + for i in 0..6 { + picture[10 + i][10 + i] = 1; + picture[10 + i][100 + i] = 1; + picture[10 + i][60 - i] = 1; + } + for row in &mut picture[0..257] { + row[280] = 1; + } + picture[30][10] = 1; + picture[30][12] = 1; + picture[40][0..256].fill(1); + picture[60][0..257].fill(1); + for row in &mut picture[44..300] { + row[290] = 1; + } + let data = samples(&picture); + let page = page(&data, 300).unwrap(); + let boxes: Vec<(u32, u32, u32, u32)> = page.marks.iter().map(|m| (m.x0, m.y0, m.x1 - m.x0, m.y1 - m.y0)).collect(); + assert_eq!( + boxes, + [ + (280, 0, 1, 257), + (10, 10, 6, 6), + (55, 10, 6, 6), + (100, 10, 6, 6), + (10, 30, 1, 1), + (12, 30, 1, 1), + (0, 40, 256, 1), + (290, 44, 1, 256), + (0, 60, 257, 1) + ] + ); + let large: Vec<(u32, u32)> = page.large.iter().map(|m| (page.marks[*m as usize].x0, page.marks[*m as usize].y0)).collect(); + assert_eq!(large, [(280, 0), (0, 60)]); + assert_eq!(every_plan_round_trips(&picture), 8); + } + + /// Past 2^20 black runs the image is left to generic coding; at the cap it is still looked at. + #[test] + fn busy_images_are_left_to_generic_coding() { + // 0x55: four black pixels a byte, each its own run, under the same ones on the next row. + let columns = 1 << 20; + let row = vec![0x55u8; columns / 8]; + assert!(page(&row.repeat(2), columns as u32).is_some(), "2^20 runs"); + assert!(page(&row.repeat(3), columns as u32).is_none(), "1.5 × 2^20 runs"); + } + + /// A plan codes at most 32,767 symbols, which an independent decoder reads back; with more, + /// the second plan can still code the repeated ones. Past 65,536 distinct marks, or 2^24 + /// pixels of them, the image is left to generic coding; at either cap it is still looked at. + #[test] + fn too_many_symbols_are_left_to_generic_coding() { + // `n` distinct marks of 17 × 3 (full top and bottom rows joined by a code of 1 to n, in + // binary, on the row between), then `again` more with the codes 1 and 2 in turn. + let codes = |n: usize, again: usize| { + let (columns, rows) = (256 * 18, (n + again).div_ceil(256) * 4); + let mut picture = vec![vec![0u8; columns]; rows]; + for k in 0..n + again { + let (x, y) = (k % 256 * 18, k / 256 * 4); + let code = if k < n { k + 1 } else { (k - n) % 2 + 1 }; + picture[y][x..x + 17].fill(1); + picture[y + 2][x..x + 17].fill(1); + for b in (0..17).filter(|b| code >> b & 1 == 1) { + picture[y + 1][x + b] = 1; + } + } + (samples(&picture), columns as u32, rows as u32) + }; + let (data, columns, rows) = codes(MAX_CODED_SYMBOLS, 0); + let at_cap = page(&data, columns).unwrap(); + assert_eq!(decode(&plan(&at_cap, columns, rows, 1, None, None).unwrap()), unpack(&data, columns, rows)); + // One more distinct mark, each once: neither plan is made. + let (data, columns, rows) = codes(MAX_CODED_SYMBOLS + 1, 0); + let past = page(&data, columns).unwrap(); + assert!(plan(&past, columns, rows, 1, None, None).is_none() && plan(&past, columns, rows, 2, None, None).is_none()); + assert_eq!(encode_jbig2(&data, columns, rows).unwrap(), encode_jbig2_generic(&data, columns, rows).unwrap()); + // With two of them repeated, the second plan codes those two. + let (data, columns, rows) = codes(MAX_CODED_SYMBOLS + 1, 200); + let past = page(&data, columns).unwrap(); + assert!(plan(&past, columns, rows, 1, None, None).is_none()); + assert_eq!(decode(&plan(&past, columns, rows, 2, None, None).unwrap()), unpack(&data, columns, rows)); + // Above a page of text whose letters repeat, that second plan is the one `encode_jbig2` + // keeps: the first, of every mark, would code too many symbols. + let mut picture: Bitmap = vec![vec![0u8; 256 * 18]; 512]; + for k in 0..=MAX_CODED_SYMBOLS { + let (x, y) = (k % 256 * 18, k / 256 * 4); + picture[y][x..x + 17].fill(1); + picture[y + 2][x..x + 17].fill(1); + for b in (0..17).filter(|b| (k + 1) >> b & 1 == 1) { + picture[y + 1][x + b] = 1; + } + } + picture.extend(text(256 * 18, 600, &glyphs(40, 7), 11)); + let data = samples(&picture); + let (columns, rows) = (picture[0].len() as u32, picture.len() as u32); + let coded = encode_jbig2(&data, columns, rows).unwrap(); + assert_eq!(segment_types(&coded)[1], 0, "symbol coded"); + assert_eq!(decode(&coded), unpack(&data, columns, rows)); + let (data, columns, _) = codes(MAX_SYMBOLS, 0); + assert_eq!(page(&data, columns).map(|p| p.symbols.len()), Some(MAX_SYMBOLS)); + let (data, columns, _) = codes(MAX_SYMBOLS + 1, 0); + assert!(page(&data, columns).is_none()); + // `n` black squares of 256 × 256, each with a white pixel in a different place: 2^16 + // pixels a symbol, so 256 of them are exactly the cap. + let squares = |n: usize| { + let (columns, rows) = (16 * 258, n.div_ceil(16) * 258); + let mut picture = vec![vec![0u8; columns]; rows]; + for k in 0..n { + let (x, y) = (k % 16 * 258, k / 16 * 258); + for row in &mut picture[y..y + 256] { + row[x..x + 256].fill(1); + } + picture[y + 1 + k / 254][x + 1 + k % 254] = 0; + } + (samples(&picture), columns as u32, rows as u32) + }; + let (data, columns, rows) = squares(256); + let at_cap = page(&data, columns).unwrap(); + assert_eq!(at_cap.symbols.len(), 256); + assert_eq!(decode(&plan(&at_cap, columns, rows, 1, None, None).unwrap()), unpack(&data, columns, rows)); + let (data, columns, _) = squares(257); + assert!(page(&data, columns).is_none()); + } + + /// More than 10,000 marks are placed by several text regions, each of at most 10,000 (more + /// than `hayro-jbig2` accepts in one), over bands of rows, all referring to + /// the dictionary, which each region but the last keeps retained; the generic region comes + /// after them, every segment numbered one more than the one before. + #[test] + fn more_than_10000_marks_are_placed_by_several_regions() { + // 25,000 copies of two 17 × 3 shapes, 256 a line, the lines 5 rows apart, and a bar too + // tall to be a symbol beside them. + let mut picture = vec![vec![0u8; 256 * 18 + 100]; 25_000usize.div_ceil(256) * 5]; + for k in 0..25_000 { + let (x, y) = (k % 256 * 18, k / 256 * 5); + picture[y][x..x + 17].fill(1); + picture[y + 2][x..x + 17].fill(1); + picture[y + 1][x + k % 2] = 1; + } + for row in &mut picture[0..300] { + row[256 * 18 + 50] = 1; + } + let data = samples(&picture); + let (columns, rows) = (picture[0].len() as u32, picture.len() as u32); + let coded = encode_jbig2(&data, columns, rows).unwrap(); + let region = |number, kept| (number, 7, false, vec![(1, kept)]); + assert_eq!( + segments(&coded), + [(0, 48, false, vec![]), (1, 0, true, vec![]), region(2, true), region(3, true), region(4, false), (5, 39, false, vec![])] + ); + assert_eq!(decode(&coded), unpack(&data, columns, rows)); + assert_eq!(every_plan_round_trips(&picture), 8); + } + + /// A page of text whose letters repeat exactly, as rendered text does, keeps each letter once: + /// a symbol dictionary and a text region, much smaller than the generic coding. + #[test] + fn repeated_marks_are_stored_once() { + let picture = text(1700, 400, &glyphs(40, 7), 11); + let data = samples(&picture); + let coded = encode_jbig2(&data, 1700, 400).unwrap(); + let generic = encode_jbig2_generic(&data, 1700, 400).unwrap(); + // The dictionary is retained for the text region, which refers to it (7.2.4, 7.2.5). + assert_eq!(segments(&coded), [(0, 48, false, vec![]), (1, 0, true, vec![]), (2, 7, false, vec![(1, false)])]); + assert_eq!(decode(&coded), unpack(&data, 1700, 400)); + assert!(coded.len() * 2 < generic.len(), "symbols {} bytes, generic {}", coded.len(), generic.len()); + assert_eq!(every_plan_round_trips(&picture), 8); + } + + /// Marks too large to be symbols (a frame, a rule) go to a generic region over their box, and + /// with the second plan so do marks that occur once; the page still decodes to its bits. + #[test] + fn large_and_lone_marks_are_left_to_a_generic_region() { + let mut picture = text(900, 300, &glyphs(25, 3), 5); + picture[10][100..800].fill(1); + picture[290][100..800].fill(1); + for row in &mut picture[10..=290] { + row[100] = 1; + row[799] = 1; + } + // A lone blob, unlike any letter. + for (y, row) in picture[150..170].iter_mut().enumerate() { + row[850 + y % 7] = 1; + } + let data = samples(&picture); + let coded = encode_jbig2(&data, 900, 300).unwrap(); + assert_eq!(segment_types(&coded), [48, 0, 7, 39]); + assert_eq!(decode(&coded), unpack(&data, 900, 300)); + assert_eq!(every_plan_round_trips(&picture), 8); + } + + /// Marks on every edge, marks touching only at a corner (one mark), a mark inside another's + /// box, and marks over each other's boxes. + #[test] + fn marks_at_the_edges_and_corners_round_trip() { + let dot = vec![vec![1u8, 1], vec![1, 1]]; + let ell = vec![vec![1u8, 0, 0], vec![1, 0, 0], vec![1, 1, 1]]; + let mut picture = vec![vec![0u8; 41]; 23]; + for (x, y) in [(0, 0), (39, 0), (0, 21), (39, 21), (20, 0), (0, 10), (39, 10), (20, 21)] { + draw(&mut picture, &dot, x, y); + } + for (x, y) in [(10, 5), (28, 14), (12, 14)] { + draw(&mut picture, &ell, x, y); + } + // Corners touching: one mark (`marks_join_at_corners_and_large_ones_are_left_out` checks it). + picture[3][6] = 1; + picture[4][7] = 1; + picture[5][8] = 1; + // A ring with a dot inside, and an ell over the ring's box. + for i in 0..7 { + picture[8][20 + i] = 1; + picture[14][20 + i] = 1; + picture[8 + i][20] = 1; + picture[8 + i][26] = 1; + } + draw(&mut picture, &dot, 22, 10); + draw(&mut picture, &ell, 25, 15); + assert_eq!(every_plan_round_trips(&picture), 8); + } + + /// One shape, however often, would need symbol IDs of no bits: that image is coded as a generic + /// region, as are a blank page and pure noise. + #[test] + fn one_shape_blank_pages_and_noise_are_coded_generically() { + let letter = &glyphs(1, 9)[..]; + let mut r = Lcg(1); + let noise: Bitmap = (0..60).map(|_| (0..200).map(|_| u8::from(r.below(2) == 1)).collect()).collect(); + for picture in [text(600, 200, letter, 2), vec![vec![0u8; 300]; 40], noise] { + let (columns, rows) = (picture[0].len() as u32, picture.len() as u32); + let data = samples(&picture); + assert_eq!(encode_jbig2(&data, columns, rows).unwrap(), encode_jbig2_generic(&data, columns, rows).unwrap()); + } + } + + /// Each segment's page association (7.2.6): the byte after the numbers it refers to. + fn associations(data: &[u8]) -> Vec { + let (mut pages, mut at) = (Vec::new(), 0); + while at < data.len() { + let refers = usize::from(data[at + 5] >> 5); + pages.push(data[at + 6 + refers]); + let length_at = at + 6 + refers + 1; + at = length_at + 4 + u32::from_be_bytes(data[length_at..length_at + 4].try_into().unwrap()) as usize; + } + pages + } + + /// `pictures`, `columns` × `rows` each, coded sharing a dictionary; each image's stream, which + /// has to be there, decodes with the globals to its picture. + fn shared_round_trip(pictures: &[Bitmap]) -> (Vec, Vec>) { + let (columns, rows) = (pictures[0][0].len() as u32, pictures[0].len() as u32); + let data: Vec> = pictures.iter().map(|p| samples(p)).collect(); + let images: Vec<(&[u8], u32, u32)> = data.iter().map(|d| (&d[..], columns, rows)).collect(); + let shared = encode_jbig2_shared(&images).unwrap().unwrap(); + let (globals, coded) = (shared.globals, shared.images); + let coded: Vec> = coded.into_iter().map(Option::unwrap).collect(); + for (d, c) in data.iter().zip(&coded) { + assert_eq!(decode_with(c, Some(&globals)), unpack(d, columns, rows)); + } + (globals, coded) + } + + /// `font` with a stem 20 pixels tall down each glyph's left side: a font sharing no glyph with + /// any [`glyphs`] gives, whose glyphs are at most 18 pixels tall. + fn stemmed(font: &[Bitmap]) -> Vec { + font.iter() + .map(|g| { + (0..g.len().max(20)).map(|y| std::iter::once(1).chain(g.get(y).cloned().unwrap_or_else(|| vec![0; g[0].len()])).collect()).collect() + }) + .collect() + } + + /// Mark `k` of a set of distinct ones: a stroke 33 pixels long with a pixel above every other + /// column for each 1 bit of `k`, below 2^17. + fn comb(k: usize) -> Bitmap { + let mut g = vec![vec![0u8; 33]; 2]; + g[1].fill(1); + for bit in 0..17 { + g[0][2 * bit] = u8::from(k >> bit & 1 == 1); + } + g + } + + /// The combs `ks`, 120 a row, the rows four pixels apart. + fn combs(ks: impl ExactSizeIterator) -> Bitmap { + let mut page = vec![vec![0u8; 120 * 36]; ks.len().div_ceil(120) * 4]; + for (i, k) in ks.enumerate() { + draw(&mut page, &comb(k), (i % 120 * 36) as i64, (i / 120 * 4) as i64); + } + page + } + + /// The symbols a symbol dictionary segment exports (SDNUMEXSYMS, 7.4.2.1.4), in a stream + /// that is that segment alone: after its header, flags and adaptive pixels. + fn exported(globals: &[u8]) -> usize { + u32::from_be_bytes(globals[21..25].try_into().unwrap()) as usize + } + + /// Pages in one font share its letters: one symbol dictionary, segment 0, associated with no + /// page and retained (7.2.4, 7.2.6), which each page's text region refers to first, its own + /// segments numbered after it (7.2.5). Together they take less than they take alone. + #[test] + fn pages_in_one_font_share_one_dictionary() { + let font = glyphs(40, 7); + let pictures: Vec = (0..4).map(|seed| text(1200, 300, &font, 20 + seed)).collect(); + let (globals, coded) = shared_round_trip(&pictures); + assert_eq!(segments(&globals), [(0, 0, true, vec![])]); + assert_eq!(associations(&globals), [0]); + let (mut shared, mut alone) = (globals.len(), 0); + for (picture, c) in pictures.iter().zip(&coded) { + let segments = segments(c); + assert_eq!(segments.first().map(|s| s.0), Some(1), "the page's own segments follow"); + assert!(segments.iter().filter(|s| s.1 == 7).all(|s| s.3.first() == Some(&(0, false))), "{segments:?}"); + assert!(associations(c).iter().all(|page| *page == 1)); + shared += c.len(); + alone += encode_jbig2(&samples(picture), 1200, 300).unwrap().len(); + } + assert!(shared < alone, "shared {shared} bytes, alone {alone}"); + } + + /// Two pages alike place every mark from the shared dictionary and have none of their own; a + /// page with letters of its own as well keeps them in its own dictionary, referred to after the + /// shared one (its symbols' IDs follow), and the page with nothing shared has no stream. + #[test] + fn what_a_page_has_alone_stays_its_own() { + let (font, other) = (glyphs(30, 7), stemmed(&glyphs(20, 99))); + let same = text(900, 240, &font, 3); + let mut more = text(900, 240, &font, 4); + more.extend(text(900, 120, &other, 5)); + let (globals, coded) = shared_round_trip(&[same.clone(), same.clone()]); + assert!(!globals.is_empty()); + assert!(coded.iter().all(|c| segment_types(c) == [48, 7]), "{:?}", coded.iter().map(|c| segment_types(c)).collect::>()); + let with_more = |p: &Bitmap| p.iter().cloned().chain(std::iter::repeat_n(vec![0u8; 900], 120)).collect::(); + let (_, coded) = shared_round_trip(&[with_more(&same), more]); + assert_eq!(segment_types(&coded[1])[..3], [48, 0, 7]); + assert_eq!(segments(&coded[1])[2].3, [(0, false), (2, false)]); + let alone = text(900, 360, &other, 6); + let data: Vec> = [&same, &same, &alone].iter().map(|p| samples(p)).collect(); + let images: Vec<(&[u8], u32, u32)> = vec![(&data[0], 900, 240), (&data[1], 900, 240), (&data[2], 900, 360)]; + let (_, coded) = encode_shared(&images).unwrap(); + assert!(coded[0].is_some() && coded[1].is_some() && coded[2].is_none()); + } + + /// Images that share fewer than two marks share nothing; two are enough. + #[test] + fn images_with_nothing_in_common_share_nothing() { + let font = glyphs(30, 7); + let (a, b) = (samples(&text(600, 200, &font, 1)), samples(&text(600, 200, &stemmed(&font), 1))); + assert!(encode_shared(&[(&a, 600, 200), (&b, 600, 200)]).is_none()); + assert!(encode_shared(&[(&a, 600, 200)]).is_none(), "one image"); + assert!(encode_shared(&[]).is_none()); + // The two commonest letters of the first page, below the last line of the second. + let mut two = text(600, 200, &stemmed(&font), 1); + draw(&mut two, &font[0], 10, 180); + draw(&mut two, &font[1], 100, 180); + let two = samples(&two); + let (globals, coded) = encode_shared(&[(&a, 600, 200), (&two, 600, 200)]).unwrap(); + assert_eq!(exported(&globals), 2); + assert_eq!(decode_with(coded[1].as_ref().unwrap(), Some(&globals)), unpack(&two, 600, 200)); + } + + /// The images of a group are checked as `encode_jbig2_generic` checks one, and an image it + /// would refuse refuses the group. + #[test] + fn a_group_with_an_image_the_encoder_refuses_is_refused() { + let page = samples(&text(600, 200, &glyphs(30, 7), 1)); + assert!(matches!(encode_jbig2_shared(&[(&page, 600, 200), (&page[..10], 600, 200)]), Err(CodecError::Corrupt { .. }))); + assert!(matches!(encode_jbig2_shared(&[(&page, 600, 200), (&page, 0, 200)]), Err(CodecError::Unsupported(_)))); + assert!(encode_jbig2_shared(&[(&page, 600, 200), (&page, 600, 200)]).unwrap().is_some()); + } + + /// A group holds its distinct marks within what one image's are held to: past 65,536 a mark + /// first seen later isn't held, so isn't shared, while those held count every image they're + /// in. Ten marks of the first image are in two images after it, with 464 first seen there. + #[test] + fn past_the_marks_a_group_holds_later_ones_arent_shared() { + let (a, b) = (samples(&combs(0..40_000)), samples(&combs(40_000..65_536))); + let late = combs((0..10).chain(65_536..66_000).collect::>().into_iter()); + let (rows, late) = (late.len() as u32, samples(&late)); + let (columns, a_rows, b_rows) = (120 * 36, 40_000u32.div_ceil(120) * 4, 25_536u32.div_ceil(120) * 4); + let images = [(&a[..], columns, a_rows), (&b[..], columns, b_rows), (&late[..], columns, rows), (&late[..], columns, rows)]; + let (globals, coded) = encode_shared(&images).unwrap(); + assert_eq!(exported(&globals), 10); + for c in &coded[2..] { + assert_eq!(decode_with(c.as_ref().unwrap(), Some(&globals)), unpack(&late, columns, rows)); + } + } + + /// Past what one dictionary shares, it keeps the marks the most images hold: 20,000 shared, + /// the first 10,000 in all three images and the rest in two. The third image then places all + /// its marks from it, and the other two code the rest themselves (in a dictionary of their own + /// or a generic region, whichever is smaller). + #[test] + fn past_the_shared_limit_the_most_widely_held_marks_are_shared() { + let (both, first) = (combs(0..20_000), combs(0..10_000)); + let rows = both.len(); + let first: Bitmap = first.iter().cloned().chain(std::iter::repeat_n(vec![0u8; 120 * 36], rows - first.len())).collect(); + let (globals, coded) = shared_round_trip(&[both.clone(), both, first]); + assert_eq!(exported(&globals), MAX_SHARED_SYMBOLS); + assert_eq!(segment_types(&coded[2]), [48, 7]); + assert!(segment_types(&coded[0]).iter().any(|t| *t == 0 || *t == 39), "{:?}", segment_types(&coded[0])); + } + + proptest::proptest! { + /// Glyphs scattered over pages of one font: whatever is shared, every page with a stream + /// decodes with the globals to its picture. + #[test] + fn shared_groups_round_trip( + columns in 1usize..200, + rows in 1usize..100, + shapes in 2usize..8, + n in 0usize..60, + pages in 2usize..5, + seed in proptest::num::u64::ANY, + ) { + let font = glyphs(shapes, seed); + let data: Vec> = (0..pages).map(|p| samples(&scatter(columns, rows, &font, seed ^ p as u64, n))).collect(); + let (c, r) = (columns as u32, rows as u32); + let images: Vec<(&[u8], u32, u32)> = data.iter().map(|d| (&d[..], c, r)).collect(); + if let Some((globals, coded)) = encode_shared(&images) { + for (d, coded) in data.iter().zip(coded) { + if let Some(coded) = coded { + proptest::prop_assert_eq!(decode_with(&coded, Some(&globals)), unpack(d, c, r)); + } + } + } + } + + /// Glyphs anywhere, over the edges and each other, with stray pixels: every plan decodes to + /// the picture, and `encode_jbig2` is never larger than the generic coding. + #[test] + fn scattered_marks_round_trip( + columns in 1usize..240, + rows in 1usize..120, + shapes in 2usize..10, + n in 0usize..80, + seed in proptest::num::u64::ANY, + stray in proptest::collection::vec((0usize..240, 0usize..120), 0..20), + ) { + let mut picture = scatter(columns, rows, &glyphs(shapes, seed), seed, n); + for (x, y) in stray { + if let Some(cell) = picture.get_mut(y).and_then(|r| r.get_mut(x)) { + *cell ^= 1; + } + } + every_plan_round_trips(&picture); + let data = samples(&picture); + let (c, r) = (columns as u32, rows as u32); + let coded = encode_jbig2(&data, c, r).unwrap(); + proptest::prop_assert!(coded.len() <= encode_jbig2_generic(&data, c, r).unwrap().len()); + proptest::prop_assert_eq!(decode(&coded), unpack(&data, c, r)); + } + + /// Whatever it is given, it returns. + #[test] + fn any_input_returns(data in proptest::collection::vec(0u8..=255, 0..600), columns in 0u32..5000, rows in 0u32..80) { + let _ = encode_jbig2(&data, columns, rows); + let _ = encode(&data, columns, rows); + let _ = encode_shared(&[(&data, columns, rows), (&data, columns, rows.saturating_sub(1))]); + } + } +} diff --git a/crates/stream-codecs/src/lib.rs b/crates/stream-codecs/src/lib.rs new file mode 100644 index 000000000..04a935333 --- /dev/null +++ b/crates/stream-codecs/src/lib.rs @@ -0,0 +1,117 @@ +//! Byte-stream and bi-level image codecs that PDF, PostScript and TIFF share. +//! +//! - [`lzw`]: variable-width LZW, 9 to 12 bits, most significant bit first (PDF `LZWDecode`, +//! TIFF compression 5), with or without the early code-width change. +//! - [`ascii85`], [`asciihex`], [`runlength`]: the PostScript and PDF text and run-length +//! encodings (`ASCII85Decode`, `ASCIIHexDecode`, `RunLengthDecode`; PackBits). +//! - [`predictor`]: TIFF horizontal differencing and the PNG row filters. +//! - [`ccitt`]: CCITT Group 3 and Group 4 fax decoding, bounded, and Group 4 encoding. +//! - [`jbig2`]: lossless JBIG2 encoding (generic regions, symbol dictionaries, a dictionary +//! shared by a group of images). Decoding is `hayro-jbig2`'s. +//! +//! The crate knows no container: a caller hands it bytes and parameters. A PDF `/Filter` chain +//! (order, limits across stages, Flate) is `craft_pdf::filters`, built on this crate. +//! +//! Every decoder is bounded by a caller-supplied output limit and never panics on malformed +//! input. A decoder that fails returns what it decoded before the damage ([`Failure`]), so a +//! caller can choose to keep it. CCITT decoding spends from an allowance the caller shares +//! across a run of images ([`pay`]); it does not decode without one: +//! +//! ```compile_fail +//! let _ = craft_stream_codecs::decode_ccitt(&[], &craft_stream_codecs::CcittParams::default(), 1); +//! ``` +//! +//! ``` +//! let mut work = 1 << 20; +//! let p = craft_stream_codecs::CcittParams { columns: 8, ..Default::default() }; +//! assert!(craft_stream_codecs::decode_ccitt_within(&[], &p, 1, &mut work).is_ok()); +//! ``` +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +pub mod ascii85; +pub mod asciihex; +pub mod ccitt; +pub mod jbig2; +pub mod lzw; +mod mq; +pub mod predictor; +pub mod runlength; + +pub use ccitt::{CcittParams, MAX_CCITT_COLUMNS, MAX_G4_COLUMNS, decode_ccitt_within, encode_g4}; +pub use jbig2::{MAX_JBIG2_COLUMNS, MAX_JBIG2_PIXELS, SharedJbig2, encode_jbig2, encode_jbig2_generic, encode_jbig2_shared}; + +/// Parameters of the predictors and of LZW, as a PDF `/DecodeParms` dictionary gives them +/// (ISO 32000-2 Tables 8–9); TIFF's `Predictor`, `SamplesPerPixel`, `BitsPerSample` and +/// `ImageWidth` map onto the same four. +#[derive(Clone, Debug, PartialEq)] +pub struct Params { + /// 1 = none, 2 = TIFF predictor 2, 10–15 = PNG predictors. + pub predictor: i64, + pub colors: i64, + pub bits_per_component: i64, + pub columns: i64, + /// LZW only: 1 (default) switches code width one code early, 0 does not. + pub early_change: i64, +} + +impl Default for Params { + fn default() -> Self { + Params { predictor: 1, colors: 1, bits_per_component: 8, columns: 1, early_change: 1 } + } +} + +/// Errors from decoding or encoding. +#[derive(Debug, thiserror::Error)] +pub enum CodecError { + /// `filter` names the codec, as PDF names its filter (`LZWDecode`, `CCITTFaxDecode`, …). + #[error("{filter}: corrupt data: {detail}")] + Corrupt { filter: &'static str, detail: String }, + #[error("decoded output exceeds the limit of {0} bytes")] + LimitExceeded(usize), + #[error("unsupported filter: {0}")] + Unsupported(String), + /// A run of images has nothing left to spend ([`pay`]). + #[error("more image decoding than one operation may do")] + Spent, +} + +/// A decoder failure carrying whatever was decoded before the error. +pub struct Failure { + pub error: CodecError, + pub partial: Vec, + /// Bytes decoded on the way that `partial` doesn't hold: they took as long. + pub discarded: usize, +} + +impl Failure { + pub fn corrupt(filter: &'static str, detail: impl Into, partial: Vec) -> Self { + Failure { error: CodecError::Corrupt { filter, detail: detail.into() }, partial, discarded: 0 } + } + + pub fn limit(max: usize) -> Self { + Failure { error: CodecError::LimitExceeded(max), partial: Vec::new(), discarded: 0 } + } +} + +/// What a byte-stream decoder returns: the decoded bytes, or the failure with its partial output. +pub type Step = Result, Failure>; + +/// Takes `steps` from `work`, what a run of images may still cost. What it can't pay for spends +/// all of it. +pub fn pay(work: &mut u64, steps: u64) -> Result<(), CodecError> { + match work.checked_sub(steps) { + Some(left) => { + *work = left; + Ok(()) + } + None => { + *work = 0; + Err(CodecError::Spent) + } + } +} + +/// PDF and PostScript white-space characters (ISO 32000-2 Table 1). +pub fn is_pdf_whitespace(b: u8) -> bool { + matches!(b, 0 | 9 | 10 | 12 | 13 | 32) +} diff --git a/crates/stream-codecs/src/lzw.rs b/crates/stream-codecs/src/lzw.rs new file mode 100644 index 000000000..e16d78288 --- /dev/null +++ b/crates/stream-codecs/src/lzw.rs @@ -0,0 +1,314 @@ +//! LZWDecode (ISO 32000-2 §7.4.4.2): variable-width codes of 9–12 bits, MSB first, +//! 256 = clear-table, 257 = EOD, first free code 258. With `EarlyChange` 1 (the +//! default) the code width grows one code early. +//! +//! Tolerated: data ending without EOD, a missing initial clear-table code, a table that +//! fills up without a clear-table code (entries are then no longer added). + +use std::collections::HashMap; + +use crate::{Failure, Step}; + +const NAME: &str = "LZWDecode"; +const CLEAR: u16 = 256; +const EOD: u16 = 257; +const FIRST: usize = 258; +const TABLE: usize = 4096; +/// The encoder emits clear-table when its next free code reaches this value. +const ENCODER_RESET_AT: usize = TABLE; + +/// The code width a decoder uses when its next free code is `next`. +fn width(next: usize, early: bool) -> u32 { + match next + usize::from(early) { + 0..512 => 9, + 512..1024 => 10, + 1024..2048 => 11, + _ => 12, + } +} + +struct BitReader<'a> { + data: &'a [u8], + pos: usize, + acc: u32, + bits: u32, +} + +impl BitReader<'_> { + fn read(&mut self, n: u32) -> Option { + while self.bits < n { + let &b = self.data.get(self.pos)?; + self.pos += 1; + self.acc = (self.acc << 8) | u32::from(b); + self.bits += 8; + } + self.bits -= n; + let v = (self.acc >> self.bits) & ((1 << n) - 1); + self.acc &= (1 << self.bits) - 1; + Some(v as u16) + } +} + +pub fn decode(data: &[u8], early: bool, max: usize) -> Step { + // Table entry = (prefix code, last byte, first byte, length). + let mut prefix = vec![0u16; TABLE]; + let mut last = vec![0u8; TABLE]; + let mut first = vec![0u8; TABLE]; + let mut len = vec![0u16; TABLE]; + for i in 0..256 { + last[i] = i as u8; + first[i] = i as u8; + len[i] = 1; + } + let mut next = FIRST; + let mut w = 9; + let mut prev: Option = None; + let mut r = BitReader { data, pos: 0, acc: 0, bits: 0 }; + let mut out = Vec::with_capacity(data.len().saturating_mul(2).min(max)); + + while let Some(code) = r.read(w) { + match code { + CLEAR => { + next = FIRST; + w = 9; + prev = None; + continue; + } + EOD => break, + _ => {} + } + let code = usize::from(code); + if let Some(p) = prev { + let head = if code < next { + first[code] + } else if code == next && next < TABLE { + first[p] + } else { + return Err(Failure::corrupt(NAME, format!("code {code} not in table (next {next})"), out)); + }; + if next < TABLE { + prefix[next] = p as u16; + last[next] = head; + first[next] = first[p]; + len[next] = len[p] + 1; + next += 1; + } + } else if code >= 256 { + return Err(Failure::corrupt(NAME, format!("code {code} before any table entry"), out)); + } + // Emit the string for `code`, walking the prefix chain backwards. + let n = usize::from(len[code]); + if out.len() + n > max { + return Err(Failure::limit(max)); + } + let start = out.len(); + out.resize(start + n, 0); + let mut c = code; + for slot in out[start..].iter_mut().rev() { + *slot = last[c]; + c = usize::from(prefix[c]); + } + prev = Some(code); + w = width(next, early); + } + Ok(out) +} + +struct BitWriter { + out: Vec, + acc: u32, + bits: u32, +} + +impl BitWriter { + fn write(&mut self, code: u16, n: u32) { + self.acc = (self.acc << n) | u32::from(code); + self.bits += n; + while self.bits >= 8 { + self.bits -= 8; + self.out.push((self.acc >> self.bits) as u8); + } + self.acc &= (1 << self.bits) - 1; + } + + fn finish(mut self) -> Vec { + if self.bits > 0 { + self.out.push((self.acc << (8 - self.bits)) as u8); + } + self.out + } +} + +/// Greedy LZW with a leading clear-table code and a trailing EOD. +pub fn encode(data: &[u8], early: bool) -> Vec { + let mut wr = BitWriter { out: Vec::with_capacity(data.len() / 2 + 8), acc: 0, bits: 0 }; + let mut dict: HashMap<(u16, u8), u16> = HashMap::new(); + // `next` is the encoder's next free code; `dnext` mirrors the decoder's, which lags + // one entry behind (the decoder adds an entry only once it sees the following code). + let mut next = FIRST; + let mut dnext = FIRST; + let mut codes_since_clear = 0usize; + wr.write(CLEAR, 9); + let emit = |wr: &mut BitWriter, code: u16, dnext: &mut usize, since: &mut usize| { + wr.write(code, width(*dnext, early)); + if *since > 0 && *dnext < TABLE { + *dnext += 1; + } + *since += 1; + }; + let Some((&b0, rest)) = data.split_first() else { + wr.write(EOD, 9); + return wr.finish(); + }; + let mut cur = u16::from(b0); + for &b in rest { + if let Some(&c) = dict.get(&(cur, b)) { + cur = c; + continue; + } + emit(&mut wr, cur, &mut dnext, &mut codes_since_clear); + dict.insert((cur, b), next as u16); + next += 1; + if next == ENCODER_RESET_AT { + wr.write(CLEAR, width(dnext, early)); + dict.clear(); + next = FIRST; + dnext = FIRST; + codes_since_clear = 0; + } + cur = u16::from(b); + } + emit(&mut wr, cur, &mut dnext, &mut codes_since_clear); + wr.write(EOD, width(dnext, early)); + wr.finish() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::CodecError; + + fn dec(d: &[u8], early: bool) -> Result, CodecError> { + decode(d, early, 1 << 26).map_err(|f| f.error) + } + + /// ISO 32000-2 §7.4.4.2 example: codes 256 45 258 258 65 259 66 257. + const SPEC_IN: [u8; 10] = [45, 45, 45, 45, 45, 65, 45, 45, 45, 66]; + const SPEC_OUT: [u8; 9] = [0x80, 0x0B, 0x60, 0x50, 0x22, 0x0C, 0x0C, 0x85, 0x01]; + + #[test] + fn spec_example() { + assert_eq!(dec(&SPEC_OUT, true).unwrap(), SPEC_IN); + assert_eq!(encode(&SPEC_IN, true), SPEC_OUT); + // Below 511 codes the early-change setting makes no difference. + assert_eq!(dec(&SPEC_OUT, false).unwrap(), SPEC_IN); + } + + #[test] + fn missing_eod_and_clear() { + // Drop the EOD (last 9 bits span the final 2 bytes; keep bits for 66 only). + let mut w = BitWriter { out: vec![], acc: 0, bits: 0 }; + for c in [45u16, 258, 258, 65, 259, 66] { + w.write(c, 9); + } + assert_eq!(dec(&w.finish(), true).unwrap(), SPEC_IN); + } + + #[test] + fn invalid_codes() { + let mut w = BitWriter { out: vec![], acc: 0, bits: 0 }; + for c in [256u16, 65, 300] { + w.write(c, 9); + } + let f = decode(&w.finish(), true, 1000).err().unwrap(); + assert!(matches!(f.error, CodecError::Corrupt { .. })); + assert_eq!(f.partial, b"A"); + let mut w = BitWriter { out: vec![], acc: 0, bits: 0 }; + w.write(258, 9); + assert!(matches!(dec(&w.finish(), true), Err(CodecError::Corrupt { .. }))); + } + + #[test] + fn widths() { + assert_eq!((width(510, true), width(511, true), width(511, false), width(512, false)), (9, 10, 9, 10)); + assert_eq!((width(1022, true), width(1023, true), width(2047, true), width(4095, false)), (10, 11, 12, 12)); + } + + fn noisy(n: usize, seed: u32) -> Vec { + let mut s = seed; + (0..n) + .map(|_| { + s = s.wrapping_mul(1_103_515_245).wrapping_add(12_345); + ((s >> 16) % 7) as u8 * 31 + }) + .collect() + } + + #[test] + fn all_widths_and_clears_round_trip() { + for early in [false, true] { + for data in [noisy(200_000, 1), (0..=255u8).cycle().take(70_000).collect(), vec![9u8; 100_000]] { + let e = encode(&data, early); + assert_eq!(dec(&e, early).unwrap(), data); + } + } + } + + #[test] + fn cross_check_with_weezl() { + use weezl::BitOrder::Msb; + for data in [noisy(150_000, 7), SPEC_IN.to_vec(), vec![]] { + // early change 1 is the TIFF size switch; 0 is the plain variant. + let tiff = weezl::encode::Encoder::with_tiff_size_switch(Msb, 8).encode(&data).unwrap(); + assert_eq!(dec(&tiff, true).unwrap(), data); + let plain = weezl::encode::Encoder::new(Msb, 8).encode(&data).unwrap(); + assert_eq!(dec(&plain, false).unwrap(), data); + let ours = encode(&data, true); + assert_eq!(weezl::decode::Decoder::with_tiff_size_switch(Msb, 8).decode(&ours).unwrap(), data); + let ours = encode(&data, false); + assert_eq!(weezl::decode::Decoder::new(Msb, 8).decode(&ours).unwrap(), data); + } + } + + #[test] + fn full_table_without_clear() { + // Hand-roll a stream that fills the table and keeps going at 12 bits. + let data = noisy(40_000, 3); + let mut w = BitWriter { out: vec![], acc: 0, bits: 0 }; + let mut dict: HashMap, u16> = (0..=255u8).map(|b| (vec![b], u16::from(b))).collect(); + let (mut next, mut dnext, mut since) = (FIRST, FIRST, 0usize); + let mut cur: Vec = vec![]; + let mut put = |w: &mut BitWriter, code: u16, dnext: &mut usize| { + w.write(code, width(*dnext, true)); + if since > 0 && *dnext < TABLE { + *dnext += 1; + } + since += 1; + }; + w.write(CLEAR, 9); + for &b in &data { + let mut ext = cur.clone(); + ext.push(b); + if dict.contains_key(&ext) { + cur = ext; + continue; + } + put(&mut w, dict[&cur], &mut dnext); + if next < TABLE { + dict.insert(ext, next as u16); + next += 1; + } + cur = vec![b]; + } + put(&mut w, dict[&cur], &mut dnext); + w.write(EOD, width(dnext, true)); + assert_eq!(dec(&w.finish(), true).unwrap(), data); + } + + #[test] + fn limit() { + let e = encode(&vec![0u8; 100_000], true); + assert!(matches!(decode(&e, true, 50_000).map_err(|f| f.error), Err(CodecError::LimitExceeded(50_000)))); + assert_eq!(decode(&e, true, 100_000).ok().unwrap().len(), 100_000); + } +} diff --git a/crates/stream-codecs/src/mq.rs b/crates/stream-codecs/src/mq.rs new file mode 100644 index 000000000..04dda2bd9 --- /dev/null +++ b/crates/stream-codecs/src/mq.rs @@ -0,0 +1,209 @@ +//! The MQ arithmetic encoder of JBIG2 (ITU-T T.88 | ISO/IEC 14492, Annex E), written from the +//! procedures in its Figures E.2–E.12 and the probability estimates of Table E.1. + +/// Table E.1: for each index, the LPS probability estimate Qe, the next index after an MPS and +/// after an LPS renormalisation, and whether an LPS switches the MPS sense. +const STATES: [(u32, u8, u8, bool); 47] = [ + (0x5601, 1, 1, true), + (0x3401, 2, 6, false), + (0x1801, 3, 9, false), + (0x0AC1, 4, 12, false), + (0x0521, 5, 29, false), + (0x0221, 38, 33, false), + (0x5601, 7, 6, true), + (0x5401, 8, 14, false), + (0x4801, 9, 14, false), + (0x3801, 10, 14, false), + (0x3001, 11, 17, false), + (0x2401, 12, 18, false), + (0x1C01, 13, 20, false), + (0x1601, 29, 21, false), + (0x5601, 15, 14, true), + (0x5401, 16, 14, false), + (0x5101, 17, 15, false), + (0x4801, 18, 16, false), + (0x3801, 19, 17, false), + (0x3401, 20, 18, false), + (0x3001, 21, 19, false), + (0x2801, 22, 19, false), + (0x2401, 23, 20, false), + (0x2201, 24, 21, false), + (0x1C01, 25, 22, false), + (0x1801, 26, 23, false), + (0x1601, 27, 24, false), + (0x1401, 28, 25, false), + (0x1201, 29, 26, false), + (0x1101, 30, 27, false), + (0x0AC1, 31, 28, false), + (0x09C1, 32, 29, false), + (0x08A1, 33, 30, false), + (0x0521, 34, 31, false), + (0x0441, 35, 32, false), + (0x02A1, 36, 33, false), + (0x0221, 37, 34, false), + (0x0141, 38, 35, false), + (0x0111, 39, 36, false), + (0x0085, 40, 37, false), + (0x0049, 41, 38, false), + (0x0025, 42, 39, false), + (0x0015, 43, 40, false), + (0x0009, 44, 41, false), + (0x0005, 45, 42, false), + (0x0001, 45, 43, false), + (0x5601, 46, 46, false), +]; + +#[cfg(test)] +thread_local! { + /// Decisions coded on this thread. + pub(crate) static DECISIONS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// A context's adaptive state: the index into [`STATES`] (low 7 bits) and the MPS sense (bit 7). +/// 0, the initial value, is index 0 with MPS 0 (E.2.8). +pub(crate) type Context = u8; + +/// The encoder's registers (E.2.1) and the bytes written so far. +pub(crate) struct MqEncoder { + a: u32, + c: u32, + ct: u32, + /// The output, after the byte before the first one (B at INITENC), which is 0x00 and is + /// dropped by [`MqEncoder::finish`]: a carry can't reach it before the first byte is out. + out: Vec, +} + +impl MqEncoder { + /// INITENC (Figure E.10). The byte before the first is 0x00, not 0xFF, so CT starts at 12. + pub(crate) fn new() -> Self { + MqEncoder { a: 0x8000, c: 0, ct: 12, out: vec![0] } + } + + /// The byte B, the last one written (or the one before the first). + fn b(&self) -> u8 { + self.out.last().copied().unwrap_or(0) + } + + /// ENCODE (Figures E.3–E.7): code decision `d` in context `cx`, updating its state. + pub(crate) fn encode(&mut self, cx: &mut Context, d: bool) { + #[cfg(test)] + DECISIONS.with(|n| n.set(n.get() + 1)); + let (index, mps) = (usize::from(*cx & 0x7F), *cx >> 7 == 1); + let Some(&(qe, nmps, nlps, switch)) = STATES.get(index) else { return }; + self.a -= qe; + if d == mps { + // CODEMPS. + if self.a & 0x8000 != 0 { + self.c += qe; + return; + } + if self.a < qe { + self.a = qe; + } else { + self.c += qe; + } + *cx = nmps | (*cx & 0x80); + } else { + // CODELPS. + if self.a < qe { + self.c += qe; + } else { + self.a = qe; + } + let mps = mps != switch; + *cx = nlps | (u8::from(mps) << 7); + } + self.renorm(); + } + + /// RENORME (Figure E.8). + fn renorm(&mut self) { + loop { + self.a <<= 1; + self.c <<= 1; + self.ct -= 1; + if self.ct == 0 { + self.byte_out(); + } + if self.a & 0x8000 != 0 { + break; + } + } + } + + /// BYTEOUT (Figure E.9), with bit stuffing after 0xFF and carry into the byte before. + fn byte_out(&mut self) { + if self.b() == 0xFF { + self.stuffed(); + } else if self.c < 0x800_0000 { + self.plain(); + } else { + let b = self.b().wrapping_add(1); + if let Some(last) = self.out.last_mut() { + *last = b; + } + if b == 0xFF { + self.c &= 0x7FF_FFFF; + self.stuffed(); + } else { + self.plain(); + } + } + } + + fn plain(&mut self) { + self.out.push(((self.c >> 19) & 0xFF) as u8); + self.c &= 0x7_FFFF; + self.ct = 8; + } + + fn stuffed(&mut self) { + self.out.push(((self.c >> 20) & 0xFF) as u8); + self.c &= 0xF_FFFF; + self.ct = 7; + } + + /// FLUSH (Figures E.11 and E.12): the final bytes, then the 0xFF 0xAC marker. + pub(crate) fn finish(mut self) -> Vec { + let top = self.c + self.a; + self.c |= 0xFFFF; + if self.c >= top { + self.c -= 0x8000; + } + self.c <<= self.ct; + self.byte_out(); + self.c <<= self.ct; + self.byte_out(); + if self.b() != 0xFF { + self.out.push(0xFF); + } + self.out.push(0xAC); + self.out.split_off(1) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Annex H.2's test sequence: 256 decisions in one context give these 30 bytes. + #[test] + fn the_annex_h_test_sequence_encodes_to_its_bytes() { + let decisions = [ + 0x00, 0x02, 0x00, 0x51, 0x00, 0x00, 0x00, 0xC0, 0x03, 0x52, 0x87, 0x2A, 0xAA, 0xAA, 0xAA, 0xAA, 0x82, 0xC0, 0x20, 0x00, 0xFC, 0xD7, 0x9E, + 0xF6, 0xBF, 0x7F, 0xED, 0x90, 0x4F, 0x46, 0xA3, 0xBF, + ]; + let mut mq = MqEncoder::new(); + let mut cx = 0; + for byte in decisions { + for bit in (0..8).rev() { + mq.encode(&mut cx, byte >> bit & 1 == 1); + } + } + let expected = [ + 0x84, 0xC7, 0x3B, 0xFC, 0xE1, 0xA1, 0x43, 0x04, 0x02, 0x20, 0x00, 0x00, 0x41, 0x0D, 0xBB, 0x86, 0xF4, 0x31, 0x7F, 0xFF, 0x88, 0xFF, 0x37, + 0x47, 0x1A, 0xDB, 0x6A, 0xDF, 0xFF, 0xAC, + ]; + assert_eq!(mq.finish(), expected); + } +} diff --git a/crates/stream-codecs/src/predictor.rs b/crates/stream-codecs/src/predictor.rs new file mode 100644 index 000000000..8e76e3fc2 --- /dev/null +++ b/crates/stream-codecs/src/predictor.rs @@ -0,0 +1,356 @@ +//! Predictor functions for FlateDecode and LZWDecode (ISO 32000-2 §7.4.4.4). +//! +//! - Predictor 2: TIFF 6.0 horizontal differencing (Section 14), per component, for +//! 1, 2, 4, 8 and 16 bits per component. +//! - Predictors 10–15: PNG filters (RFC 2083 §6): each row carries its own filter-type +//! byte (0 None, 1 Sub, 2 Up, 3 Average, 4 Paeth), whatever the predictor value is. +//! +//! A final partial row is decoded as far as its data goes. +//! +//! The samples of an image whose rows carry no filter-type byte, although a PNG predictor is +//! declared, are taken as they are and reported as damage ([`decode`]). + +use crate::{CodecError, Failure, Params, Step}; + +struct Layout { + bpc: usize, + colors: usize, + /// Bytes per complete pixel, at least 1 (the PNG `bpp`). + bpp: usize, + /// Bytes per row (without the PNG tag byte). + row: usize, + /// Samples per row (colors × columns). + samples: usize, +} + +fn layout(p: &Params) -> Result { + let bpc = match p.bits_per_component { + b @ (1 | 2 | 4 | 8 | 16) => b as usize, + b => return Err(format!("invalid BitsPerComponent {b}")), + }; + if !(1..=255).contains(&p.colors) { + return Err(format!("invalid Colors {}", p.colors)); + } + if p.columns < 1 { + return Err(format!("invalid Columns {}", p.columns)); + } + let colors = p.colors as usize; + let samples = usize::try_from(p.columns).ok().and_then(|c| c.checked_mul(colors)).ok_or_else(|| format!("Columns {} too large", p.columns))?; + let bits = samples.checked_mul(bpc).ok_or_else(|| format!("Columns {} too large", p.columns))?; + Ok(Layout { bpc, colors, bpp: (colors * bpc).div_ceil(8), row: bits.div_ceil(8), samples }) +} + +/// Undoes the predictor named by `p.predictor` (1 = identity). `rows` is the height of the image +/// `data` is the samples of, if it is: with a PNG predictor, data of exactly `rows` rows without +/// their filter-type bytes ([`untagged`]) is given back as it is, as a [`Failure`] whose partial +/// output is all of it, so that tolerant decoding takes the samples and reports the repair, and +/// strict decoding refuses them. +pub fn decode(p: &Params, data: Vec, name: &'static str, rows: Option) -> Step { + match p.predictor { + 1 => Ok(data), + 2 | 10..=15 => { + let l = layout(p).map_err(|e| Failure::corrupt(name, e, Vec::new()))?; + if p.predictor == 2 { + return Ok(tiff_decode(&l, data)); + } + if let Some(rows) = rows.filter(|&rows| untagged(&l, data.len(), rows)) { + let detail = + format!("PNG predictor {} declared, but its {rows} rows carry no filter-type bytes: read as unpredicted samples", p.predictor); + return Err(Failure::corrupt(name, detail, data)); + } + png_decode(&l, &data, name) + } + other => Err(Failure::corrupt(name, format!("invalid Predictor {other}"), Vec::new())), + } +} + +/// Whether `len` bytes are exactly `rows` rows of `l` without the filter-type byte that starts each +/// PNG-predicted row (ISO 32000-2 §7.4.4.4), where predicted data would be `rows` × (row + 1): a +/// writer that declared the predictor and didn't apply it. Read with the predictor, every row's +/// first sample was taken for a filter type and the image came out a row short and shifted. +/// PDFium, MuPDF and Poppler read such data as the samples it is. +fn untagged(l: &Layout, len: usize, rows: usize) -> bool { + rows > 0 && l.row > 0 && rows.checked_mul(l.row) == Some(len) +} + +/// Applies the predictor named by `p.predictor` (1 = identity; 15 = per-row optimum). +pub fn encode(p: &Params, data: &[u8]) -> Result, CodecError> { + match p.predictor { + 1 => Ok(data.to_vec()), + 2 | 10..=15 => { + let l = layout(p).map_err(CodecError::Unsupported)?; + Ok(if p.predictor == 2 { tiff_encode(&l, data) } else { png_encode(&l, data, p.predictor) }) + } + other => Err(CodecError::Unsupported(format!("Predictor {other}"))), + } +} + +// ---- TIFF predictor 2 ---- + +fn get(row: &[u8], i: usize, bpc: usize) -> u16 { + match bpc { + 16 => u16::from_be_bytes([row[2 * i], row[2 * i + 1]]), + 8 => u16::from(row[i]), + _ => { + let bit = i * bpc; + let shift = 8 - bpc - bit % 8; + u16::from(row[bit / 8] >> shift) & ((1 << bpc) - 1) + } + } +} + +fn set(row: &mut [u8], i: usize, bpc: usize, v: u16) { + match bpc { + 16 => row[2 * i..2 * i + 2].copy_from_slice(&v.to_be_bytes()), + 8 => row[i] = v as u8, + _ => { + let bit = i * bpc; + let shift = 8 - bpc - bit % 8; + let mask = (((1u16 << bpc) - 1) << shift) as u8; + row[bit / 8] = (row[bit / 8] & !mask) | (((v << shift) as u8) & mask); + } + } +} + +/// Complete samples available in a (possibly partial) row. +fn samples_in(l: &Layout, row_len: usize) -> usize { + (row_len * 8 / l.bpc).min(l.samples) +} + +fn tiff_decode(l: &Layout, mut data: Vec) -> Vec { + let mask = if l.bpc == 16 { u16::MAX } else { (1u16 << l.bpc) - 1 }; + for row in data.chunks_mut(l.row) { + for i in l.colors..samples_in(l, row.len()) { + let v = get(row, i, l.bpc).wrapping_add(get(row, i - l.colors, l.bpc)) & mask; + set(row, i, l.bpc, v); + } + } + data +} + +fn tiff_encode(l: &Layout, data: &[u8]) -> Vec { + let mask = if l.bpc == 16 { u16::MAX } else { (1u16 << l.bpc) - 1 }; + let mut out = data.to_vec(); + for row in out.chunks_mut(l.row) { + // Descending, so the left neighbour still holds its original value. + for i in (l.colors..samples_in(l, row.len())).rev() { + let v = get(row, i, l.bpc).wrapping_sub(get(row, i - l.colors, l.bpc)) & mask; + set(row, i, l.bpc, v); + } + } + out +} + +// ---- PNG predictors ---- + +fn paeth(a: u8, b: u8, c: u8) -> u8 { + let p = i16::from(a) + i16::from(b) - i16::from(c); + let (pa, pb, pc) = ((p - i16::from(a)).abs(), (p - i16::from(b)).abs(), (p - i16::from(c)).abs()); + if pa <= pb && pa <= pc { + a + } else if pb <= pc { + b + } else { + c + } +} + +/// The PNG predictor value for byte `i` given the reconstructed current row `cur` +/// (bytes before `i`) and the prior row `up`. +fn predict(tag: u8, cur: &[u8], up: &[u8], i: usize, bpp: usize) -> u8 { + let a = if i >= bpp { cur[i - bpp] } else { 0 }; + let b = up[i]; + let c = if i >= bpp { up[i - bpp] } else { 0 }; + match tag { + 1 => a, + 2 => b, + 3 => ((u16::from(a) + u16::from(b)) / 2) as u8, + 4 => paeth(a, b, c), + _ => 0, + } +} + +fn png_decode(l: &Layout, data: &[u8], name: &'static str) -> Step { + let mut out = Vec::with_capacity(data.len()); + // The prior row: never longer than the input, even for absurd Columns values. + let mut up = vec![0u8; l.row.min(data.len())]; + for chunk in data.chunks(l.row.saturating_add(1)) { + let (tag, src) = (chunk[0], &chunk[1..]); + if tag > 4 { + return Err(Failure::corrupt(name, format!("invalid PNG filter type {tag}"), out)); + } + let start = out.len(); + for (i, &x) in src.iter().enumerate() { + let pred = predict(tag, &out[start..], &up, i, l.bpp); + out.push(x.wrapping_add(pred)); + } + up[..src.len()].copy_from_slice(&out[start..]); + } + Ok(out) +} + +fn png_filter(tag: u8, cur: &[u8], up: &[u8], bpp: usize, dst: &mut Vec) { + dst.clear(); + dst.push(tag); + dst.extend(cur.iter().enumerate().map(|(i, &x)| x.wrapping_sub(predict(tag, cur, up, i, bpp)))); +} + +/// Sum of residuals as signed bytes: the usual heuristic for choosing a PNG filter. +fn cost(filtered: &[u8]) -> u64 { + filtered[1..].iter().map(|&b| u64::from((b as i8).unsigned_abs())).sum() +} + +fn png_encode(l: &Layout, data: &[u8], predictor: i64) -> Vec { + let rows = data.len().div_ceil(l.row.max(1)); + let mut out = Vec::with_capacity(data.len() + rows); + let zeros = vec![0u8; l.row.min(data.len())]; + let (mut best, mut trial) = (Vec::new(), Vec::new()); + let mut up: &[u8] = &zeros; + for cur in data.chunks(l.row) { + let up_row = &up[..cur.len()]; + if predictor == 15 { + png_filter(0, cur, up_row, l.bpp, &mut best); + let mut best_cost = cost(&best); + for tag in 1..=4 { + png_filter(tag, cur, up_row, l.bpp, &mut trial); + let c = cost(&trial); + if c < best_cost { + best_cost = c; + std::mem::swap(&mut best, &mut trial); + } + } + } else { + png_filter((predictor - 10) as u8, cur, up_row, l.bpp, &mut best); + } + out.extend_from_slice(&best); + up = cur; + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn p(predictor: i64, colors: i64, bpc: i64, columns: i64) -> Params { + Params { predictor, colors, bits_per_component: bpc, columns, early_change: 1 } + } + + fn dec(params: &Params, data: &[u8]) -> Result, CodecError> { + decode(params, data.to_vec(), "FlateDecode", None).map_err(|f| f.error) + } + + #[test] + fn png_hand_vectors() { + // 3 columns, 1 color, 8 bpc. Rows: None, Sub, Up, Average, Paeth. + let enc = [ + 0, 10, 20, 30, // None -> 10 20 30 + 1, 5, 1, 1, // Sub -> 5 6 7 + 2, 1, 1, 1, // Up -> 6 7 8 + 3, 10, 10, 10, // Avg -> 10+3=13, 10+(13+7)/2=20, 10+(20+8)/2=24 + 4, 1, 1, + 1, // Paeth: a=0,b=13,c=0 -> p=13 -> b: 14; a=14,b=20,c=13: p=21 -> pa=7,pb=1,pc=8 -> b: 21; + // a=21,b=24,c=20: p=25 -> pa=4,pb=1,pc=5 -> b: 25 + ]; + let out = dec(&p(15, 1, 8, 3), &enc).unwrap(); + assert_eq!(out, [10, 20, 30, 5, 6, 7, 6, 7, 8, 13, 20, 24, 14, 21, 25]); + } + + #[test] + fn paeth_tie_breaks() { + // RFC 2083 order: a, then b, then c. + assert_eq!(paeth(1, 1, 1), 1); + assert_eq!(paeth(10, 20, 15), 15); // p=15: pa=5 pb=5 pc=0 -> c + assert_eq!(paeth(20, 10, 10), 20); // p=20: pa=0 -> a + assert_eq!(paeth(5, 9, 7), 7); // p=7: pa=2 pb=2 pc=0 -> c + } + + #[test] + fn png_multibyte_pixels() { + // colors=3 bpc=16 -> bpp=6; 1 column; Sub on the first pixel uses a=0 throughout. + let enc = [1, 1, 2, 3, 4, 5, 6, 2, 1, 1, 1, 1, 1, 1]; + let out = dec(&p(11, 3, 16, 1), &enc).unwrap(); + assert_eq!(out, [1, 2, 3, 4, 5, 6, 2, 3, 4, 5, 6, 7]); + // colors=3 bpc=8, 2 columns: Sub looks 3 bytes back. + let enc = [1, 1, 2, 3, 1, 1, 1]; + assert_eq!(dec(&p(11, 3, 8, 2), &enc).unwrap(), [1, 2, 3, 2, 3, 4]); + } + + #[test] + fn png_partial_final_row() { + let enc = [0, 1, 2, 3, 4, 2, 1, 1]; + assert_eq!(dec(&p(12, 1, 8, 4), &enc).unwrap(), [1, 2, 3, 4, 2, 3]); + // A lone tag byte at the end yields nothing. + assert_eq!(dec(&p(12, 1, 8, 4), &[0, 1, 2, 3, 4, 2]).unwrap(), [1, 2, 3, 4]); + } + + #[test] + fn png_invalid_tag() { + let r = decode(&p(10, 1, 8, 2), vec![0, 1, 2, 5, 1, 1], "FlateDecode", None); + let f = r.err().unwrap(); + assert!(matches!(f.error, CodecError::Corrupt { .. })); + assert_eq!(f.partial, [1, 2]); + } + + #[test] + fn tiff_8bit() { + // 2 colors, 3 columns: differences per component. + let enc = [10, 20, 1, 2, 3, 4, /* row 2 */ 5, 5, 250, 10, 1, 1]; + assert_eq!(dec(&p(2, 2, 8, 3), &enc).unwrap(), [10, 20, 11, 22, 14, 26, 5, 5, 255, 15, 0, 16]); + } + + #[test] + fn tiff_16bit() { + let enc = [0x01, 0x00, 0xff, 0xff, 0x00, 0x02]; + assert_eq!(dec(&p(2, 1, 16, 3), &enc).unwrap(), [0x01, 0x00, 0x00, 0xff, 0x01, 0x01]); + } + + #[test] + fn tiff_sub_byte() { + // 1 bpc, 1 color, 10 columns: bits 1 0 0 1 1 0 0 0 | 1 1 (pad 000000) + // Cumulative XOR: 1 1 1 0 1 1 1 1 | 0 1 + let enc = [0b1001_1000, 0b1100_0000]; + assert_eq!(dec(&p(2, 1, 1, 10), &enc).unwrap(), [0b1110_1111, 0b0100_0000]); + // 4 bpc, 1 color, 3 columns: samples 5, 15, 3 -> 5, 4, 7 (mod 16). + assert_eq!(dec(&p(2, 1, 4, 3), &[0x5f, 0x30]).unwrap(), [0x54, 0x70]); + // 2 bpc, 2 colors, 2 columns: samples 1 2 3 3 -> 1 2 0 1. + assert_eq!(dec(&p(2, 2, 2, 2), &[0b0110_1111]).unwrap(), [0b0110_0001]); + // Padding bits are left alone. + assert_eq!(dec(&p(2, 1, 4, 1), &[0x5f]).unwrap(), [0x5f]); + } + + #[test] + fn tiff_partial_rows() { + // 16-bit with an odd trailing byte: the lone byte is left as is. + assert_eq!(dec(&p(2, 1, 16, 4), &[0, 1, 0, 1, 9]).unwrap(), [0, 1, 0, 2, 9]); + assert_eq!(dec(&p(2, 1, 8, 4), &[1, 1, 1, 1, 2, 2]).unwrap(), [1, 2, 3, 4, 2, 4]); + } + + #[test] + fn invalid_params() { + for bad in [p(2, 1, 3, 1), p(12, 0, 8, 1), p(12, 1, 8, 0), p(3, 1, 8, 1), p(12, 1, 8, i64::MAX), p(2, 300, 8, 1)] { + assert!(matches!(dec(&bad, &[0, 1, 2]), Err(CodecError::Corrupt { .. })), "{bad:?}"); + assert!(encode(&bad, &[1, 2]).is_err()); + } + // Huge but representable Columns does not allocate a huge row. + assert_eq!(dec(&p(12, 1, 8, 1 << 40), &[2, 7, 8]).unwrap(), [7, 8]); + } + + #[test] + fn encode_choices() { + let data = [1u8, 2, 3, 4, 5, 6]; + let up = encode(&p(12, 1, 8, 3), &data).unwrap(); + assert_eq!(up, [2, 1, 2, 3, 2, 3, 3, 3]); + let paeth = encode(&p(14, 1, 8, 3), &data).unwrap(); + assert_eq!(paeth[0], 4); + assert_eq!(paeth[4], 4); + // Per-row optimum picks Sub for a ramp. + let ramp: Vec = (0..64).collect(); + let opt = encode(&p(15, 1, 8, 64), &ramp).unwrap(); + assert_eq!(opt[0], 1); + for pred in [10, 11, 12, 13, 14, 15] { + let e = encode(&p(pred, 1, 8, 3), &data).unwrap(); + assert_eq!(dec(&p(pred, 1, 8, 3), &e).unwrap(), data); + } + } +} diff --git a/crates/stream-codecs/src/runlength.rs b/crates/stream-codecs/src/runlength.rs new file mode 100644 index 000000000..a6df9e19d --- /dev/null +++ b/crates/stream-codecs/src/runlength.rs @@ -0,0 +1,117 @@ +//! RunLengthDecode (ISO 32000-2 §7.4.5): a length byte L of 0–127 copies the next +//! L + 1 bytes, 129–255 repeats the next byte 257 − L times, 128 is EOD. +//! A missing EOD is tolerated; a truncated run is corrupt. + +use crate::{Failure, Step}; + +const NAME: &str = "RunLengthDecode"; + +pub fn decode(data: &[u8], max: usize) -> Step { + let mut out = Vec::with_capacity(data.len().saturating_mul(2).min(max)); + let mut i = 0; + while let Some(&l) = data.get(i) { + i += 1; + match l { + 128 => break, + 0..=127 => { + let n = usize::from(l) + 1; + let avail = n.min(data.len() - i); + if out.len() + avail > max { + return Err(Failure::limit(max)); + } + out.extend_from_slice(&data[i..i + avail]); + i += avail; + if avail < n { + return Err(Failure::corrupt(NAME, "truncated literal run", out)); + } + } + _ => { + let Some(&b) = data.get(i) else { + return Err(Failure::corrupt(NAME, "truncated repeat run", out)); + }; + i += 1; + let n = 257 - usize::from(l); + if out.len() + n > max { + return Err(Failure::limit(max)); + } + out.resize(out.len() + n, b); + } + } + } + Ok(out) +} + +pub fn encode(data: &[u8]) -> Vec { + let mut out = Vec::with_capacity(data.len() + data.len() / 128 + 2); + let n = data.len(); + let mut i = 0; + while i < n { + let mut run = 1; + while i + run < n && run < 128 && data[i + run] == data[i] { + run += 1; + } + if run >= 2 { + out.push((257 - run) as u8); + out.push(data[i]); + i += run; + continue; + } + // Literal: extend until a repeat begins or 128 bytes. + let start = i; + let mut j = i + 1; + while j < n && j - start < 128 && !(j + 1 < n && data[j] == data[j + 1]) { + j += 1; + } + out.push((j - start - 1) as u8); + out.extend_from_slice(&data[start..j]); + i = j; + } + out.push(128); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::CodecError; + + fn dec(d: &[u8]) -> Result, CodecError> { + decode(d, 10_000).map_err(|f| f.error) + } + + #[test] + fn vectors() { + assert_eq!(dec(&[2, b'a', b'b', b'c', 254, b'x', 128]).unwrap(), b"abcxxx"); + assert_eq!(dec(&[129, 7, 128]).unwrap(), [7; 128]); + assert!(dec(&[127]).is_err()); + assert_eq!(dec(&[0, b'q']).unwrap(), b"q"); // missing EOD + assert_eq!(dec(&[128, 0, b'q']).unwrap(), b""); // data after EOD ignored + assert_eq!(dec(&[]).unwrap(), b""); + } + + #[test] + fn truncated() { + let f = decode(&[4, b'a', b'b'], 100).err().unwrap(); + assert!(matches!(f.error, CodecError::Corrupt { .. })); + assert_eq!(f.partial, b"ab"); + assert!(matches!(dec(&[0, b'a', 200]), Err(CodecError::Corrupt { .. }))); + } + + #[test] + fn limit() { + assert!(matches!(decode(&[129, 0, 129, 0], 200).map_err(|f| f.error), Err(CodecError::LimitExceeded(200)))); + assert!(matches!(decode(&[3, 1, 2, 3, 4], 3).map_err(|f| f.error), Err(CodecError::LimitExceeded(3)))); + assert_eq!(decode(&[129, 0, 129, 0], 256).ok().unwrap().len(), 256); + } + + #[test] + fn encoding() { + assert_eq!(encode(b"abcxxx"), [2, b'a', b'b', b'c', 254, b'x', 128]); + assert_eq!(encode(b""), [128]); + assert_eq!(encode(&[5; 300]), [129, 5, 129, 5, 213, 5, 128]); + let lit: Vec = (0..200u8).collect(); + let e = encode(&lit); + assert_eq!((e[0], e[129]), (127, 71)); + assert_eq!(dec(&e).unwrap(), lit); + } +}