mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-13 02:35:25 +00:00
Add craft-stream-codecs: the codec modules of PdfCraft's stream filters (#953)
LZW, ASCII85, ASCIIHex, RunLength, the TIFF and PNG predictors, bounded CCITT fax decoding (through hayro-ccitt) with Group 4 encoding (through fax), and the lossless JBIG2 encoder with its MQ coder, from pdfcraft-filters at PdfCraft revision 80e26e674d1aa33078e83c8139875047f3289eb6. The crate sits below craft-pdf and craft-tiff: it knows no container. The PDF filter chain, Flate and the bounded JBIG2 decoding entry are not part of it.
This commit is contained in:
Dylan M. Taylor
Generated
+20
-2
@@ -1588,6 +1588,18 @@ dependencies = [
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-stream-codecs"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"fax",
|
||||
"hayro-ccitt",
|
||||
"hayro-jbig2",
|
||||
"proptest",
|
||||
"thiserror 2.0.21",
|
||||
"weezl 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-svg"
|
||||
version = "0.1.0"
|
||||
@@ -2777,7 +2789,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159"
|
||||
dependencies = [
|
||||
"color_quant",
|
||||
"weezl",
|
||||
"weezl 0.1.12",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6731,7 +6743,7 @@ dependencies = [
|
||||
"flate2",
|
||||
"half",
|
||||
"quick-error 2.0.1",
|
||||
"weezl",
|
||||
"weezl 0.1.12",
|
||||
"zune-jpeg",
|
||||
]
|
||||
|
||||
@@ -7590,6 +7602,12 @@ version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
|
||||
|
||||
[[package]]
|
||||
name = "weezl"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4ca08e5ef825b65b056d9efbd95c8750683f0a6d0466d02e96dc2e4e360f3d2"
|
||||
|
||||
[[package]]
|
||||
name = "wgpu"
|
||||
version = "30.0.1"
|
||||
|
||||
@@ -91,6 +91,7 @@ craft-settings = { path = "crates/settings" }
|
||||
craft-silhouette-studio = { path = "crates/silhouette-studio" }
|
||||
craft-svg = { path = "crates/svg" }
|
||||
craft-svg-paths = { path = "crates/svg-paths" }
|
||||
craft-stream-codecs = { path = "crates/stream-codecs" }
|
||||
craft-text = { path = "crates/text" }
|
||||
craft-tga = { path = "crates/tga" }
|
||||
craft-timeline = { path = "crates/timeline" }
|
||||
|
||||
@@ -143,6 +143,11 @@ craft-<name> = { git = "https://github.com/storytold/craft-libs", rev = "<commit
|
||||
origin, to RGBA8, and a run-length writer whose packets end with their scan line. Bounded on
|
||||
hostile input.
|
||||
- [`craft-idml`](crates/idml/README.md): IDML (InDesign Markup Language) packages: the ZIP container read with bounded part sizes and lenient part names, `designmap.xml` with its includes in place, the `<Properties>` form on craft-xml elements, value codecs and the package writer; and InDesign Tagged Text, read and written.
|
||||
- [`craft-stream-codecs`](crates/stream-codecs/README.md): byte-stream and bi-level image codecs that
|
||||
PDF, PostScript and TIFF share: LZW, ASCII85, ASCIIHex, RunLength, TIFF and PNG predictors,
|
||||
CCITT Group 3 and 4 decoding and Group 4 encoding, lossless JBIG2 encoding. Every
|
||||
decoder is bounded and returns its partial output on damage. From PdfCraft; a PDF filter chain
|
||||
is built on it in `craft-pdf`.
|
||||
- [`craft-timeline`](crates/timeline/README.md): editorial timeline interchange: exact media time and SMPTE timecode, import/export reports, media path and `file://` URL handling, Avid Log Exchange (ALE) and CMX 3600 EDL documents, format detection. From FilmCraft.
|
||||
- [`craft-metadata`](crates/metadata/README.md): photo metadata, from LightCraft: EXIF (read and write,
|
||||
maker-note lens and focus point, every tag by name), IPTC-IIM, XMP packets (read, write, and a merge
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
[package]
|
||||
name = "craft-stream-codecs"
|
||||
description = "Byte-stream and bi-level image codecs that PDF, PostScript and TIFF share: LZW, ASCII85, ASCIIHex, RunLength, TIFF and PNG predictors, CCITT Group 3 and 4 decoding and Group 4 encoding, lossless JBIG2 encoding"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
# CCITT Group 4 encoding (ITU-T T.6). MIT.
|
||||
fax = "0.2.7"
|
||||
# CCITT fax decoding, the decoder the vendored PDF reader uses. MIT OR Apache-2.0.
|
||||
hayro-ccitt = { version = "0.3", default-features = false }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
# The JBIG2 decoder the vendored PDF reader uses: reads back what the encoder writes.
|
||||
hayro-jbig2 = { version = "=0.3.0", default-features = false, features = ["std"] }
|
||||
proptest = { workspace = true }
|
||||
weezl = "0.2"
|
||||
@@ -0,0 +1,78 @@
|
||||
# craft-stream-codecs
|
||||
|
||||
Byte-stream and bi-level image codecs that PDF, PostScript and TIFF share. The crate knows no
|
||||
container: a caller hands it bytes and parameters. A PDF `/Filter` chain (stage order, limits
|
||||
across stages, Flate, accounting) belongs to `craft-pdf`, which builds on this crate; a TIFF
|
||||
reader can use the same LZW, predictor, CCITT and JBIG2 code.
|
||||
|
||||
- **Licence:** MIT OR Apache-2.0. Written from ISO 32000-2 §7.4, TIFF 6.0 §14 (predictor 2) and
|
||||
RFC 2083 §6 (PNG filters). CCITT Group 4 encoding (ITU-T T.6) is the `fax` crate's (MIT). CCITT
|
||||
decoding (T.4 and T.6) is `hayro-ccitt`'s (MIT OR Apache-2.0), bounded here: this crate has
|
||||
no CCITT decoder of its own, and no JBIG2 decoder (that is `hayro-jbig2`, which the tests
|
||||
read the encoder's output back with). JBIG2 encoding
|
||||
is written from the public ISO/IEC 14492 Final Committee Draft (WG1 N1359, 1999); Annex H.2
|
||||
checks the arithmetic coder byte for byte.
|
||||
- No `unsafe`. Never panics on input.
|
||||
- From PdfCraft's `pdfcraft-filters` crate (revision `80e26e674d1aa33078e83c8139875047f3289eb6`):
|
||||
its codec modules, moved whole. Its filter chain and Flate stay with the PDF code.
|
||||
|
||||
## API
|
||||
|
||||
```rust
|
||||
pub struct Params { predictor, colors, bits_per_component, columns, early_change } // i64s; Default = 1, 1, 8, 1, 1
|
||||
pub enum CodecError { Corrupt { filter, detail }, LimitExceeded(usize), Unsupported(String), Spent }
|
||||
pub struct Failure { error, partial, discarded } // what a decoder decoded before it failed
|
||||
pub type Step = Result<Vec<u8>, Failure>;
|
||||
pay(work: &mut u64, steps) -> Result<(), CodecError> // an allowance shared across a run of images
|
||||
|
||||
lzw::decode(data, early_change, max) -> Step; lzw::encode(data, early_change) -> Vec<u8>
|
||||
ascii85::decode(data, max) -> Step; ascii85::encode(data) -> Vec<u8>
|
||||
asciihex::decode(data, max) -> Step; asciihex::encode(data) -> Vec<u8>
|
||||
runlength::decode(data, max) -> Step; runlength::encode(data) -> Vec<u8>
|
||||
predictor::decode(¶ms, data, name, rows) -> Step; predictor::encode(¶ms, data) -> Result<Vec<u8>, CodecError>
|
||||
|
||||
pub struct CcittParams { k, columns, rows, end_of_line, encoded_byte_align, end_of_block, black_is_1 }
|
||||
decode_ccitt_within(data, ¶ms, max_rows, work: &mut u64) -> Result<Vec<u8>, CodecError>
|
||||
encode_g4(data, columns, rows) -> Result<Vec<u8>, CodecError>
|
||||
|
||||
encode_jbig2_generic(data, columns, rows) -> Result<Vec<u8>, CodecError>
|
||||
encode_jbig2(data, columns, rows) -> Result<Vec<u8>, CodecError>
|
||||
encode_jbig2_shared(&[(data, columns, rows)]) -> Result<Option<SharedJbig2>, CodecError>
|
||||
```
|
||||
|
||||
`max` bounds a decoder's output and gives `LimitExceeded`. `Corrupt` names the codec in `filter`
|
||||
by its PDF filter name. Bi-level images are rows of 1-bit samples, most significant bit first,
|
||||
each row on a new byte.
|
||||
|
||||
## Behaviour
|
||||
|
||||
| Codec | Decode | Encode |
|
||||
|---|---|---|
|
||||
| LZW | 9–12-bit MSB-first codes, clear (256), end (257), early change on or off; a missing end code or initial clear is tolerated; a full table stops growing | greedy, leading clear, clear when the table fills, end code |
|
||||
| Predictors | 2 = TIFF (1/2/4/8/16 bits per component), 10–15 = PNG with per-row tags 0–4; partial final rows decoded; bad parameters or tags are `Corrupt`; with `rows`, data that is exactly that many rows without tag bytes is taken as the samples and reported as damage | 2, 10 None, 11 Sub, 12 Up, 13 Average, 14 Paeth, 15 per-row optimum |
|
||||
| ASCIIHex | white space ignored, odd digit padded with 0, `>` ends (optional), other characters are `Corrupt` | upper case, newline every 64 digits, `>` |
|
||||
| ASCII85 | `<~` prefix, `z`, white space, `~>` (or bare `~`, or none), final partial group; `z` inside a group, a 1-character final group, overflow or bad characters are `Corrupt` | `z` for zero groups, newline every 75 characters, `~>` |
|
||||
| RunLength | 0–127 literal, 129–255 repeat, 128 ends (optional); a truncated run is `Corrupt` | runs of 2 or more repeat, literals up to 128, 128 |
|
||||
| CCITT fax | Group 3 (1-D and 2-D) and Group 4 through `hayro-ccitt`, as rows of 1-bit samples, black 0 unless `black_is_1`, at most `max_rows` rows (fewer with `rows`), the rows before any damage, up to `MAX_CCITT_COLUMNS` (1,048,576) samples a row; the rows are paid for from `work` before they are decoded | `encode_g4`: Group 4 with an end-of-block mark, 0 bits coded black; up to `MAX_G4_COLUMNS` (32,767) samples a row |
|
||||
| JBIG2 | not here: `hayro-jbig2` | `encode_jbig2_generic`: one immediate lossless generic region (template 0, typical prediction, MQ coder). `encode_jbig2`: the smaller of that and symbol coding (exact copies only, so the stream decodes to the same bits). `encode_jbig2_shared`: a group of images sharing one symbol dictionary (the PDF `/JBIG2Globals` data) |
|
||||
|
||||
The limits of the JBIG2 encoder (symbols per plan, instances per text region, marks, runs) are
|
||||
those `hayro-jbig2` reads back; they are documented on the constants in `src/jbig2/symbols.rs`.
|
||||
|
||||
## Tests
|
||||
|
||||
- Unit tests per codec with hand-computed vectors (the ISO 32000-2 LZW example, PNG and TIFF rows).
|
||||
- LZW is cross-checked against `weezl` (dev-dependency) in both directions for both early-change settings.
|
||||
- `ccitt.rs`: decoding gives back the rows `encode_g4` was given; Group 3 layouts over seeded
|
||||
pictures; Group 4 output decoded by `hayro-ccitt` matches the input bits, with property tests
|
||||
over bitmaps of noise and of runs; any input returns rather than panics.
|
||||
- `mq.rs`: the arithmetic coder reproduces Annex H.2's 30 bytes from its 256 decisions.
|
||||
- `jbig2.rs`, `jbig2/symbols.rs`: output decoded by `hayro-jbig2` matches the input bits for
|
||||
every coding, plan and strip size, with property tests; size and limit refusals.
|
||||
|
||||
## Not here
|
||||
|
||||
A bounded JBIG2 decoding entry (`decode_jbig2_within` in PdfCraft): it needs the caller-owned
|
||||
work allowance PdfCraft added to its `hayro-jbig2` (`Image::decode_within`, `charge_reading`),
|
||||
which the copy vendored here does not have yet. Flate (a container's choice of zlib settings and tolerances), DCT and JPEG 2000, CCITT Group 3
|
||||
encoding.
|
||||
@@ -0,0 +1,156 @@
|
||||
//! ASCII85Decode (ISO 32000-2 §7.4.3): groups of five characters `!`..`u` encode four
|
||||
//! bytes base-85, `z` stands for four zero bytes, `~>` is EOD, white space is ignored and
|
||||
//! a final partial group of n characters (2–5) yields n − 1 bytes. A leading `<~` and a
|
||||
//! missing EOD are tolerated.
|
||||
|
||||
use crate::{Failure, Step, is_pdf_whitespace};
|
||||
|
||||
const NAME: &str = "ASCII85Decode";
|
||||
|
||||
fn push(out: &mut Vec<u8>, bytes: &[u8], max: usize) -> Result<(), Failure> {
|
||||
if out.len() + bytes.len() > max {
|
||||
return Err(Failure::limit(max));
|
||||
}
|
||||
out.extend_from_slice(bytes);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn decode(data: &[u8], max: usize) -> Step {
|
||||
let start = data.iter().position(|&b| !is_pdf_whitespace(b)).unwrap_or(data.len());
|
||||
let body = data[start..].strip_prefix(b"<~").unwrap_or(&data[start..]);
|
||||
let mut out = Vec::with_capacity((body.len() / 5 * 4 + 4).min(max));
|
||||
let mut acc: u64 = 0;
|
||||
let mut n = 0usize;
|
||||
for (i, &b) in body.iter().enumerate() {
|
||||
match b {
|
||||
b'!'..=b'u' => {
|
||||
acc = acc * 85 + u64::from(b - b'!');
|
||||
n += 1;
|
||||
if n == 5 {
|
||||
let Ok(v) = u32::try_from(acc) else {
|
||||
return Err(Failure::corrupt(NAME, format!("group ending at {i} exceeds 2^32"), out));
|
||||
};
|
||||
push(&mut out, &v.to_be_bytes(), max)?;
|
||||
acc = 0;
|
||||
n = 0;
|
||||
}
|
||||
}
|
||||
b'z' if n == 0 => push(&mut out, &[0; 4], max)?,
|
||||
b'z' => return Err(Failure::corrupt(NAME, format!("'z' inside a group at {i}"), out)),
|
||||
// `~` starts the EOD marker; tolerate it without the following `>`.
|
||||
b'~' => break,
|
||||
b if is_pdf_whitespace(b) => {}
|
||||
b => return Err(Failure::corrupt(NAME, format!("invalid character 0x{b:02x} at {i}"), out)),
|
||||
}
|
||||
}
|
||||
match n {
|
||||
0 => {}
|
||||
1 => return Err(Failure::corrupt(NAME, "final group has a single character", out)),
|
||||
_ => {
|
||||
// Pad with `u` (84) to a full group and keep n − 1 bytes.
|
||||
for _ in n..5 {
|
||||
acc = acc * 85 + 84;
|
||||
}
|
||||
let Ok(v) = u32::try_from(acc) else {
|
||||
return Err(Failure::corrupt(NAME, "final group exceeds 2^32", out));
|
||||
};
|
||||
push(&mut out, &v.to_be_bytes()[..n - 1], max)?;
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Encodes with `z` for zero groups, a newline every 75 characters, and a `~>` EOD.
|
||||
pub fn encode(data: &[u8]) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(data.len() * 5 / 4 + data.len() / 60 + 8);
|
||||
let mut col = 0;
|
||||
let mut put = |out: &mut Vec<u8>, s: &[u8]| {
|
||||
for &c in s {
|
||||
if col == 75 {
|
||||
out.push(b'\n');
|
||||
col = 0;
|
||||
}
|
||||
out.push(c);
|
||||
col += 1;
|
||||
}
|
||||
};
|
||||
for chunk in data.chunks(4) {
|
||||
let mut word = [0u8; 4];
|
||||
word[..chunk.len()].copy_from_slice(chunk);
|
||||
let mut v = u32::from_be_bytes(word);
|
||||
if v == 0 && chunk.len() == 4 {
|
||||
put(&mut out, b"z");
|
||||
continue;
|
||||
}
|
||||
let mut digits = [0u8; 5];
|
||||
for d in digits.iter_mut().rev() {
|
||||
*d = (v % 85) as u8 + b'!';
|
||||
v /= 85;
|
||||
}
|
||||
put(&mut out, &digits[..chunk.len() + 1]);
|
||||
}
|
||||
out.extend_from_slice(b"~>");
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::CodecError;
|
||||
|
||||
fn dec(d: &[u8]) -> Result<Vec<u8>, CodecError> {
|
||||
decode(d, 1000).map_err(|f| f.error)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vectors() {
|
||||
// "Man " = 0x4D616E20 = 24·85⁴ + 73·85³ + 80·85² + 78·85 + 61 -> "9jqo^".
|
||||
assert_eq!(dec(b"9jqo^~>").unwrap(), b"Man ");
|
||||
assert_eq!(encode(b"Man "), b"9jqo^~>");
|
||||
// "M" padded to 0x4D000000 -> "9`" (24, 63); decoding pads with 'u'.
|
||||
assert_eq!(encode(b"M"), b"9`~>");
|
||||
assert_eq!(dec(b"9`~>").unwrap(), b"M");
|
||||
assert_eq!(dec(b"z~>").unwrap(), [0; 4]);
|
||||
assert_eq!(encode(&[0; 4]), b"z~>");
|
||||
// A partial zero group is not `z`.
|
||||
assert_eq!(encode(&[0; 3]), b"!!!!~>");
|
||||
assert_eq!(dec(b"!!!!~>").unwrap(), [0; 3]);
|
||||
assert_eq!(dec(b"s8W-!~>").unwrap(), [0xff; 4]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn syntax_tolerance() {
|
||||
assert_eq!(dec(b"<~9jqo^~>").unwrap(), b"Man ");
|
||||
assert_eq!(dec(b" \n<~9j\r\nq o\t^z~>").unwrap(), b"Man \0\0\0\0");
|
||||
assert_eq!(dec(b"9jqo^").unwrap(), b"Man "); // missing EOD
|
||||
assert_eq!(dec(b"9jqo^~").unwrap(), b"Man "); // bare ~
|
||||
assert_eq!(dec(b"9jqo^~>garbage{").unwrap(), b"Man ");
|
||||
assert_eq!(dec(b"~>").unwrap(), b"");
|
||||
assert_eq!(dec(b"").unwrap(), b"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt() {
|
||||
assert!(matches!(dec(b"9jz~>"), Err(CodecError::Corrupt { .. })));
|
||||
assert!(matches!(dec(b"9jqo^9~>"), Err(CodecError::Corrupt { .. })));
|
||||
assert!(matches!(dec(b"uuuuu~>"), Err(CodecError::Corrupt { .. }))); // > 2^32 - 1
|
||||
assert!(matches!(dec(b"9jqo^v~>"), Err(CodecError::Corrupt { .. })));
|
||||
let f = decode(b"9jqo^{", 100).err().unwrap();
|
||||
assert_eq!(f.partial, b"Man ");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limit() {
|
||||
assert!(matches!(decode(b"zz", 7).map_err(|f| f.error), Err(CodecError::LimitExceeded(7))));
|
||||
assert!(matches!(decode(b"z9`", 4).map_err(|f| f.error), Err(CodecError::LimitExceeded(4))));
|
||||
assert_eq!(decode(b"zz", 8).ok().unwrap().len(), 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn line_wrapping() {
|
||||
let data: Vec<u8> = (0..=255u8).collect();
|
||||
let e = encode(&data);
|
||||
assert!(e.split(|&b| b == b'\n').all(|l| l.len() <= 77));
|
||||
assert_eq!(dec(&e).unwrap_or_default().len(), 256);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
//! ASCIIHexDecode (ISO 32000-2 §7.4.2): pairs of hex digits, white space ignored,
|
||||
//! `>` is EOD, an odd final digit is padded with 0. A missing EOD is tolerated.
|
||||
|
||||
use crate::{Failure, Step, is_pdf_whitespace};
|
||||
|
||||
const NAME: &str = "ASCIIHexDecode";
|
||||
|
||||
fn nibble(b: u8) -> Option<u8> {
|
||||
match b {
|
||||
b'0'..=b'9' => Some(b - b'0'),
|
||||
b'a'..=b'f' => Some(b - b'a' + 10),
|
||||
b'A'..=b'F' => Some(b - b'A' + 10),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decode(data: &[u8], max: usize) -> Step {
|
||||
let mut out = Vec::with_capacity((data.len() / 2).min(max));
|
||||
let mut hi: Option<u8> = None;
|
||||
for (i, &b) in data.iter().enumerate() {
|
||||
if is_pdf_whitespace(b) {
|
||||
continue;
|
||||
}
|
||||
if b == b'>' {
|
||||
break;
|
||||
}
|
||||
let Some(n) = nibble(b) else {
|
||||
return Err(Failure::corrupt(NAME, format!("invalid character 0x{b:02x} at offset {i}"), out));
|
||||
};
|
||||
match hi.take() {
|
||||
None => hi = Some(n),
|
||||
Some(h) => {
|
||||
if out.len() == max {
|
||||
return Err(Failure::limit(max));
|
||||
}
|
||||
out.push(h << 4 | n);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(h) = hi {
|
||||
if out.len() == max {
|
||||
return Err(Failure::limit(max));
|
||||
}
|
||||
out.push(h << 4);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Upper-case hex, a newline every 64 digits, terminated by `>`.
|
||||
pub fn encode(data: &[u8]) -> Vec<u8> {
|
||||
const HEX: &[u8; 16] = b"0123456789ABCDEF";
|
||||
let mut out = Vec::with_capacity(data.len() * 2 + data.len() / 32 + 1);
|
||||
for (i, &b) in data.iter().enumerate() {
|
||||
if i > 0 && i % 32 == 0 {
|
||||
out.push(b'\n');
|
||||
}
|
||||
out.push(HEX[usize::from(b >> 4)]);
|
||||
out.push(HEX[usize::from(b & 15)]);
|
||||
}
|
||||
out.push(b'>');
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::CodecError;
|
||||
|
||||
fn dec(d: &[u8]) -> Result<Vec<u8>, CodecError> {
|
||||
decode(d, 1000).map_err(|f| f.error)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vectors() {
|
||||
assert_eq!(dec(b"48656c6C6F>").unwrap(), b"Hello");
|
||||
assert_eq!(dec(b" 4 8\n6\r5\t6\x0c c\x006c 6f >").unwrap(), b"Hello");
|
||||
assert_eq!(dec(b"901FA>").unwrap(), [0x90, 0x1f, 0xa0]);
|
||||
assert_eq!(dec(b"901FA").unwrap(), [0x90, 0x1f, 0xa0]); // missing EOD
|
||||
assert_eq!(dec(b"41>zz garbage after EOD").unwrap(), b"A");
|
||||
assert_eq!(dec(b">").unwrap(), b"");
|
||||
assert_eq!(dec(b"").unwrap(), b"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_char() {
|
||||
let f = decode(b"4142G43>", 1000).err().unwrap();
|
||||
assert!(matches!(f.error, CodecError::Corrupt { .. }));
|
||||
assert_eq!(f.partial, b"AB");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limit() {
|
||||
assert!(matches!(decode(b"414243", 2).map_err(|f| f.error), Err(CodecError::LimitExceeded(2))));
|
||||
assert!(matches!(decode(b"41424", 2).map_err(|f| f.error), Err(CodecError::LimitExceeded(2))));
|
||||
assert_eq!(decode(b"4142", 2).ok().unwrap(), b"AB");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoding() {
|
||||
assert_eq!(encode(b"Hello"), b"48656C6C6F>");
|
||||
assert_eq!(encode(b""), b">");
|
||||
let long = encode(&[0xab; 40]);
|
||||
assert_eq!(long[64], b'\n');
|
||||
assert_eq!(dec(&long).unwrap(), [0xab; 40]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,546 @@
|
||||
//! CCITT fax for `CCITTFaxDecode` (ISO 32000-2 §7.4.6): Group 4 (ITU-T T.6) encoding, with `/K -1`,
|
||||
//! and decoding of Group 3 (T.4) and Group 4. The coding is the `fax` crate's and the decoding the
|
||||
//! `hayro-ccitt` crate's; this module checks what they are given and bounds what they give.
|
||||
|
||||
use fax::encoder::Encoder;
|
||||
use fax::{Color, VecWriter};
|
||||
|
||||
use crate::CodecError;
|
||||
|
||||
/// The widest row [`encode_g4`] encodes. The encoder counts the positions in a row in 16 bits
|
||||
/// and adds two of them, so a row stays within half that range.
|
||||
pub const MAX_G4_COLUMNS: u32 = 32_767;
|
||||
|
||||
/// `rows` rows of `columns` 1-bit samples, most significant bit first, each row starting on a new
|
||||
/// byte (as a PDF image's samples are packed), encoded as CCITT Group 4 with an end-of-block mark.
|
||||
///
|
||||
/// A 0 bit is coded black and a 1 bit white, so the stream decodes to the same bits with
|
||||
/// `/BlackIs1 false`, the default: an image keeps its colour space and decode array. Bytes after
|
||||
/// the last row are ignored, as are the bits that pad a row to a whole byte.
|
||||
///
|
||||
/// Errors: no rows or columns, or rows wider than [`MAX_G4_COLUMNS`] ([`CodecError::Unsupported`]);
|
||||
/// fewer bytes than the rows need ([`CodecError::Corrupt`]).
|
||||
pub fn encode_g4(data: &[u8], columns: u32, rows: u32) -> Result<Vec<u8>, CodecError> {
|
||||
let width = match u16::try_from(columns) {
|
||||
Ok(w) if w > 0 && columns <= MAX_G4_COLUMNS => w,
|
||||
_ => return Err(CodecError::Unsupported(format!("CCITT Group 4 encoding of rows {columns} samples wide"))),
|
||||
};
|
||||
if rows == 0 {
|
||||
return Err(CodecError::Unsupported("CCITT Group 4 encoding of an image without rows".into()));
|
||||
}
|
||||
let stride = usize::from(width).div_ceil(8);
|
||||
let samples = usize::try_from(rows).ok().and_then(|r| r.checked_mul(stride)).and_then(|n| data.get(..n)).ok_or_else(|| CodecError::Corrupt {
|
||||
filter: "CCITTFaxDecode",
|
||||
detail: format!("{} bytes for {rows} rows of {columns} samples", data.len()),
|
||||
})?;
|
||||
let mut encoder = Encoder::new(VecWriter::new());
|
||||
for row in samples.chunks_exact(stride) {
|
||||
let pels = (0..usize::from(width)).map(|x| {
|
||||
let white = row.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0);
|
||||
if white { Color::White } else { Color::Black }
|
||||
});
|
||||
// `VecWriter` writes to memory: its error type has no values.
|
||||
if let Err(never) = encoder.encode_line(pels, width) {
|
||||
match never {}
|
||||
}
|
||||
}
|
||||
match encoder.finish() {
|
||||
Ok(writer) => Ok(writer.finish()),
|
||||
Err(never) => match never {},
|
||||
}
|
||||
}
|
||||
|
||||
/// The widest row [`decode_ccitt_within`] decodes: its row buffer stays within 128 KiB.
|
||||
pub const MAX_CCITT_COLUMNS: u32 = 1 << 20;
|
||||
|
||||
/// A `CCITTFaxDecode` stream's `/DecodeParms` (ISO 32000-2 Table 11). [`Default`] gives the
|
||||
/// table's defaults.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct CcittParams {
|
||||
/// `/K`: negative for Group 4, 0 for Group 3 one-dimensional, positive for Group 3 mixed.
|
||||
pub k: i64,
|
||||
pub columns: i64,
|
||||
/// `/Rows`: 0 when not given.
|
||||
pub rows: i64,
|
||||
pub end_of_line: bool,
|
||||
pub encoded_byte_align: bool,
|
||||
pub end_of_block: bool,
|
||||
pub black_is_1: bool,
|
||||
}
|
||||
|
||||
impl Default for CcittParams {
|
||||
fn default() -> Self {
|
||||
CcittParams { k: 0, columns: 1728, rows: 0, end_of_line: false, encoded_byte_align: false, end_of_block: true, black_is_1: false }
|
||||
}
|
||||
}
|
||||
|
||||
/// What [`decode_ccitt_within`] decodes once it is paid for.
|
||||
fn decode_ccitt(data: &[u8], p: &CcittParams, max_rows: u32) -> Result<Vec<u8>, CodecError> {
|
||||
let columns = u32::try_from(p.columns)
|
||||
.ok()
|
||||
.filter(|c| (1..=MAX_CCITT_COLUMNS).contains(c))
|
||||
.ok_or_else(|| CodecError::Unsupported(format!("CCITT fax rows {} samples wide", p.columns)))?;
|
||||
let rows = u32::try_from(p.rows).ok().filter(|r| *r > 0).map_or(max_rows, |r| r.min(max_rows));
|
||||
if rows == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let settings = hayro_ccitt::DecodeSettings {
|
||||
columns,
|
||||
rows,
|
||||
end_of_block: p.end_of_block,
|
||||
end_of_line: p.end_of_line,
|
||||
// Group 3 rows on bytes may have end-of-line codes whatever `end_of_line` says (the filter
|
||||
// always accepts them), with the fill starting each code on a byte or ending it on one (T.4's
|
||||
// fill, as TIFF writes it). Read unaligned, the codes mark the rows either way; without
|
||||
// codes, the rows are read on bytes. A reading that comes up short isn't tried the other
|
||||
// way: data cut short would then read as other rows.
|
||||
rows_are_byte_aligned: p.encoded_byte_align && (p.k < 0 || !starts_with_end_of_line(data)),
|
||||
encoding: match p.k {
|
||||
..0 => hayro_ccitt::EncodingMode::Group4,
|
||||
0 => hayro_ccitt::EncodingMode::Group3_1D,
|
||||
k => hayro_ccitt::EncodingMode::Group3_2D { k: u32::try_from(k).unwrap_or(u32::MAX) },
|
||||
},
|
||||
invert_black: false,
|
||||
};
|
||||
// u32 to usize: wasm32 and up have at least 32-bit usize.
|
||||
let mut out = Rows::new(columns as usize, rows as usize, p.black_is_1);
|
||||
// The rows decoded before damaged data are kept.
|
||||
let _ = hayro_ccitt::decode(data, &mut out, &mut hayro_ccitt::DecoderContext::new(settings));
|
||||
Ok(out.out)
|
||||
}
|
||||
|
||||
/// The steps of reading a byte of fax code ([`decode_ccitt_within`]): noise takes 114 ns a byte of
|
||||
/// rows and about two bytes of code to a byte of rows, so that and a step a byte of rows is about
|
||||
/// what a step of JBIG2 decoding takes.
|
||||
const CODE_WORK: u64 = 4;
|
||||
|
||||
/// CCITT fax `data` decoded as `CCITTFaxDecode` gives it: rows of `columns` 1-bit samples, most
|
||||
/// significant bit first, each starting on a new byte, with black as 0 (as 1 with `black_is_1`).
|
||||
///
|
||||
/// At most `max_rows` rows are decoded (fewer when `rows` says so), so the output is bounded by
|
||||
/// the caller. Damaged data gives the rows decoded before the damage, as viewers show them: a
|
||||
/// caller that needs a number of rows checks it got them.
|
||||
///
|
||||
/// Group 3 rows on bytes (`k` from 0, `encoded_byte_align`) are read by their end-of-line codes,
|
||||
/// whether the fill starts each code on a byte or ends it on one, when `data` starts with one (T.4
|
||||
/// sends one before the first row when it sends them at all), and on bytes otherwise, whatever
|
||||
/// `end_of_line` says. Group 4 is read as its parameters say. Data with codes but none before the
|
||||
/// first row is read on bytes, as hayro's renderer draws it; with `k` 0 and the fill ending each
|
||||
/// code on a byte, that may be other rows than were coded.
|
||||
///
|
||||
/// Errors: `columns` that aren't from 1 to [`MAX_CCITT_COLUMNS`] ([`CodecError::Unsupported`]).
|
||||
///
|
||||
/// Decoding spends from `work`, what decoding may still cost across a run of images, and there is
|
||||
/// no way in without it: a
|
||||
/// byte of rows a step, whether or not the data holds them, and [`CODE_WORK`] a byte of `data`.
|
||||
/// A blank row takes a bit of code, so the rows are what a few bytes can ask for. It is all paid
|
||||
/// before anything is decoded, and what `work` can't pay for spends all of it.
|
||||
pub fn decode_ccitt_within(data: &[u8], p: &CcittParams, max_rows: u32, work: &mut u64) -> Result<Vec<u8>, CodecError> {
|
||||
let rows = u32::try_from(p.rows).ok().filter(|r| *r > 0).map_or(max_rows, |r| r.min(max_rows));
|
||||
let stride = u64::try_from(p.columns).unwrap_or(0).div_ceil(8);
|
||||
crate::pay(work, stride.saturating_mul(u64::from(rows)).saturating_add((data.len() as u64).saturating_mul(CODE_WORK)))?;
|
||||
decode_ccitt(data, p, max_rows)
|
||||
}
|
||||
|
||||
/// Whether CCITT fax `data` starts with an end-of-line code: a 1 after 11 zeros or more, fill
|
||||
/// included. T.4 sends one before the first row when it sends them at all, and no other code starts
|
||||
/// with more than eight zeros.
|
||||
fn starts_with_end_of_line(data: &[u8]) -> bool {
|
||||
let Some((i, b)) = data.iter().enumerate().find(|(_, b)| **b != 0) else { return false };
|
||||
i.saturating_mul(8).saturating_add(b.leading_zeros() as usize) >= 11
|
||||
}
|
||||
|
||||
/// Rows of 1-bit samples filled from a decoder's pixels: `columns` a row, each row on a new byte, at
|
||||
/// most `rows` rows. Pixels past a row's end and rows past the last are dropped.
|
||||
pub(crate) struct Rows {
|
||||
pub(crate) out: Vec<u8>,
|
||||
row: Vec<u8>,
|
||||
x: usize,
|
||||
columns: usize,
|
||||
rows: usize,
|
||||
done: usize,
|
||||
black_is_1: bool,
|
||||
}
|
||||
|
||||
impl Rows {
|
||||
pub(crate) fn new(columns: usize, rows: usize, black_is_1: bool) -> Self {
|
||||
Rows { out: Vec::new(), row: vec![0; columns.div_ceil(8)], x: 0, columns, rows, done: 0, black_is_1 }
|
||||
}
|
||||
|
||||
/// `count` pixels, white or black.
|
||||
pub(crate) fn push(&mut self, white: bool, count: usize) {
|
||||
let end = self.x.saturating_add(count);
|
||||
if white != self.black_is_1 {
|
||||
for x in self.x..end.min(self.columns) {
|
||||
if let Some(b) = self.row.get_mut(x / 8) {
|
||||
*b |= 0x80 >> (x % 8);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.x = end;
|
||||
}
|
||||
|
||||
/// The row filled so far kept, if fewer than `rows` are, and the next one begun.
|
||||
pub(crate) fn end_row(&mut self) {
|
||||
if self.done < self.rows {
|
||||
self.out.extend_from_slice(&self.row);
|
||||
self.done += 1;
|
||||
}
|
||||
self.row.fill(0);
|
||||
self.x = 0;
|
||||
}
|
||||
}
|
||||
|
||||
impl hayro_ccitt::Decoder for Rows {
|
||||
fn push_pixel(&mut self, white: bool) {
|
||||
self.push(white, 1);
|
||||
}
|
||||
fn push_pixel_chunk(&mut self, white: bool, chunks: u32) {
|
||||
self.push(white, (chunks as usize).saturating_mul(8));
|
||||
}
|
||||
fn next_line(&mut self) {
|
||||
self.end_row();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A run pays for the rows an image asks for and the code it reads, before any of it is
|
||||
/// decoded: rows the data doesn't hold are paid for too, as a few bytes can ask for any number.
|
||||
#[test]
|
||||
fn a_run_of_images_pays_for_the_rows_it_asks_for() {
|
||||
let (w, h) = (4096u32, 64u32);
|
||||
let stride = (w as usize).div_ceil(8);
|
||||
let data = encode_g4(&vec![0xFF; stride * h as usize], w, h).unwrap();
|
||||
let p = CcittParams { k: -1, columns: i64::from(w), ..CcittParams::default() };
|
||||
let whole = decode_ccitt(&data, &p, h).unwrap();
|
||||
let cost = (whole.len() + 4 * data.len()) as u64;
|
||||
let mut work = cost;
|
||||
assert_eq!(decode_ccitt_within(&data, &p, h, &mut work).unwrap(), whole, "exactly enough");
|
||||
assert_eq!(work, 0);
|
||||
let mut work = cost + 7;
|
||||
decode_ccitt_within(&data, &p, h, &mut work).unwrap();
|
||||
assert_eq!(work, 7, "only what it took");
|
||||
let mut work = cost - 1;
|
||||
let refused = decode_ccitt_within(&data, &p, h, &mut work).unwrap_err();
|
||||
assert!(matches!(refused, CodecError::Spent), "a step short: {refused}");
|
||||
assert_eq!(work, 0, "spends all that was left");
|
||||
// Nothing left: not read at all, so bad columns aren't reported as bad columns.
|
||||
let bad = CcittParams { columns: 0, ..p };
|
||||
assert!(decode_ccitt_within(&data, &bad, h, &mut 0).is_err_and(|e| matches!(e, CodecError::Spent)));
|
||||
// No code at all, for the same rows.
|
||||
let mut work = u64::MAX;
|
||||
decode_ccitt_within(&[], &p, h, &mut work).unwrap();
|
||||
assert_eq!(u64::MAX - work, whole.len() as u64);
|
||||
// `/Rows` below what is asked for, and no rows.
|
||||
let mut work = u64::MAX;
|
||||
decode_ccitt_within(&data, &CcittParams { rows: 3, ..p }, h, &mut work).unwrap();
|
||||
assert_eq!(u64::MAX - work, (3 * stride + 4 * data.len()) as u64);
|
||||
let mut work = u64::MAX;
|
||||
decode_ccitt_within(&data, &p, 0, &mut work).unwrap();
|
||||
assert_eq!(u64::MAX - work, 4 * data.len() as u64);
|
||||
}
|
||||
|
||||
/// The rows decoded by an independent decoder (`hayro-ccitt`), as 1 for white and 0 for black.
|
||||
pub(crate) fn decode(data: &[u8], columns: u32, rows: u32) -> Vec<Vec<bool>> {
|
||||
struct Rows(Vec<Vec<bool>>, Vec<bool>, usize);
|
||||
impl hayro_ccitt::Decoder for Rows {
|
||||
fn push_pixel(&mut self, white: bool) {
|
||||
self.1.push(white);
|
||||
}
|
||||
fn push_pixel_chunk(&mut self, white: bool, chunks: u32) {
|
||||
self.1.extend(std::iter::repeat_n(white, chunks as usize * 8));
|
||||
}
|
||||
fn next_line(&mut self) {
|
||||
let mut row = std::mem::take(&mut self.1);
|
||||
row.truncate(self.2);
|
||||
self.0.push(row);
|
||||
}
|
||||
}
|
||||
let settings = hayro_ccitt::DecodeSettings {
|
||||
columns,
|
||||
rows,
|
||||
end_of_block: true,
|
||||
end_of_line: false,
|
||||
rows_are_byte_aligned: false,
|
||||
encoding: hayro_ccitt::EncodingMode::Group4,
|
||||
invert_black: false,
|
||||
};
|
||||
let mut out = Rows(Vec::new(), Vec::new(), columns as usize);
|
||||
hayro_ccitt::decode(data, &mut out, &mut hayro_ccitt::DecoderContext::new(settings)).unwrap();
|
||||
out.0
|
||||
}
|
||||
|
||||
/// The rows of packed samples `data`, as 1 for a 1 bit.
|
||||
pub(crate) fn unpack(data: &[u8], columns: u32, rows: u32) -> Vec<Vec<bool>> {
|
||||
let stride = (columns as usize).div_ceil(8);
|
||||
(0..rows as usize).map(|y| (0..columns as usize).map(|x| data[y * stride + x / 8] & (0x80 >> (x % 8)) != 0).collect()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_page_of_text_like_marks_round_trips_and_shrinks() {
|
||||
let (w, h) = (1700u32, 220u32);
|
||||
let stride = (w as usize).div_ceil(8);
|
||||
let mut data = vec![0xFF; stride * h as usize];
|
||||
// Short black strokes in lines, like glyphs on a white page.
|
||||
for y in (20..h as usize).step_by(40) {
|
||||
for dy in 0..12 {
|
||||
for x in (100..1600).step_by(9) {
|
||||
if (x / 9 + dy) % 5 != 0 {
|
||||
data[(y + dy) * stride + x / 8] &= !(0x80 >> (x % 8));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let g4 = encode_g4(&data, w, h).unwrap();
|
||||
assert_eq!(decode(&g4, w, h), unpack(&data, w, h));
|
||||
assert!(g4.len() * 4 < data.len(), "{} bytes from {}", g4.len(), data.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn padding_bits_and_trailing_bytes_are_ignored() {
|
||||
// 3 columns: the last 5 bits of each byte pad the row; set them to anything.
|
||||
let data = [0b1010_1111, 0b0101_0000, 0b1110_1010, 0xAB, 0xCD];
|
||||
let g4 = encode_g4(&data, 3, 3).unwrap();
|
||||
assert_eq!(decode(&g4, 3, 3), [[true, false, true], [false, true, false], [true, true, true]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_it_cannot_encode_is_an_error() {
|
||||
assert!(matches!(encode_g4(&[0; 8], 0, 1), Err(CodecError::Unsupported(_))));
|
||||
assert!(matches!(encode_g4(&[0; 8], 8, 0), Err(CodecError::Unsupported(_))));
|
||||
assert!(matches!(encode_g4(&vec![0; 4097], MAX_G4_COLUMNS + 1, 1), Err(CodecError::Unsupported(_))));
|
||||
assert!(matches!(encode_g4(&[0; 3], 16, 2), Err(CodecError::Corrupt { .. })), "two rows of 16 need 4 bytes");
|
||||
assert!(matches!(encode_g4(&[], 1, u32::MAX), Err(CodecError::Corrupt { .. })));
|
||||
let widest = vec![0x55; (MAX_G4_COLUMNS as usize).div_ceil(8) * 2];
|
||||
let g4 = encode_g4(&widest, MAX_G4_COLUMNS, 2).unwrap();
|
||||
assert_eq!(decode(&g4, MAX_G4_COLUMNS, 2), unpack(&widest, MAX_G4_COLUMNS, 2));
|
||||
}
|
||||
|
||||
/// Rows of `columns` samples built from `runs`: alternating white and black runs one longer
|
||||
/// than those values (so never empty, even when shrinking makes them all 0), repeated until the
|
||||
/// rows are full; or, with `noise`, the bytes of `runs` as they are.
|
||||
fn bitmap(columns: u32, rows: u32, runs: &[u8], noise: bool) -> Vec<u8> {
|
||||
let stride = (columns as usize).div_ceil(8);
|
||||
let mut data = vec![0u8; stride * rows as usize];
|
||||
if runs.is_empty() {
|
||||
return data;
|
||||
}
|
||||
if noise {
|
||||
data.iter_mut().zip(runs.iter().cycle()).for_each(|(d, r)| *d = *r);
|
||||
return data;
|
||||
}
|
||||
let (mut white, mut left, mut next) = (true, 1 + usize::from(runs[0]), 1);
|
||||
for y in 0..rows as usize {
|
||||
for x in 0..columns as usize {
|
||||
if left == 0 {
|
||||
white = !white;
|
||||
left = 1 + usize::from(runs[next % runs.len()]);
|
||||
next += 1;
|
||||
}
|
||||
if white {
|
||||
data[y * stride + x / 8] |= 0x80 >> (x % 8);
|
||||
}
|
||||
left -= 1;
|
||||
}
|
||||
}
|
||||
data
|
||||
}
|
||||
|
||||
fn g4(columns: u32) -> CcittParams {
|
||||
CcittParams { k: -1, columns: i64::from(columns), ..CcittParams::default() }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoding_gives_the_rows_back() {
|
||||
// 3 columns, padded with anything: the decoded padding is 0.
|
||||
let data = [0b1010_1111, 0b0101_0000, 0b1110_1010];
|
||||
let coded = encode_g4(&data, 3, 3).unwrap();
|
||||
assert_eq!(decode_ccitt(&coded, &g4(3), 3).unwrap(), [0b1010_0000, 0b0100_0000, 0b1110_0000]);
|
||||
// /BlackIs1 gives black as 1.
|
||||
let p = CcittParams { black_is_1: true, ..g4(3) };
|
||||
assert_eq!(decode_ccitt(&coded, &p, 3).unwrap(), [0b0100_0000, 0b1010_0000, 0b0000_0000]);
|
||||
// At most /Rows rows, and at most max_rows.
|
||||
assert_eq!(decode_ccitt(&coded, &CcittParams { rows: 2, ..g4(3) }, 3).unwrap().len(), 2);
|
||||
assert_eq!(decode_ccitt(&coded, &g4(3), 1).unwrap().len(), 1);
|
||||
assert_eq!(decode_ccitt(&coded, &CcittParams { rows: 3, ..g4(3) }, 1).unwrap().len(), 1, "max_rows bounds /Rows");
|
||||
assert_eq!(decode_ccitt(&coded, &g4(3), 0).unwrap(), [0u8; 0]);
|
||||
// Damaged data gives the rows before the damage; none here.
|
||||
assert!(decode_ccitt(&coded[..1], &g4(3), 3).unwrap().len() < 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_it_cannot_decode_is_an_error() {
|
||||
for columns in [0, -1, i64::from(MAX_CCITT_COLUMNS) + 1, i64::MAX] {
|
||||
let p = CcittParams { columns, ..CcittParams::default() };
|
||||
assert!(matches!(decode_ccitt(&[0; 8], &p, 4), Err(CodecError::Unsupported(_))), "{columns}");
|
||||
}
|
||||
assert_eq!(CcittParams::default().columns, 1728, "Table 11");
|
||||
}
|
||||
|
||||
/// Group 3 data: rows of run lengths below 64, white first, coded one-dimensionally with T.4's
|
||||
/// terminating codes (ITU-T T.4 Tables 2 and 3). Each row starts with an end-of-line code with
|
||||
/// `eol`, then with `tag` a 1 bit (the next row one-dimensional, as `k` above 0 has it). Fill
|
||||
/// bits (0s) go where `fill` says.
|
||||
fn group_3(rows: &[Vec<u32>], eol: bool, tag: bool, fill: Fill) -> Vec<u8> {
|
||||
// The codes for runs 0 to 63, in order.
|
||||
const WHITE: &str = concat!(
|
||||
"00110101 000111 0111 1000 1011 1100 1110 1111 10011 10100 00111 01000 001000 000011 110100 110101 101010 101011 ",
|
||||
"0100111 0001100 0001000 0010111 0000011 0000100 0101000 0101011 0010011 0100100 0011000 00000010 00000011 00011010 ",
|
||||
"00011011 00010010 00010011 00010100 00010101 00010110 00010111 00101000 00101001 00101010 00101011 00101100 00101101 ",
|
||||
"00000100 00000101 00001010 00001011 01010010 01010011 01010100 01010101 00100100 00100101 01011000 01011001 01011010 ",
|
||||
"01011011 01001010 01001011 00110010 00110011 00110100",
|
||||
);
|
||||
const BLACK: &str = concat!(
|
||||
"0000110111 010 11 10 011 0011 0010 00011 000101 000100 0000100 0000101 0000111 00000100 00000111 000011000 0000010111 ",
|
||||
"0000011000 0000001000 00001100111 00001101000 00001101100 00000110111 00000101000 00000010111 00000011000 ",
|
||||
"000011001010 000011001011 000011001100 000011001101 000001101000 000001101001 000001101010 000001101011 000011010010 ",
|
||||
"000011010011 000011010100 000011010101 000011010110 000011010111 000001101100 000001101101 000011011010 000011011011 ",
|
||||
"000001010100 000001010101 000001010110 000001010111 000001100100 000001100101 000001010010 000001010011 000000100100 ",
|
||||
"000000110111 000000111000 000000100111 000000101000 000001011000 000001011001 000000101011 000000101100 000001011010 ",
|
||||
"000001100110 000001100111",
|
||||
);
|
||||
let code = |white: bool, run: u32| (if white { WHITE } else { BLACK }).split(' ').nth(run as usize).expect("a run below 64");
|
||||
let mut bits = String::new();
|
||||
for row in rows {
|
||||
// The end-of-line code is 12 bits.
|
||||
let ends_code = if fill == Fill::CodeEnd && eol { 12 } else { 0 };
|
||||
while fill != Fill::None && !(bits.len() + ends_code).is_multiple_of(8) {
|
||||
bits.push('0');
|
||||
}
|
||||
if eol {
|
||||
bits.push_str("000000000001");
|
||||
}
|
||||
if tag {
|
||||
bits.push('1');
|
||||
}
|
||||
row.iter().enumerate().for_each(|(i, run)| bits.push_str(code(i % 2 == 0, *run)));
|
||||
}
|
||||
bits.as_bytes().chunks(8).map(|c| c.iter().enumerate().fold(0u8, |b, (i, d)| b | (u8::from(*d == b'1') << (7 - i)))).collect()
|
||||
}
|
||||
|
||||
/// Where [`group_3`] puts fill bits: none; so each row, its end-of-line code first, starts on a
|
||||
/// byte; or so each end-of-line code ends on one (T.4's fill, as TIFF writes it).
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
enum Fill {
|
||||
None,
|
||||
RowStart,
|
||||
CodeEnd,
|
||||
}
|
||||
|
||||
/// `rows` text-like rows of `columns` samples as runs, white first, from `seed`: long white runs
|
||||
/// and short black ones; a row may start black.
|
||||
fn text_like(seed: u64, columns: u32, rows: u32) -> Vec<Vec<u32>> {
|
||||
let mut seed = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1;
|
||||
let mut random = |n: u64| {
|
||||
seed ^= seed << 13;
|
||||
seed ^= seed >> 7;
|
||||
seed ^= seed << 17;
|
||||
(seed % n) as u32
|
||||
};
|
||||
(0..rows)
|
||||
.map(|_| {
|
||||
let (mut runs, mut left) = (Vec::new(), columns);
|
||||
while left > 0 {
|
||||
let run = match runs.len() % 2 {
|
||||
0 if runs.is_empty() => random(31),
|
||||
0 => 1 + random(30),
|
||||
_ => 1 + random(5),
|
||||
};
|
||||
runs.push(run.min(left));
|
||||
left -= run.min(left);
|
||||
}
|
||||
runs
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Rows of runs, white first, packed as `decode_ccitt` gives them (white 1).
|
||||
fn pack(runs: &[Vec<u32>], columns: u32) -> Vec<u8> {
|
||||
let stride = (columns as usize).div_ceil(8);
|
||||
let mut out = vec![0u8; stride * runs.len()];
|
||||
for (y, row) in runs.iter().enumerate() {
|
||||
let mut x = 0;
|
||||
for (i, run) in row.iter().enumerate() {
|
||||
if i % 2 == 0 {
|
||||
(x..x + *run as usize).for_each(|x| out[y * stride + x / 8] |= 0x80 >> (x % 8));
|
||||
}
|
||||
x += *run as usize;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Group 3 rows on bytes in each layout, over many pictures, as how a misreading comes out
|
||||
/// depends on the picture: no codes (as TIFF's Modified Huffman has it), and `end_of_line`
|
||||
/// without them; codes, each starting on a byte or ending on one, `end_of_line` or not. Each
|
||||
/// decodes to its picture, and a byte short to fewer rows, not to other rows read another way.
|
||||
/// Group 4 rows on bytes keep their aligned read, `end_of_line` or not.
|
||||
#[test]
|
||||
fn group_3_rows_on_bytes_read_by_the_codes_in_the_data() {
|
||||
for seed in 0..32u64 {
|
||||
let (columns, rows) = (8 + (seed as u32 * 37) % 200, 2 + seed as u32 % 20);
|
||||
let runs = text_like(seed, columns, rows);
|
||||
let picture = pack(&runs, columns);
|
||||
for k in [0, 1] {
|
||||
for (codes, fill, end_of_line) in [
|
||||
(false, Fill::RowStart, false),
|
||||
(false, Fill::RowStart, true),
|
||||
(true, Fill::RowStart, false),
|
||||
(true, Fill::RowStart, true),
|
||||
(true, Fill::CodeEnd, false),
|
||||
(true, Fill::CodeEnd, true),
|
||||
] {
|
||||
let p = CcittParams { k, columns: i64::from(columns), end_of_line, encoded_byte_align: true, ..CcittParams::default() };
|
||||
let mut data = group_3(&runs, codes, k > 0, fill);
|
||||
let case = format!("seed {seed}, {columns} × {rows}, k {k}, codes {codes}, fill {fill:?}, end_of_line {end_of_line}");
|
||||
assert_eq!(decode_ccitt(&data, &p, rows).unwrap(), picture, "{case}");
|
||||
data.pop();
|
||||
assert!(decode_ccitt(&data, &p, rows).unwrap().len() < picture.len(), "{case}, a byte short");
|
||||
}
|
||||
}
|
||||
}
|
||||
// White rows, each the one code 1 (vertical, no change) and its fill.
|
||||
for end_of_line in [false, true] {
|
||||
let p = CcittParams { k: -1, columns: 64, end_of_line, encoded_byte_align: true, ..CcittParams::default() };
|
||||
assert_eq!(decode_ccitt(&[0x80; 16], &p, 16).unwrap(), [0xFF; 8 * 16], "Group 4, end_of_line {end_of_line}");
|
||||
}
|
||||
}
|
||||
|
||||
proptest::proptest! {
|
||||
/// Any bitmap, of noise or of runs, decodes to its own bits, through an independent decoder
|
||||
/// and through `decode_ccitt`.
|
||||
#[test]
|
||||
fn bitmaps_round_trip(
|
||||
columns in 1u32..400,
|
||||
rows in 1u32..24,
|
||||
runs in proptest::collection::vec(0u8..=255, 1..48),
|
||||
noise in proptest::bool::ANY,
|
||||
) {
|
||||
let data = bitmap(columns, rows, &runs, noise);
|
||||
let coded = encode_g4(&data, columns, rows).unwrap();
|
||||
proptest::prop_assert_eq!(decode(&coded, columns, rows), unpack(&data, columns, rows));
|
||||
let decoded = decode_ccitt(&coded, &g4(columns), rows).unwrap();
|
||||
proptest::prop_assert_eq!(unpack(&decoded, columns, rows), unpack(&data, columns, rows));
|
||||
}
|
||||
|
||||
/// Whatever it is given, it returns.
|
||||
#[test]
|
||||
fn any_input_returns(
|
||||
data in proptest::collection::vec(0u8..=255, 0..600),
|
||||
columns in 0u32..40_000,
|
||||
rows in 0u32..80,
|
||||
k in -2i64..3,
|
||||
end_of_line in proptest::bool::ANY,
|
||||
encoded_byte_align in proptest::bool::ANY,
|
||||
) {
|
||||
let _ = encode_g4(&data, columns, rows);
|
||||
let p = CcittParams { k, columns: i64::from(columns), end_of_line, encoded_byte_align, ..CcittParams::default() };
|
||||
if let Ok(out) = decode_ccitt(&data, &p, rows) {
|
||||
proptest::prop_assert!(out.len() <= (columns as usize).div_ceil(8) * rows as usize);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,468 @@
|
||||
//! JBIG2 encoding (ITU-T T.88 | ISO/IEC 14492), for `JBIG2Decode` (ISO 32000-2 §7.4.7), lossless:
|
||||
//! a generic region, template 0 with its nominal adaptive pixels, typical prediction on, MQ
|
||||
//! arithmetic coding ([`crate::mq`]); or the image's marks as symbols ([`symbols`]), which a group
|
||||
//! of images may share through `/JBIG2Globals`. Written from the standard's 6.2.5 (coding a generic
|
||||
//! region), 7.2 (segment headers), 7.4.1, 7.4.6 and 7.4.8 (the segments) and D.3 (the embedded
|
||||
//! organisation PDF uses). Decoding is in [`decode`].
|
||||
|
||||
mod symbols;
|
||||
|
||||
/// The largest image the encoders take, in pixels: the largest JBIG2 page `hayro-jbig2` decodes
|
||||
/// (it decodes a page whole, and takes 20,000 × 10,000), so what they store reads back.
|
||||
pub const MAX_JBIG2_PIXELS: u64 = 1 << 28;
|
||||
|
||||
use crate::CodecError;
|
||||
use crate::mq::{Context, MqEncoder};
|
||||
|
||||
/// The widest row, and the most rows, [`encode_jbig2_generic`] encodes: what `hayro-jbig2`, the
|
||||
/// decoder behind `JBIG2Decode` here, takes on a side. JBIG2 allows 32-bit sizes, but past this
|
||||
/// nothing here could read the stream, and the optimizer, which doesn't read back what it stores,
|
||||
/// replaced such an image with one that draws nothing.
|
||||
pub const MAX_JBIG2_COLUMNS: u32 = 65_535;
|
||||
|
||||
/// Template 0's pixels (Figure 3), as (x, y) offsets from the pixel being coded, each a bit of the
|
||||
/// context in this order: the four on the current row, the five fixed ones on the row above, the
|
||||
/// three on the row above that, then the adaptive pixels A1–A4 at their nominal places (Table 5).
|
||||
const TEMPLATE: [(i32, i32); 16] = [
|
||||
(-1, 0),
|
||||
(-2, 0),
|
||||
(-3, 0),
|
||||
(-4, 0),
|
||||
(-2, -1),
|
||||
(-1, -1),
|
||||
(0, -1),
|
||||
(1, -1),
|
||||
(2, -1),
|
||||
(-1, -2),
|
||||
(0, -2),
|
||||
(1, -2),
|
||||
(3, -1),
|
||||
(-3, -1),
|
||||
(2, -2),
|
||||
(-2, -2),
|
||||
];
|
||||
|
||||
/// The context that codes SLTP when template 0 is used (Figure 8): those pixel values, in
|
||||
/// [`TEMPLATE`]'s order. Figure 8 reads, by row from x = -4: `0 1 0 1` on the current row,
|
||||
/// `0 1 1 0 0 1 0` (x = -3 to 3) above it, `1 0 0 1 1` (x = -2 to 2) above that.
|
||||
const SLTP: [u8; 16] = [1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1];
|
||||
|
||||
/// Columns of padding either side of a row, so every template pixel of a row's pixels is in it.
|
||||
const PAD: usize = 4;
|
||||
|
||||
/// `rows` rows of `columns` 1-bit PDF samples as lossless JBIG2, the smaller of two codings:
|
||||
/// [`encode_jbig2_generic`]'s, or one that stores each distinct mark of the image once, as a
|
||||
/// symbol, and places its copies (see [`symbols`]). Either way the stream is in the embedded
|
||||
/// organisation and decodes to the same bits without `/JBIG2Globals`.
|
||||
///
|
||||
/// Errors: as [`encode_jbig2_generic`].
|
||||
pub fn encode_jbig2(data: &[u8], columns: u32, rows: u32) -> Result<Vec<u8>, CodecError> {
|
||||
let generic = encode_jbig2_generic(data, columns, rows)?;
|
||||
let samples = rows_to_encode(data, columns, rows).ok();
|
||||
Ok(match samples.and_then(|s| symbols::encode(s, columns, rows)) {
|
||||
Some(coded) if coded.len() < generic.len() => coded,
|
||||
_ => generic,
|
||||
})
|
||||
}
|
||||
|
||||
/// `rows` rows of `columns` 1-bit PDF samples (most significant bit first, each row starting on
|
||||
/// a new byte), as a JBIG2 stream in the embedded organisation: a page information segment and an
|
||||
/// immediate lossless generic region segment, which `JBIG2Decode` decodes to the same bits
|
||||
/// without `/JBIG2Globals`.
|
||||
///
|
||||
/// A 0 sample is coded as a JBIG2 1 (black), since `JBIG2Decode` gives black as 0, as a grey image's
|
||||
/// samples have it. Bytes after the last row are ignored, as are the bits that pad a row to a whole
|
||||
/// byte.
|
||||
///
|
||||
/// Errors: no rows or columns, more of either than [`MAX_JBIG2_COLUMNS`], or more samples than
|
||||
/// [`MAX_JBIG2_PIXELS`], what the decoder takes ([`CodecError::Unsupported`]); fewer bytes than the
|
||||
/// rows need ([`CodecError::Corrupt`]).
|
||||
pub fn encode_jbig2_generic(data: &[u8], columns: u32, rows: u32) -> Result<Vec<u8>, CodecError> {
|
||||
let samples = rows_to_encode(data, columns, rows)?;
|
||||
let width = columns as usize;
|
||||
let stride = width.div_ceil(8);
|
||||
let mut contexts: Vec<Context> = vec![0; 1 << 16];
|
||||
let mut mq = MqEncoder::new();
|
||||
let height = samples.len() / stride;
|
||||
code_generic(&mut mq, &mut contexts, width, height, true, |y, row| {
|
||||
let packed = samples.get(y * stride..).unwrap_or_default();
|
||||
for (x, pixel) in row.iter_mut().enumerate() {
|
||||
let sample = packed.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0);
|
||||
*pixel = u8::from(!sample);
|
||||
}
|
||||
});
|
||||
let region = mq.finish();
|
||||
let mut out = Vec::with_capacity(region.len() + 64);
|
||||
segment(&mut out, 0, 48, &[], false, &page_information(columns, rows));
|
||||
segment(&mut out, 1, 39, &[], false, &generic_region(columns, rows, 0, 0, ®ion));
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// The samples of `rows` rows of `columns` 1-bit samples in `data`, when the encoders here take
|
||||
/// that size; the errors are [`encode_jbig2_generic`]'s.
|
||||
fn rows_to_encode(data: &[u8], columns: u32, rows: u32) -> Result<&[u8], CodecError> {
|
||||
let too_large = columns > MAX_JBIG2_COLUMNS || rows > MAX_JBIG2_COLUMNS || u64::from(columns) * u64::from(rows) > MAX_JBIG2_PIXELS;
|
||||
if columns == 0 || rows == 0 || too_large {
|
||||
return Err(CodecError::Unsupported(format!("JBIG2 encoding of {columns} × {rows} samples")));
|
||||
}
|
||||
let stride = (columns as usize).div_ceil(8);
|
||||
usize::try_from(rows)
|
||||
.ok()
|
||||
.and_then(|r| r.checked_mul(stride))
|
||||
.and_then(|n| data.get(..n))
|
||||
.ok_or_else(|| CodecError::Corrupt { filter: "JBIG2Decode", detail: format!("{} bytes for {rows} rows of {columns} samples", data.len()) })
|
||||
}
|
||||
|
||||
/// A group of images coded by [`encode_jbig2_shared`]: the data of the `/JBIG2Globals` stream
|
||||
/// they share, and each image's own stream, in the order given, or `None` for one that shares
|
||||
/// nothing with the others.
|
||||
#[derive(Debug)]
|
||||
pub struct SharedJbig2 {
|
||||
pub globals: Vec<u8>,
|
||||
pub images: Vec<Option<Vec<u8>>>,
|
||||
}
|
||||
|
||||
/// Images, each `rows` rows of `columns` 1-bit PDF samples, as lossless JBIG2 sharing one symbol
|
||||
/// dictionary: the marks found in more than one of them are stored once, in a symbol dictionary
|
||||
/// segment not associated with a page, which is the `/JBIG2Globals` stream's data, and each image's
|
||||
/// stream, in the embedded organisation, places them from there, with its other marks coded as
|
||||
/// [`encode_jbig2`]'s symbol coding codes them (see [`symbols`]). With the globals, each decodes to
|
||||
/// its own bits. An image that holds none of those marks, or that symbol coding doesn't take, has
|
||||
/// no stream here; it is stored alone. `None` when the images share fewer than two marks.
|
||||
///
|
||||
/// What each image's stream saves is left for the caller to weigh against what the globals take,
|
||||
/// which the images that use them share.
|
||||
///
|
||||
/// Errors: as [`encode_jbig2_generic`], for the first image it would refuse.
|
||||
pub fn encode_jbig2_shared(images: &[(&[u8], u32, u32)]) -> Result<Option<SharedJbig2>, CodecError> {
|
||||
let images =
|
||||
images.iter().map(|(data, columns, rows)| Ok((rows_to_encode(data, *columns, *rows)?, *columns, *rows))).collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(symbols::encode_shared(&images).map(|(globals, images)| SharedJbig2 { globals, images }))
|
||||
}
|
||||
|
||||
/// A page information segment's data (7.4.8): the page's size, resolution unknown, eventually
|
||||
/// lossless, default pixel 0 (white), OR combination, not striped.
|
||||
fn page_information(columns: u32, rows: u32) -> Vec<u8> {
|
||||
let mut page = Vec::with_capacity(19);
|
||||
page.extend_from_slice(&columns.to_be_bytes());
|
||||
page.extend_from_slice(&rows.to_be_bytes());
|
||||
page.extend_from_slice(&[0; 8]);
|
||||
page.push(0x01);
|
||||
page.extend_from_slice(&[0, 0]);
|
||||
page
|
||||
}
|
||||
|
||||
/// A region segment information field (7.4.1): a `width` × `height` region at (`x`, `y`) on the
|
||||
/// page, combined with it by OR.
|
||||
fn region_information(out: &mut Vec<u8>, width: u32, height: u32, x: u32, y: u32) {
|
||||
for v in [width, height, x, y] {
|
||||
out.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
out.push(0);
|
||||
}
|
||||
|
||||
/// An immediate lossless generic region segment's data (7.4.6): its region information, flags
|
||||
/// (MMR 0, template 0, TPGDON 1), the AT pixels at their nominal places, then `coded`.
|
||||
fn generic_region(width: u32, height: u32, x: u32, y: u32, coded: &[u8]) -> Vec<u8> {
|
||||
let mut data = Vec::with_capacity(26 + coded.len());
|
||||
region_information(&mut data, width, height, x, y);
|
||||
data.push(0x08);
|
||||
at_pixels(&mut data);
|
||||
data.extend_from_slice(coded);
|
||||
data
|
||||
}
|
||||
|
||||
/// Template 0's four AT pixels at their nominal places, as AT flags (7.4.6.3, 7.4.2.1.2).
|
||||
fn at_pixels(out: &mut Vec<u8>) {
|
||||
for (x, y) in &TEMPLATE[12..] {
|
||||
out.push(*x as i8 as u8);
|
||||
out.push(*y as i8 as u8);
|
||||
}
|
||||
}
|
||||
|
||||
/// A segment (7.2) of page 1: its header (number, type, the segments it refers to, the page, data
|
||||
/// length), then its data. `retained`: a later segment refers to this one. `refers_to` holds each
|
||||
/// referred-to segment's number and whether a segment after this one refers to it too (its
|
||||
/// retain bit, 7.2.4). Segment numbers here stay below 256, so each referred-to one takes a byte
|
||||
/// (7.2.5); a segment refers to at most four, so their count and retain bits take one byte too.
|
||||
fn segment(out: &mut Vec<u8>, number: u32, kind: u8, refers_to: &[(u8, bool)], retained: bool, data: &[u8]) {
|
||||
segment_of(out, 1, number, kind, refers_to, retained, data);
|
||||
}
|
||||
|
||||
/// [`segment`], of page `page`: 0 for a segment associated with no page (7.2.6), such as the
|
||||
/// global ones `/JBIG2Globals` holds, which a page's segments may refer to.
|
||||
fn segment_of(out: &mut Vec<u8>, page: u8, number: u32, kind: u8, refers_to: &[(u8, bool)], retained: bool, data: &[u8]) {
|
||||
out.extend_from_slice(&number.to_be_bytes());
|
||||
out.push(kind);
|
||||
// At most four, so the count fits its three bits.
|
||||
let refers_to = refers_to.get(..refers_to.len().min(4)).unwrap_or_default();
|
||||
let kept = refers_to.iter().enumerate().fold(0u8, |bits, (k, (_, kept))| bits | u8::from(*kept) << (k + 1));
|
||||
out.push((refers_to.len() as u8) << 5 | kept | u8::from(retained));
|
||||
out.extend(refers_to.iter().map(|(n, _)| *n));
|
||||
out.push(page);
|
||||
// A segment's data never comes near 4 GiB here: an image is at most 2^28 samples.
|
||||
out.extend_from_slice(&u32::try_from(data.len()).unwrap_or(u32::MAX).to_be_bytes());
|
||||
out.extend_from_slice(data);
|
||||
}
|
||||
|
||||
/// Codes a `width` × `height` bitmap as a generic region (6.2.5.7) with template 0 and the nominal
|
||||
/// AT pixels, its decisions in `contexts` (2^16 of them), with typical prediction when `typical`
|
||||
/// (TPGDON). `fill(y, row)` sets row `y`'s `width` pixels, 1 for black. Pixels outside the bitmap
|
||||
/// count as 0 (white), as the decoder takes them.
|
||||
fn code_generic(mq: &mut MqEncoder, contexts: &mut [Context], width: usize, height: usize, typical: bool, mut fill: impl FnMut(usize, &mut [u8])) {
|
||||
let sltp = SLTP.iter().enumerate().fold(0usize, |cx, (k, v)| cx | usize::from(*v) << k);
|
||||
// The two rows above the current one and the current one, as JBIG2 pixels (1 black), padded.
|
||||
let blank = vec![0u8; width + 2 * PAD];
|
||||
let (mut above2, mut above1) = (blank.clone(), blank.clone());
|
||||
let mut row = blank;
|
||||
let mut ltp = false;
|
||||
for y in 0..height {
|
||||
if let Some(pixels) = row.get_mut(PAD..PAD + width) {
|
||||
fill(y, pixels);
|
||||
}
|
||||
if typical {
|
||||
// Typical prediction (6.2.5.5): a row the same as the one above is coded as one decision.
|
||||
let same = row == above1;
|
||||
if let Some(cx) = contexts.get_mut(sltp) {
|
||||
mq.encode(cx, same != ltp);
|
||||
}
|
||||
ltp = same;
|
||||
}
|
||||
if !ltp {
|
||||
for x in 0..width {
|
||||
let at = |dx: i32, dy: i32| {
|
||||
let line = match dy {
|
||||
0 => &row,
|
||||
-1 => &above1,
|
||||
_ => &above2,
|
||||
};
|
||||
let i = (PAD + x).checked_add_signed(dx as isize);
|
||||
i.and_then(|i| line.get(i)).copied().unwrap_or(0)
|
||||
};
|
||||
let cx = TEMPLATE.iter().enumerate().fold(0usize, |cx, (k, (dx, dy))| cx | usize::from(at(*dx, *dy)) << k);
|
||||
if let Some(state) = contexts.get_mut(cx) {
|
||||
mq.encode(state, row.get(PAD + x) == Some(&1));
|
||||
}
|
||||
}
|
||||
}
|
||||
std::mem::swap(&mut above2, &mut above1);
|
||||
std::mem::swap(&mut above1, &mut row);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The rows an independent decoder (`hayro-jbig2`) reads from `data`, as PDF samples: true
|
||||
/// for white (a JBIG2 0).
|
||||
pub(super) fn decode(data: &[u8]) -> Vec<Vec<bool>> {
|
||||
decode_with(data, None)
|
||||
}
|
||||
|
||||
/// [`decode`], with `/JBIG2Globals`.
|
||||
pub(super) fn decode_with(data: &[u8], globals: Option<&[u8]>) -> Vec<Vec<bool>> {
|
||||
struct Rows(Vec<Vec<bool>>, Vec<bool>, usize);
|
||||
impl hayro_jbig2::Decoder for Rows {
|
||||
fn push_pixel(&mut self, black: bool) {
|
||||
self.1.push(!black);
|
||||
}
|
||||
fn push_pixel_chunk(&mut self, black: bool, chunks: u32) {
|
||||
self.1.extend(std::iter::repeat_n(!black, chunks as usize * 8));
|
||||
}
|
||||
fn next_line(&mut self) {
|
||||
let mut row = std::mem::take(&mut self.1);
|
||||
row.truncate(self.2);
|
||||
self.0.push(row);
|
||||
}
|
||||
}
|
||||
let image = hayro_jbig2::Image::new_embedded(data, globals).unwrap();
|
||||
let mut rows = Rows(Vec::new(), Vec::new(), image.width() as usize);
|
||||
image.decode(&mut rows).unwrap();
|
||||
rows.0
|
||||
}
|
||||
|
||||
/// The rows of packed samples `data`, true for a 1 bit.
|
||||
pub(super) fn unpack(data: &[u8], columns: u32, rows: u32) -> Vec<Vec<bool>> {
|
||||
let stride = (columns as usize).div_ceil(8);
|
||||
(0..rows as usize).map(|y| (0..columns as usize).map(|x| data[y * stride + x / 8] & (0x80 >> (x % 8)) != 0).collect()).collect()
|
||||
}
|
||||
|
||||
/// Rows of `columns` samples from `runs`: alternating white and black runs one longer than
|
||||
/// those values (never empty, even when shrinking makes them all 0), until the rows are full;
|
||||
/// or, with `noise`, the bytes of `runs` as they are.
|
||||
fn bitmap(columns: u32, rows: u32, runs: &[u8], noise: bool) -> Vec<u8> {
|
||||
let stride = (columns as usize).div_ceil(8);
|
||||
let mut data = vec![0u8; stride * rows as usize];
|
||||
if runs.is_empty() {
|
||||
return data;
|
||||
}
|
||||
if noise {
|
||||
data.iter_mut().zip(runs.iter().cycle()).for_each(|(d, r)| *d = *r);
|
||||
return data;
|
||||
}
|
||||
let (mut white, mut left, mut next) = (true, 1 + usize::from(runs[0]), 1);
|
||||
for y in 0..rows as usize {
|
||||
for x in 0..columns as usize {
|
||||
if left == 0 {
|
||||
white = !white;
|
||||
left = 1 + usize::from(runs[next % runs.len()]);
|
||||
next += 1;
|
||||
}
|
||||
if white {
|
||||
data[y * stride + x / 8] |= 0x80 >> (x % 8);
|
||||
}
|
||||
left -= 1;
|
||||
}
|
||||
}
|
||||
data
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_page_of_text_like_marks_round_trips_and_shrinks() {
|
||||
let (w, h) = (1700u32, 220u32);
|
||||
let stride = (w as usize).div_ceil(8);
|
||||
let mut data = vec![0xFF; stride * h as usize];
|
||||
for y in (20..h as usize).step_by(40) {
|
||||
for dy in 0..12 {
|
||||
for x in (100..1600).step_by(9) {
|
||||
if (x / 9 + dy) % 5 != 0 {
|
||||
data[(y + dy) * stride + x / 8] &= !(0x80 >> (x % 8));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let jbig2 = encode_jbig2_generic(&data, w, h).unwrap();
|
||||
assert_eq!(decode(&jbig2), unpack(&data, w, h));
|
||||
let g4 = crate::encode_g4(&data, w, h).unwrap();
|
||||
assert!(jbig2.len() < g4.len(), "JBIG2 {} bytes, Group 4 {}", jbig2.len(), g4.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edges_padding_and_typical_rows_round_trip() {
|
||||
// 3 columns: the last 5 bits of each byte pad the row; set them to anything.
|
||||
let data = [0b1010_1111, 0b0101_0000, 0b1110_1010, 0xAB, 0xCD];
|
||||
assert_eq!(decode(&encode_jbig2_generic(&data, 3, 3).unwrap()), [[true, false, true], [false, true, false], [true, true, true]]);
|
||||
for (w, h, fill) in [(1, 1, 0x00), (1, 1, 0xFF), (13, 9, 0x00), (13, 9, 0xFF), (64, 40, 0x5A)] {
|
||||
let data = vec![fill; (w as usize).div_ceil(8) * h as usize];
|
||||
assert_eq!(decode(&encode_jbig2_generic(&data, w, h).unwrap()), unpack(&data, w, h), "{w} × {h} of {fill:#04x}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Packed samples from JBIG2 pixels (1 black): a 0 sample for each 1.
|
||||
pub(super) fn samples(pixels: &[Vec<u8>]) -> Vec<u8> {
|
||||
let width = pixels.first().map_or(0, Vec::len);
|
||||
let stride = width.div_ceil(8);
|
||||
let mut data = vec![0xFF; stride * pixels.len()];
|
||||
for (y, row) in pixels.iter().enumerate() {
|
||||
for (x, p) in row.iter().enumerate() {
|
||||
if *p == 1 {
|
||||
data[y * stride + x / 8] &= !(0x80 >> (x % 8));
|
||||
}
|
||||
}
|
||||
}
|
||||
data
|
||||
}
|
||||
|
||||
/// A row the same as the one above is typical (6.2.5.5): one decision, not a row of pixels.
|
||||
#[test]
|
||||
fn a_repeated_row_is_one_decision() {
|
||||
let row: Vec<u8> = (0..800u32).map(|x| u8::from((x * 7919 + x / 3) % 5 < 2)).collect();
|
||||
let pixels = vec![row; 1000];
|
||||
let data = samples(&pixels);
|
||||
crate::mq::DECISIONS.with(|n| n.set(0));
|
||||
let jbig2 = encode_jbig2_generic(&data, 800, 1000).unwrap();
|
||||
// The first row's 800 pixels, and one typical-row decision for each of the 1000 rows.
|
||||
assert_eq!(crate::mq::DECISIONS.with(|n| n.get()), 800 + 1000);
|
||||
assert_eq!(decode(&jbig2), unpack(&data, 800, 1000));
|
||||
}
|
||||
|
||||
/// SLTP is coded in the context Figure 8 shows, which ordinary pixels can share: an image
|
||||
/// tiled with that very neighbourhood decodes only if both coders use the same context.
|
||||
#[test]
|
||||
fn the_typical_row_flag_shares_figure_8s_context() {
|
||||
// Figure 8, around a pixel at x = 4 of an 8-column tile: two rows up x = 2..6, one row up
|
||||
// x = 1..7, the pixel's own row x = 0..3.
|
||||
let tile = [[0, 0, 1, 0, 0, 1, 1, 0], [0, 0, 1, 1, 0, 0, 1, 0], [0, 1, 0, 1, 1, 0, 0, 0]];
|
||||
let pixels: Vec<Vec<u8>> = (0..90).map(|y| (0..400).map(|x| tile[y % 3][x % 8]).collect()).collect();
|
||||
let data = samples(&pixels);
|
||||
assert_eq!(decode(&encode_jbig2_generic(&data, 400, 90).unwrap()), unpack(&data, 400, 90));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn what_it_cannot_encode_is_an_error() {
|
||||
assert!(matches!(encode_jbig2_generic(&[0; 8], 0, 1), Err(CodecError::Unsupported(_))));
|
||||
assert!(matches!(encode_jbig2_generic(&[0; 8], 8, 0), Err(CodecError::Unsupported(_))));
|
||||
assert!(matches!(encode_jbig2_generic(&[0; 3], 16, 2), Err(CodecError::Corrupt { .. })), "two rows of 16 need 4 bytes");
|
||||
assert!(matches!(encode_jbig2_generic(&[], 1, 2), Err(CodecError::Corrupt { .. })));
|
||||
}
|
||||
|
||||
/// The encoder stops where the decoder does, on each side and in total: past that it would
|
||||
/// write a stream nothing here can read. 16,384 squared is exactly the bound on samples.
|
||||
/// `encode_jbig2_shared` refuses what `encode_jbig2_generic` refuses: an image taller than
|
||||
/// the decoder takes is `Unsupported`, one at the bound isn't refused. (Its one tall mark is
|
||||
/// too large to be a symbol, so nothing is shared: `Ok(None)`.)
|
||||
#[test]
|
||||
fn shared_coding_takes_only_what_the_decoder_takes() {
|
||||
let past = vec![0x7F; MAX_JBIG2_COLUMNS as usize + 1];
|
||||
let tall = (&past[..], 1, MAX_JBIG2_COLUMNS + 1);
|
||||
assert!(matches!(encode_jbig2_shared(&[tall, tall]), Err(CodecError::Unsupported(_))), "too tall");
|
||||
let at = (&past[..MAX_JBIG2_COLUMNS as usize], 1, MAX_JBIG2_COLUMNS);
|
||||
assert!(matches!(encode_jbig2_shared(&[at, at]), Ok(None)), "at the bound");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn it_encodes_only_what_the_decoder_takes() {
|
||||
let unsupported = |columns, rows| matches!(encode_jbig2_generic(&[], columns, rows), Err(CodecError::Unsupported(_)));
|
||||
assert!(unsupported(MAX_JBIG2_COLUMNS + 1, 1), "too wide");
|
||||
assert!(unsupported(1, MAX_JBIG2_COLUMNS + 1), "too tall");
|
||||
assert!(unsupported(16_385, 16_384), "too many samples, each side within the bound");
|
||||
assert!(unsupported(u32::MAX, u32::MAX), "the product doesn't overflow");
|
||||
// At the bounds what it writes reads back, through the decoder that has to read it.
|
||||
let wide = vec![0x0F; (MAX_JBIG2_COLUMNS as usize).div_ceil(8) * 4];
|
||||
assert_eq!(decode(&encode_jbig2_generic(&wide, MAX_JBIG2_COLUMNS, 4).unwrap()), unpack(&wide, MAX_JBIG2_COLUMNS, 4));
|
||||
let tall = vec![0x7F; MAX_JBIG2_COLUMNS as usize];
|
||||
assert_eq!(decode(&encode_jbig2_generic(&tall, 1, MAX_JBIG2_COLUMNS).unwrap()), unpack(&tall, 1, MAX_JBIG2_COLUMNS));
|
||||
struct Rows(u32, u64);
|
||||
impl hayro_jbig2::Decoder for Rows {
|
||||
fn push_pixel(&mut self, black: bool) {
|
||||
self.1 += u64::from(black);
|
||||
}
|
||||
fn push_pixel_chunk(&mut self, black: bool, chunks: u32) {
|
||||
self.1 += u64::from(black) * u64::from(chunks) * 8;
|
||||
}
|
||||
fn next_line(&mut self) {
|
||||
self.0 += 1;
|
||||
}
|
||||
}
|
||||
// One black pixel, so a page that came back blank wouldn't pass.
|
||||
let mut largest = vec![0xFF; 2_048 * 16_384];
|
||||
largest[0] = 0x7F;
|
||||
let largest = encode_jbig2_generic(&largest, 16_384, 16_384).unwrap();
|
||||
let mut rows = Rows(0, 0);
|
||||
hayro_jbig2::Image::new_embedded(&largest, None).unwrap().decode(&mut rows).unwrap();
|
||||
assert_eq!(rows.0, 16_384);
|
||||
assert_eq!(rows.1, 1);
|
||||
}
|
||||
|
||||
proptest::proptest! {
|
||||
/// Any bitmap, of noise or of runs, decodes to its own bits through an independent decoder.
|
||||
#[test]
|
||||
fn bitmaps_round_trip(
|
||||
columns in 1u32..300,
|
||||
rows in 1u32..24,
|
||||
runs in proptest::collection::vec(0u8..=255, 1..48),
|
||||
noise in proptest::bool::ANY,
|
||||
) {
|
||||
let data = bitmap(columns, rows, &runs, noise);
|
||||
let jbig2 = encode_jbig2_generic(&data, columns, rows).unwrap();
|
||||
proptest::prop_assert_eq!(decode(&jbig2), unpack(&data, columns, rows));
|
||||
}
|
||||
|
||||
/// Whatever it is given, it returns.
|
||||
#[test]
|
||||
fn any_input_returns(data in proptest::collection::vec(0u8..=255, 0..600), columns in 0u32..5000, rows in 0u32..80) {
|
||||
let _ = encode_jbig2_generic(&data, columns, rows);
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,117 @@
|
||||
//! Byte-stream and bi-level image codecs that PDF, PostScript and TIFF share.
|
||||
//!
|
||||
//! - [`lzw`]: variable-width LZW, 9 to 12 bits, most significant bit first (PDF `LZWDecode`,
|
||||
//! TIFF compression 5), with or without the early code-width change.
|
||||
//! - [`ascii85`], [`asciihex`], [`runlength`]: the PostScript and PDF text and run-length
|
||||
//! encodings (`ASCII85Decode`, `ASCIIHexDecode`, `RunLengthDecode`; PackBits).
|
||||
//! - [`predictor`]: TIFF horizontal differencing and the PNG row filters.
|
||||
//! - [`ccitt`]: CCITT Group 3 and Group 4 fax decoding, bounded, and Group 4 encoding.
|
||||
//! - [`jbig2`]: lossless JBIG2 encoding (generic regions, symbol dictionaries, a dictionary
|
||||
//! shared by a group of images). Decoding is `hayro-jbig2`'s.
|
||||
//!
|
||||
//! The crate knows no container: a caller hands it bytes and parameters. A PDF `/Filter` chain
|
||||
//! (order, limits across stages, Flate) is `craft_pdf::filters`, built on this crate.
|
||||
//!
|
||||
//! Every decoder is bounded by a caller-supplied output limit and never panics on malformed
|
||||
//! input. A decoder that fails returns what it decoded before the damage ([`Failure`]), so a
|
||||
//! caller can choose to keep it. CCITT decoding spends from an allowance the caller shares
|
||||
//! across a run of images ([`pay`]); it does not decode without one:
|
||||
//!
|
||||
//! ```compile_fail
|
||||
//! let _ = craft_stream_codecs::decode_ccitt(&[], &craft_stream_codecs::CcittParams::default(), 1);
|
||||
//! ```
|
||||
//!
|
||||
//! ```
|
||||
//! let mut work = 1 << 20;
|
||||
//! let p = craft_stream_codecs::CcittParams { columns: 8, ..Default::default() };
|
||||
//! assert!(craft_stream_codecs::decode_ccitt_within(&[], &p, 1, &mut work).is_ok());
|
||||
//! ```
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
pub mod ascii85;
|
||||
pub mod asciihex;
|
||||
pub mod ccitt;
|
||||
pub mod jbig2;
|
||||
pub mod lzw;
|
||||
mod mq;
|
||||
pub mod predictor;
|
||||
pub mod runlength;
|
||||
|
||||
pub use ccitt::{CcittParams, MAX_CCITT_COLUMNS, MAX_G4_COLUMNS, decode_ccitt_within, encode_g4};
|
||||
pub use jbig2::{MAX_JBIG2_COLUMNS, MAX_JBIG2_PIXELS, SharedJbig2, encode_jbig2, encode_jbig2_generic, encode_jbig2_shared};
|
||||
|
||||
/// Parameters of the predictors and of LZW, as a PDF `/DecodeParms` dictionary gives them
|
||||
/// (ISO 32000-2 Tables 8–9); TIFF's `Predictor`, `SamplesPerPixel`, `BitsPerSample` and
|
||||
/// `ImageWidth` map onto the same four.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Params {
|
||||
/// 1 = none, 2 = TIFF predictor 2, 10–15 = PNG predictors.
|
||||
pub predictor: i64,
|
||||
pub colors: i64,
|
||||
pub bits_per_component: i64,
|
||||
pub columns: i64,
|
||||
/// LZW only: 1 (default) switches code width one code early, 0 does not.
|
||||
pub early_change: i64,
|
||||
}
|
||||
|
||||
impl Default for Params {
|
||||
fn default() -> Self {
|
||||
Params { predictor: 1, colors: 1, bits_per_component: 8, columns: 1, early_change: 1 }
|
||||
}
|
||||
}
|
||||
|
||||
/// Errors from decoding or encoding.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CodecError {
|
||||
/// `filter` names the codec, as PDF names its filter (`LZWDecode`, `CCITTFaxDecode`, …).
|
||||
#[error("{filter}: corrupt data: {detail}")]
|
||||
Corrupt { filter: &'static str, detail: String },
|
||||
#[error("decoded output exceeds the limit of {0} bytes")]
|
||||
LimitExceeded(usize),
|
||||
#[error("unsupported filter: {0}")]
|
||||
Unsupported(String),
|
||||
/// A run of images has nothing left to spend ([`pay`]).
|
||||
#[error("more image decoding than one operation may do")]
|
||||
Spent,
|
||||
}
|
||||
|
||||
/// A decoder failure carrying whatever was decoded before the error.
|
||||
pub struct Failure {
|
||||
pub error: CodecError,
|
||||
pub partial: Vec<u8>,
|
||||
/// Bytes decoded on the way that `partial` doesn't hold: they took as long.
|
||||
pub discarded: usize,
|
||||
}
|
||||
|
||||
impl Failure {
|
||||
pub fn corrupt(filter: &'static str, detail: impl Into<String>, partial: Vec<u8>) -> Self {
|
||||
Failure { error: CodecError::Corrupt { filter, detail: detail.into() }, partial, discarded: 0 }
|
||||
}
|
||||
|
||||
pub fn limit(max: usize) -> Self {
|
||||
Failure { error: CodecError::LimitExceeded(max), partial: Vec::new(), discarded: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
/// What a byte-stream decoder returns: the decoded bytes, or the failure with its partial output.
|
||||
pub type Step = Result<Vec<u8>, Failure>;
|
||||
|
||||
/// Takes `steps` from `work`, what a run of images may still cost. What it can't pay for spends
|
||||
/// all of it.
|
||||
pub fn pay(work: &mut u64, steps: u64) -> Result<(), CodecError> {
|
||||
match work.checked_sub(steps) {
|
||||
Some(left) => {
|
||||
*work = left;
|
||||
Ok(())
|
||||
}
|
||||
None => {
|
||||
*work = 0;
|
||||
Err(CodecError::Spent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// PDF and PostScript white-space characters (ISO 32000-2 Table 1).
|
||||
pub fn is_pdf_whitespace(b: u8) -> bool {
|
||||
matches!(b, 0 | 9 | 10 | 12 | 13 | 32)
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
//! LZWDecode (ISO 32000-2 §7.4.4.2): variable-width codes of 9–12 bits, MSB first,
|
||||
//! 256 = clear-table, 257 = EOD, first free code 258. With `EarlyChange` 1 (the
|
||||
//! default) the code width grows one code early.
|
||||
//!
|
||||
//! Tolerated: data ending without EOD, a missing initial clear-table code, a table that
|
||||
//! fills up without a clear-table code (entries are then no longer added).
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::{Failure, Step};
|
||||
|
||||
const NAME: &str = "LZWDecode";
|
||||
const CLEAR: u16 = 256;
|
||||
const EOD: u16 = 257;
|
||||
const FIRST: usize = 258;
|
||||
const TABLE: usize = 4096;
|
||||
/// The encoder emits clear-table when its next free code reaches this value.
|
||||
const ENCODER_RESET_AT: usize = TABLE;
|
||||
|
||||
/// The code width a decoder uses when its next free code is `next`.
|
||||
fn width(next: usize, early: bool) -> u32 {
|
||||
match next + usize::from(early) {
|
||||
0..512 => 9,
|
||||
512..1024 => 10,
|
||||
1024..2048 => 11,
|
||||
_ => 12,
|
||||
}
|
||||
}
|
||||
|
||||
struct BitReader<'a> {
|
||||
data: &'a [u8],
|
||||
pos: usize,
|
||||
acc: u32,
|
||||
bits: u32,
|
||||
}
|
||||
|
||||
impl BitReader<'_> {
|
||||
fn read(&mut self, n: u32) -> Option<u16> {
|
||||
while self.bits < n {
|
||||
let &b = self.data.get(self.pos)?;
|
||||
self.pos += 1;
|
||||
self.acc = (self.acc << 8) | u32::from(b);
|
||||
self.bits += 8;
|
||||
}
|
||||
self.bits -= n;
|
||||
let v = (self.acc >> self.bits) & ((1 << n) - 1);
|
||||
self.acc &= (1 << self.bits) - 1;
|
||||
Some(v as u16)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decode(data: &[u8], early: bool, max: usize) -> Step {
|
||||
// Table entry = (prefix code, last byte, first byte, length).
|
||||
let mut prefix = vec![0u16; TABLE];
|
||||
let mut last = vec![0u8; TABLE];
|
||||
let mut first = vec![0u8; TABLE];
|
||||
let mut len = vec![0u16; TABLE];
|
||||
for i in 0..256 {
|
||||
last[i] = i as u8;
|
||||
first[i] = i as u8;
|
||||
len[i] = 1;
|
||||
}
|
||||
let mut next = FIRST;
|
||||
let mut w = 9;
|
||||
let mut prev: Option<usize> = None;
|
||||
let mut r = BitReader { data, pos: 0, acc: 0, bits: 0 };
|
||||
let mut out = Vec::with_capacity(data.len().saturating_mul(2).min(max));
|
||||
|
||||
while let Some(code) = r.read(w) {
|
||||
match code {
|
||||
CLEAR => {
|
||||
next = FIRST;
|
||||
w = 9;
|
||||
prev = None;
|
||||
continue;
|
||||
}
|
||||
EOD => break,
|
||||
_ => {}
|
||||
}
|
||||
let code = usize::from(code);
|
||||
if let Some(p) = prev {
|
||||
let head = if code < next {
|
||||
first[code]
|
||||
} else if code == next && next < TABLE {
|
||||
first[p]
|
||||
} else {
|
||||
return Err(Failure::corrupt(NAME, format!("code {code} not in table (next {next})"), out));
|
||||
};
|
||||
if next < TABLE {
|
||||
prefix[next] = p as u16;
|
||||
last[next] = head;
|
||||
first[next] = first[p];
|
||||
len[next] = len[p] + 1;
|
||||
next += 1;
|
||||
}
|
||||
} else if code >= 256 {
|
||||
return Err(Failure::corrupt(NAME, format!("code {code} before any table entry"), out));
|
||||
}
|
||||
// Emit the string for `code`, walking the prefix chain backwards.
|
||||
let n = usize::from(len[code]);
|
||||
if out.len() + n > max {
|
||||
return Err(Failure::limit(max));
|
||||
}
|
||||
let start = out.len();
|
||||
out.resize(start + n, 0);
|
||||
let mut c = code;
|
||||
for slot in out[start..].iter_mut().rev() {
|
||||
*slot = last[c];
|
||||
c = usize::from(prefix[c]);
|
||||
}
|
||||
prev = Some(code);
|
||||
w = width(next, early);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
struct BitWriter {
|
||||
out: Vec<u8>,
|
||||
acc: u32,
|
||||
bits: u32,
|
||||
}
|
||||
|
||||
impl BitWriter {
|
||||
fn write(&mut self, code: u16, n: u32) {
|
||||
self.acc = (self.acc << n) | u32::from(code);
|
||||
self.bits += n;
|
||||
while self.bits >= 8 {
|
||||
self.bits -= 8;
|
||||
self.out.push((self.acc >> self.bits) as u8);
|
||||
}
|
||||
self.acc &= (1 << self.bits) - 1;
|
||||
}
|
||||
|
||||
fn finish(mut self) -> Vec<u8> {
|
||||
if self.bits > 0 {
|
||||
self.out.push((self.acc << (8 - self.bits)) as u8);
|
||||
}
|
||||
self.out
|
||||
}
|
||||
}
|
||||
|
||||
/// Greedy LZW with a leading clear-table code and a trailing EOD.
|
||||
pub fn encode(data: &[u8], early: bool) -> Vec<u8> {
|
||||
let mut wr = BitWriter { out: Vec::with_capacity(data.len() / 2 + 8), acc: 0, bits: 0 };
|
||||
let mut dict: HashMap<(u16, u8), u16> = HashMap::new();
|
||||
// `next` is the encoder's next free code; `dnext` mirrors the decoder's, which lags
|
||||
// one entry behind (the decoder adds an entry only once it sees the following code).
|
||||
let mut next = FIRST;
|
||||
let mut dnext = FIRST;
|
||||
let mut codes_since_clear = 0usize;
|
||||
wr.write(CLEAR, 9);
|
||||
let emit = |wr: &mut BitWriter, code: u16, dnext: &mut usize, since: &mut usize| {
|
||||
wr.write(code, width(*dnext, early));
|
||||
if *since > 0 && *dnext < TABLE {
|
||||
*dnext += 1;
|
||||
}
|
||||
*since += 1;
|
||||
};
|
||||
let Some((&b0, rest)) = data.split_first() else {
|
||||
wr.write(EOD, 9);
|
||||
return wr.finish();
|
||||
};
|
||||
let mut cur = u16::from(b0);
|
||||
for &b in rest {
|
||||
if let Some(&c) = dict.get(&(cur, b)) {
|
||||
cur = c;
|
||||
continue;
|
||||
}
|
||||
emit(&mut wr, cur, &mut dnext, &mut codes_since_clear);
|
||||
dict.insert((cur, b), next as u16);
|
||||
next += 1;
|
||||
if next == ENCODER_RESET_AT {
|
||||
wr.write(CLEAR, width(dnext, early));
|
||||
dict.clear();
|
||||
next = FIRST;
|
||||
dnext = FIRST;
|
||||
codes_since_clear = 0;
|
||||
}
|
||||
cur = u16::from(b);
|
||||
}
|
||||
emit(&mut wr, cur, &mut dnext, &mut codes_since_clear);
|
||||
wr.write(EOD, width(dnext, early));
|
||||
wr.finish()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::CodecError;
|
||||
|
||||
fn dec(d: &[u8], early: bool) -> Result<Vec<u8>, CodecError> {
|
||||
decode(d, early, 1 << 26).map_err(|f| f.error)
|
||||
}
|
||||
|
||||
/// ISO 32000-2 §7.4.4.2 example: codes 256 45 258 258 65 259 66 257.
|
||||
const SPEC_IN: [u8; 10] = [45, 45, 45, 45, 45, 65, 45, 45, 45, 66];
|
||||
const SPEC_OUT: [u8; 9] = [0x80, 0x0B, 0x60, 0x50, 0x22, 0x0C, 0x0C, 0x85, 0x01];
|
||||
|
||||
#[test]
|
||||
fn spec_example() {
|
||||
assert_eq!(dec(&SPEC_OUT, true).unwrap(), SPEC_IN);
|
||||
assert_eq!(encode(&SPEC_IN, true), SPEC_OUT);
|
||||
// Below 511 codes the early-change setting makes no difference.
|
||||
assert_eq!(dec(&SPEC_OUT, false).unwrap(), SPEC_IN);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_eod_and_clear() {
|
||||
// Drop the EOD (last 9 bits span the final 2 bytes; keep bits for 66 only).
|
||||
let mut w = BitWriter { out: vec![], acc: 0, bits: 0 };
|
||||
for c in [45u16, 258, 258, 65, 259, 66] {
|
||||
w.write(c, 9);
|
||||
}
|
||||
assert_eq!(dec(&w.finish(), true).unwrap(), SPEC_IN);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_codes() {
|
||||
let mut w = BitWriter { out: vec![], acc: 0, bits: 0 };
|
||||
for c in [256u16, 65, 300] {
|
||||
w.write(c, 9);
|
||||
}
|
||||
let f = decode(&w.finish(), true, 1000).err().unwrap();
|
||||
assert!(matches!(f.error, CodecError::Corrupt { .. }));
|
||||
assert_eq!(f.partial, b"A");
|
||||
let mut w = BitWriter { out: vec![], acc: 0, bits: 0 };
|
||||
w.write(258, 9);
|
||||
assert!(matches!(dec(&w.finish(), true), Err(CodecError::Corrupt { .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn widths() {
|
||||
assert_eq!((width(510, true), width(511, true), width(511, false), width(512, false)), (9, 10, 9, 10));
|
||||
assert_eq!((width(1022, true), width(1023, true), width(2047, true), width(4095, false)), (10, 11, 12, 12));
|
||||
}
|
||||
|
||||
fn noisy(n: usize, seed: u32) -> Vec<u8> {
|
||||
let mut s = seed;
|
||||
(0..n)
|
||||
.map(|_| {
|
||||
s = s.wrapping_mul(1_103_515_245).wrapping_add(12_345);
|
||||
((s >> 16) % 7) as u8 * 31
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_widths_and_clears_round_trip() {
|
||||
for early in [false, true] {
|
||||
for data in [noisy(200_000, 1), (0..=255u8).cycle().take(70_000).collect(), vec![9u8; 100_000]] {
|
||||
let e = encode(&data, early);
|
||||
assert_eq!(dec(&e, early).unwrap(), data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cross_check_with_weezl() {
|
||||
use weezl::BitOrder::Msb;
|
||||
for data in [noisy(150_000, 7), SPEC_IN.to_vec(), vec![]] {
|
||||
// early change 1 is the TIFF size switch; 0 is the plain variant.
|
||||
let tiff = weezl::encode::Encoder::with_tiff_size_switch(Msb, 8).encode(&data).unwrap();
|
||||
assert_eq!(dec(&tiff, true).unwrap(), data);
|
||||
let plain = weezl::encode::Encoder::new(Msb, 8).encode(&data).unwrap();
|
||||
assert_eq!(dec(&plain, false).unwrap(), data);
|
||||
let ours = encode(&data, true);
|
||||
assert_eq!(weezl::decode::Decoder::with_tiff_size_switch(Msb, 8).decode(&ours).unwrap(), data);
|
||||
let ours = encode(&data, false);
|
||||
assert_eq!(weezl::decode::Decoder::new(Msb, 8).decode(&ours).unwrap(), data);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_table_without_clear() {
|
||||
// Hand-roll a stream that fills the table and keeps going at 12 bits.
|
||||
let data = noisy(40_000, 3);
|
||||
let mut w = BitWriter { out: vec![], acc: 0, bits: 0 };
|
||||
let mut dict: HashMap<Vec<u8>, u16> = (0..=255u8).map(|b| (vec![b], u16::from(b))).collect();
|
||||
let (mut next, mut dnext, mut since) = (FIRST, FIRST, 0usize);
|
||||
let mut cur: Vec<u8> = vec![];
|
||||
let mut put = |w: &mut BitWriter, code: u16, dnext: &mut usize| {
|
||||
w.write(code, width(*dnext, true));
|
||||
if since > 0 && *dnext < TABLE {
|
||||
*dnext += 1;
|
||||
}
|
||||
since += 1;
|
||||
};
|
||||
w.write(CLEAR, 9);
|
||||
for &b in &data {
|
||||
let mut ext = cur.clone();
|
||||
ext.push(b);
|
||||
if dict.contains_key(&ext) {
|
||||
cur = ext;
|
||||
continue;
|
||||
}
|
||||
put(&mut w, dict[&cur], &mut dnext);
|
||||
if next < TABLE {
|
||||
dict.insert(ext, next as u16);
|
||||
next += 1;
|
||||
}
|
||||
cur = vec![b];
|
||||
}
|
||||
put(&mut w, dict[&cur], &mut dnext);
|
||||
w.write(EOD, width(dnext, true));
|
||||
assert_eq!(dec(&w.finish(), true).unwrap(), data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limit() {
|
||||
let e = encode(&vec![0u8; 100_000], true);
|
||||
assert!(matches!(decode(&e, true, 50_000).map_err(|f| f.error), Err(CodecError::LimitExceeded(50_000))));
|
||||
assert_eq!(decode(&e, true, 100_000).ok().unwrap().len(), 100_000);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
//! The MQ arithmetic encoder of JBIG2 (ITU-T T.88 | ISO/IEC 14492, Annex E), written from the
|
||||
//! procedures in its Figures E.2–E.12 and the probability estimates of Table E.1.
|
||||
|
||||
/// Table E.1: for each index, the LPS probability estimate Qe, the next index after an MPS and
|
||||
/// after an LPS renormalisation, and whether an LPS switches the MPS sense.
|
||||
const STATES: [(u32, u8, u8, bool); 47] = [
|
||||
(0x5601, 1, 1, true),
|
||||
(0x3401, 2, 6, false),
|
||||
(0x1801, 3, 9, false),
|
||||
(0x0AC1, 4, 12, false),
|
||||
(0x0521, 5, 29, false),
|
||||
(0x0221, 38, 33, false),
|
||||
(0x5601, 7, 6, true),
|
||||
(0x5401, 8, 14, false),
|
||||
(0x4801, 9, 14, false),
|
||||
(0x3801, 10, 14, false),
|
||||
(0x3001, 11, 17, false),
|
||||
(0x2401, 12, 18, false),
|
||||
(0x1C01, 13, 20, false),
|
||||
(0x1601, 29, 21, false),
|
||||
(0x5601, 15, 14, true),
|
||||
(0x5401, 16, 14, false),
|
||||
(0x5101, 17, 15, false),
|
||||
(0x4801, 18, 16, false),
|
||||
(0x3801, 19, 17, false),
|
||||
(0x3401, 20, 18, false),
|
||||
(0x3001, 21, 19, false),
|
||||
(0x2801, 22, 19, false),
|
||||
(0x2401, 23, 20, false),
|
||||
(0x2201, 24, 21, false),
|
||||
(0x1C01, 25, 22, false),
|
||||
(0x1801, 26, 23, false),
|
||||
(0x1601, 27, 24, false),
|
||||
(0x1401, 28, 25, false),
|
||||
(0x1201, 29, 26, false),
|
||||
(0x1101, 30, 27, false),
|
||||
(0x0AC1, 31, 28, false),
|
||||
(0x09C1, 32, 29, false),
|
||||
(0x08A1, 33, 30, false),
|
||||
(0x0521, 34, 31, false),
|
||||
(0x0441, 35, 32, false),
|
||||
(0x02A1, 36, 33, false),
|
||||
(0x0221, 37, 34, false),
|
||||
(0x0141, 38, 35, false),
|
||||
(0x0111, 39, 36, false),
|
||||
(0x0085, 40, 37, false),
|
||||
(0x0049, 41, 38, false),
|
||||
(0x0025, 42, 39, false),
|
||||
(0x0015, 43, 40, false),
|
||||
(0x0009, 44, 41, false),
|
||||
(0x0005, 45, 42, false),
|
||||
(0x0001, 45, 43, false),
|
||||
(0x5601, 46, 46, false),
|
||||
];
|
||||
|
||||
#[cfg(test)]
|
||||
thread_local! {
|
||||
/// Decisions coded on this thread.
|
||||
pub(crate) static DECISIONS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
|
||||
}
|
||||
|
||||
/// A context's adaptive state: the index into [`STATES`] (low 7 bits) and the MPS sense (bit 7).
|
||||
/// 0, the initial value, is index 0 with MPS 0 (E.2.8).
|
||||
pub(crate) type Context = u8;
|
||||
|
||||
/// The encoder's registers (E.2.1) and the bytes written so far.
|
||||
pub(crate) struct MqEncoder {
|
||||
a: u32,
|
||||
c: u32,
|
||||
ct: u32,
|
||||
/// The output, after the byte before the first one (B at INITENC), which is 0x00 and is
|
||||
/// dropped by [`MqEncoder::finish`]: a carry can't reach it before the first byte is out.
|
||||
out: Vec<u8>,
|
||||
}
|
||||
|
||||
impl MqEncoder {
|
||||
/// INITENC (Figure E.10). The byte before the first is 0x00, not 0xFF, so CT starts at 12.
|
||||
pub(crate) fn new() -> Self {
|
||||
MqEncoder { a: 0x8000, c: 0, ct: 12, out: vec![0] }
|
||||
}
|
||||
|
||||
/// The byte B, the last one written (or the one before the first).
|
||||
fn b(&self) -> u8 {
|
||||
self.out.last().copied().unwrap_or(0)
|
||||
}
|
||||
|
||||
/// ENCODE (Figures E.3–E.7): code decision `d` in context `cx`, updating its state.
|
||||
pub(crate) fn encode(&mut self, cx: &mut Context, d: bool) {
|
||||
#[cfg(test)]
|
||||
DECISIONS.with(|n| n.set(n.get() + 1));
|
||||
let (index, mps) = (usize::from(*cx & 0x7F), *cx >> 7 == 1);
|
||||
let Some(&(qe, nmps, nlps, switch)) = STATES.get(index) else { return };
|
||||
self.a -= qe;
|
||||
if d == mps {
|
||||
// CODEMPS.
|
||||
if self.a & 0x8000 != 0 {
|
||||
self.c += qe;
|
||||
return;
|
||||
}
|
||||
if self.a < qe {
|
||||
self.a = qe;
|
||||
} else {
|
||||
self.c += qe;
|
||||
}
|
||||
*cx = nmps | (*cx & 0x80);
|
||||
} else {
|
||||
// CODELPS.
|
||||
if self.a < qe {
|
||||
self.c += qe;
|
||||
} else {
|
||||
self.a = qe;
|
||||
}
|
||||
let mps = mps != switch;
|
||||
*cx = nlps | (u8::from(mps) << 7);
|
||||
}
|
||||
self.renorm();
|
||||
}
|
||||
|
||||
/// RENORME (Figure E.8).
|
||||
fn renorm(&mut self) {
|
||||
loop {
|
||||
self.a <<= 1;
|
||||
self.c <<= 1;
|
||||
self.ct -= 1;
|
||||
if self.ct == 0 {
|
||||
self.byte_out();
|
||||
}
|
||||
if self.a & 0x8000 != 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// BYTEOUT (Figure E.9), with bit stuffing after 0xFF and carry into the byte before.
|
||||
fn byte_out(&mut self) {
|
||||
if self.b() == 0xFF {
|
||||
self.stuffed();
|
||||
} else if self.c < 0x800_0000 {
|
||||
self.plain();
|
||||
} else {
|
||||
let b = self.b().wrapping_add(1);
|
||||
if let Some(last) = self.out.last_mut() {
|
||||
*last = b;
|
||||
}
|
||||
if b == 0xFF {
|
||||
self.c &= 0x7FF_FFFF;
|
||||
self.stuffed();
|
||||
} else {
|
||||
self.plain();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn plain(&mut self) {
|
||||
self.out.push(((self.c >> 19) & 0xFF) as u8);
|
||||
self.c &= 0x7_FFFF;
|
||||
self.ct = 8;
|
||||
}
|
||||
|
||||
fn stuffed(&mut self) {
|
||||
self.out.push(((self.c >> 20) & 0xFF) as u8);
|
||||
self.c &= 0xF_FFFF;
|
||||
self.ct = 7;
|
||||
}
|
||||
|
||||
/// FLUSH (Figures E.11 and E.12): the final bytes, then the 0xFF 0xAC marker.
|
||||
pub(crate) fn finish(mut self) -> Vec<u8> {
|
||||
let top = self.c + self.a;
|
||||
self.c |= 0xFFFF;
|
||||
if self.c >= top {
|
||||
self.c -= 0x8000;
|
||||
}
|
||||
self.c <<= self.ct;
|
||||
self.byte_out();
|
||||
self.c <<= self.ct;
|
||||
self.byte_out();
|
||||
if self.b() != 0xFF {
|
||||
self.out.push(0xFF);
|
||||
}
|
||||
self.out.push(0xAC);
|
||||
self.out.split_off(1)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Annex H.2's test sequence: 256 decisions in one context give these 30 bytes.
|
||||
#[test]
|
||||
fn the_annex_h_test_sequence_encodes_to_its_bytes() {
|
||||
let decisions = [
|
||||
0x00, 0x02, 0x00, 0x51, 0x00, 0x00, 0x00, 0xC0, 0x03, 0x52, 0x87, 0x2A, 0xAA, 0xAA, 0xAA, 0xAA, 0x82, 0xC0, 0x20, 0x00, 0xFC, 0xD7, 0x9E,
|
||||
0xF6, 0xBF, 0x7F, 0xED, 0x90, 0x4F, 0x46, 0xA3, 0xBF,
|
||||
];
|
||||
let mut mq = MqEncoder::new();
|
||||
let mut cx = 0;
|
||||
for byte in decisions {
|
||||
for bit in (0..8).rev() {
|
||||
mq.encode(&mut cx, byte >> bit & 1 == 1);
|
||||
}
|
||||
}
|
||||
let expected = [
|
||||
0x84, 0xC7, 0x3B, 0xFC, 0xE1, 0xA1, 0x43, 0x04, 0x02, 0x20, 0x00, 0x00, 0x41, 0x0D, 0xBB, 0x86, 0xF4, 0x31, 0x7F, 0xFF, 0x88, 0xFF, 0x37,
|
||||
0x47, 0x1A, 0xDB, 0x6A, 0xDF, 0xFF, 0xAC,
|
||||
];
|
||||
assert_eq!(mq.finish(), expected);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
//! Predictor functions for FlateDecode and LZWDecode (ISO 32000-2 §7.4.4.4).
|
||||
//!
|
||||
//! - Predictor 2: TIFF 6.0 horizontal differencing (Section 14), per component, for
|
||||
//! 1, 2, 4, 8 and 16 bits per component.
|
||||
//! - Predictors 10–15: PNG filters (RFC 2083 §6): each row carries its own filter-type
|
||||
//! byte (0 None, 1 Sub, 2 Up, 3 Average, 4 Paeth), whatever the predictor value is.
|
||||
//!
|
||||
//! A final partial row is decoded as far as its data goes.
|
||||
//!
|
||||
//! The samples of an image whose rows carry no filter-type byte, although a PNG predictor is
|
||||
//! declared, are taken as they are and reported as damage ([`decode`]).
|
||||
|
||||
use crate::{CodecError, Failure, Params, Step};
|
||||
|
||||
struct Layout {
|
||||
bpc: usize,
|
||||
colors: usize,
|
||||
/// Bytes per complete pixel, at least 1 (the PNG `bpp`).
|
||||
bpp: usize,
|
||||
/// Bytes per row (without the PNG tag byte).
|
||||
row: usize,
|
||||
/// Samples per row (colors × columns).
|
||||
samples: usize,
|
||||
}
|
||||
|
||||
fn layout(p: &Params) -> Result<Layout, String> {
|
||||
let bpc = match p.bits_per_component {
|
||||
b @ (1 | 2 | 4 | 8 | 16) => b as usize,
|
||||
b => return Err(format!("invalid BitsPerComponent {b}")),
|
||||
};
|
||||
if !(1..=255).contains(&p.colors) {
|
||||
return Err(format!("invalid Colors {}", p.colors));
|
||||
}
|
||||
if p.columns < 1 {
|
||||
return Err(format!("invalid Columns {}", p.columns));
|
||||
}
|
||||
let colors = p.colors as usize;
|
||||
let samples = usize::try_from(p.columns).ok().and_then(|c| c.checked_mul(colors)).ok_or_else(|| format!("Columns {} too large", p.columns))?;
|
||||
let bits = samples.checked_mul(bpc).ok_or_else(|| format!("Columns {} too large", p.columns))?;
|
||||
Ok(Layout { bpc, colors, bpp: (colors * bpc).div_ceil(8), row: bits.div_ceil(8), samples })
|
||||
}
|
||||
|
||||
/// Undoes the predictor named by `p.predictor` (1 = identity). `rows` is the height of the image
|
||||
/// `data` is the samples of, if it is: with a PNG predictor, data of exactly `rows` rows without
|
||||
/// their filter-type bytes ([`untagged`]) is given back as it is, as a [`Failure`] whose partial
|
||||
/// output is all of it, so that tolerant decoding takes the samples and reports the repair, and
|
||||
/// strict decoding refuses them.
|
||||
pub fn decode(p: &Params, data: Vec<u8>, name: &'static str, rows: Option<usize>) -> Step {
|
||||
match p.predictor {
|
||||
1 => Ok(data),
|
||||
2 | 10..=15 => {
|
||||
let l = layout(p).map_err(|e| Failure::corrupt(name, e, Vec::new()))?;
|
||||
if p.predictor == 2 {
|
||||
return Ok(tiff_decode(&l, data));
|
||||
}
|
||||
if let Some(rows) = rows.filter(|&rows| untagged(&l, data.len(), rows)) {
|
||||
let detail =
|
||||
format!("PNG predictor {} declared, but its {rows} rows carry no filter-type bytes: read as unpredicted samples", p.predictor);
|
||||
return Err(Failure::corrupt(name, detail, data));
|
||||
}
|
||||
png_decode(&l, &data, name)
|
||||
}
|
||||
other => Err(Failure::corrupt(name, format!("invalid Predictor {other}"), Vec::new())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `len` bytes are exactly `rows` rows of `l` without the filter-type byte that starts each
|
||||
/// PNG-predicted row (ISO 32000-2 §7.4.4.4), where predicted data would be `rows` × (row + 1): a
|
||||
/// writer that declared the predictor and didn't apply it. Read with the predictor, every row's
|
||||
/// first sample was taken for a filter type and the image came out a row short and shifted.
|
||||
/// PDFium, MuPDF and Poppler read such data as the samples it is.
|
||||
fn untagged(l: &Layout, len: usize, rows: usize) -> bool {
|
||||
rows > 0 && l.row > 0 && rows.checked_mul(l.row) == Some(len)
|
||||
}
|
||||
|
||||
/// Applies the predictor named by `p.predictor` (1 = identity; 15 = per-row optimum).
|
||||
pub fn encode(p: &Params, data: &[u8]) -> Result<Vec<u8>, CodecError> {
|
||||
match p.predictor {
|
||||
1 => Ok(data.to_vec()),
|
||||
2 | 10..=15 => {
|
||||
let l = layout(p).map_err(CodecError::Unsupported)?;
|
||||
Ok(if p.predictor == 2 { tiff_encode(&l, data) } else { png_encode(&l, data, p.predictor) })
|
||||
}
|
||||
other => Err(CodecError::Unsupported(format!("Predictor {other}"))),
|
||||
}
|
||||
}
|
||||
|
||||
// ---- TIFF predictor 2 ----
|
||||
|
||||
fn get(row: &[u8], i: usize, bpc: usize) -> u16 {
|
||||
match bpc {
|
||||
16 => u16::from_be_bytes([row[2 * i], row[2 * i + 1]]),
|
||||
8 => u16::from(row[i]),
|
||||
_ => {
|
||||
let bit = i * bpc;
|
||||
let shift = 8 - bpc - bit % 8;
|
||||
u16::from(row[bit / 8] >> shift) & ((1 << bpc) - 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn set(row: &mut [u8], i: usize, bpc: usize, v: u16) {
|
||||
match bpc {
|
||||
16 => row[2 * i..2 * i + 2].copy_from_slice(&v.to_be_bytes()),
|
||||
8 => row[i] = v as u8,
|
||||
_ => {
|
||||
let bit = i * bpc;
|
||||
let shift = 8 - bpc - bit % 8;
|
||||
let mask = (((1u16 << bpc) - 1) << shift) as u8;
|
||||
row[bit / 8] = (row[bit / 8] & !mask) | (((v << shift) as u8) & mask);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Complete samples available in a (possibly partial) row.
|
||||
fn samples_in(l: &Layout, row_len: usize) -> usize {
|
||||
(row_len * 8 / l.bpc).min(l.samples)
|
||||
}
|
||||
|
||||
fn tiff_decode(l: &Layout, mut data: Vec<u8>) -> Vec<u8> {
|
||||
let mask = if l.bpc == 16 { u16::MAX } else { (1u16 << l.bpc) - 1 };
|
||||
for row in data.chunks_mut(l.row) {
|
||||
for i in l.colors..samples_in(l, row.len()) {
|
||||
let v = get(row, i, l.bpc).wrapping_add(get(row, i - l.colors, l.bpc)) & mask;
|
||||
set(row, i, l.bpc, v);
|
||||
}
|
||||
}
|
||||
data
|
||||
}
|
||||
|
||||
fn tiff_encode(l: &Layout, data: &[u8]) -> Vec<u8> {
|
||||
let mask = if l.bpc == 16 { u16::MAX } else { (1u16 << l.bpc) - 1 };
|
||||
let mut out = data.to_vec();
|
||||
for row in out.chunks_mut(l.row) {
|
||||
// Descending, so the left neighbour still holds its original value.
|
||||
for i in (l.colors..samples_in(l, row.len())).rev() {
|
||||
let v = get(row, i, l.bpc).wrapping_sub(get(row, i - l.colors, l.bpc)) & mask;
|
||||
set(row, i, l.bpc, v);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// ---- PNG predictors ----
|
||||
|
||||
fn paeth(a: u8, b: u8, c: u8) -> u8 {
|
||||
let p = i16::from(a) + i16::from(b) - i16::from(c);
|
||||
let (pa, pb, pc) = ((p - i16::from(a)).abs(), (p - i16::from(b)).abs(), (p - i16::from(c)).abs());
|
||||
if pa <= pb && pa <= pc {
|
||||
a
|
||||
} else if pb <= pc {
|
||||
b
|
||||
} else {
|
||||
c
|
||||
}
|
||||
}
|
||||
|
||||
/// The PNG predictor value for byte `i` given the reconstructed current row `cur`
|
||||
/// (bytes before `i`) and the prior row `up`.
|
||||
fn predict(tag: u8, cur: &[u8], up: &[u8], i: usize, bpp: usize) -> u8 {
|
||||
let a = if i >= bpp { cur[i - bpp] } else { 0 };
|
||||
let b = up[i];
|
||||
let c = if i >= bpp { up[i - bpp] } else { 0 };
|
||||
match tag {
|
||||
1 => a,
|
||||
2 => b,
|
||||
3 => ((u16::from(a) + u16::from(b)) / 2) as u8,
|
||||
4 => paeth(a, b, c),
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn png_decode(l: &Layout, data: &[u8], name: &'static str) -> Step {
|
||||
let mut out = Vec::with_capacity(data.len());
|
||||
// The prior row: never longer than the input, even for absurd Columns values.
|
||||
let mut up = vec![0u8; l.row.min(data.len())];
|
||||
for chunk in data.chunks(l.row.saturating_add(1)) {
|
||||
let (tag, src) = (chunk[0], &chunk[1..]);
|
||||
if tag > 4 {
|
||||
return Err(Failure::corrupt(name, format!("invalid PNG filter type {tag}"), out));
|
||||
}
|
||||
let start = out.len();
|
||||
for (i, &x) in src.iter().enumerate() {
|
||||
let pred = predict(tag, &out[start..], &up, i, l.bpp);
|
||||
out.push(x.wrapping_add(pred));
|
||||
}
|
||||
up[..src.len()].copy_from_slice(&out[start..]);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn png_filter(tag: u8, cur: &[u8], up: &[u8], bpp: usize, dst: &mut Vec<u8>) {
|
||||
dst.clear();
|
||||
dst.push(tag);
|
||||
dst.extend(cur.iter().enumerate().map(|(i, &x)| x.wrapping_sub(predict(tag, cur, up, i, bpp))));
|
||||
}
|
||||
|
||||
/// Sum of residuals as signed bytes: the usual heuristic for choosing a PNG filter.
|
||||
fn cost(filtered: &[u8]) -> u64 {
|
||||
filtered[1..].iter().map(|&b| u64::from((b as i8).unsigned_abs())).sum()
|
||||
}
|
||||
|
||||
fn png_encode(l: &Layout, data: &[u8], predictor: i64) -> Vec<u8> {
|
||||
let rows = data.len().div_ceil(l.row.max(1));
|
||||
let mut out = Vec::with_capacity(data.len() + rows);
|
||||
let zeros = vec![0u8; l.row.min(data.len())];
|
||||
let (mut best, mut trial) = (Vec::new(), Vec::new());
|
||||
let mut up: &[u8] = &zeros;
|
||||
for cur in data.chunks(l.row) {
|
||||
let up_row = &up[..cur.len()];
|
||||
if predictor == 15 {
|
||||
png_filter(0, cur, up_row, l.bpp, &mut best);
|
||||
let mut best_cost = cost(&best);
|
||||
for tag in 1..=4 {
|
||||
png_filter(tag, cur, up_row, l.bpp, &mut trial);
|
||||
let c = cost(&trial);
|
||||
if c < best_cost {
|
||||
best_cost = c;
|
||||
std::mem::swap(&mut best, &mut trial);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
png_filter((predictor - 10) as u8, cur, up_row, l.bpp, &mut best);
|
||||
}
|
||||
out.extend_from_slice(&best);
|
||||
up = cur;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn p(predictor: i64, colors: i64, bpc: i64, columns: i64) -> Params {
|
||||
Params { predictor, colors, bits_per_component: bpc, columns, early_change: 1 }
|
||||
}
|
||||
|
||||
fn dec(params: &Params, data: &[u8]) -> Result<Vec<u8>, CodecError> {
|
||||
decode(params, data.to_vec(), "FlateDecode", None).map_err(|f| f.error)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn png_hand_vectors() {
|
||||
// 3 columns, 1 color, 8 bpc. Rows: None, Sub, Up, Average, Paeth.
|
||||
let enc = [
|
||||
0, 10, 20, 30, // None -> 10 20 30
|
||||
1, 5, 1, 1, // Sub -> 5 6 7
|
||||
2, 1, 1, 1, // Up -> 6 7 8
|
||||
3, 10, 10, 10, // Avg -> 10+3=13, 10+(13+7)/2=20, 10+(20+8)/2=24
|
||||
4, 1, 1,
|
||||
1, // Paeth: a=0,b=13,c=0 -> p=13 -> b: 14; a=14,b=20,c=13: p=21 -> pa=7,pb=1,pc=8 -> b: 21;
|
||||
// a=21,b=24,c=20: p=25 -> pa=4,pb=1,pc=5 -> b: 25
|
||||
];
|
||||
let out = dec(&p(15, 1, 8, 3), &enc).unwrap();
|
||||
assert_eq!(out, [10, 20, 30, 5, 6, 7, 6, 7, 8, 13, 20, 24, 14, 21, 25]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn paeth_tie_breaks() {
|
||||
// RFC 2083 order: a, then b, then c.
|
||||
assert_eq!(paeth(1, 1, 1), 1);
|
||||
assert_eq!(paeth(10, 20, 15), 15); // p=15: pa=5 pb=5 pc=0 -> c
|
||||
assert_eq!(paeth(20, 10, 10), 20); // p=20: pa=0 -> a
|
||||
assert_eq!(paeth(5, 9, 7), 7); // p=7: pa=2 pb=2 pc=0 -> c
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn png_multibyte_pixels() {
|
||||
// colors=3 bpc=16 -> bpp=6; 1 column; Sub on the first pixel uses a=0 throughout.
|
||||
let enc = [1, 1, 2, 3, 4, 5, 6, 2, 1, 1, 1, 1, 1, 1];
|
||||
let out = dec(&p(11, 3, 16, 1), &enc).unwrap();
|
||||
assert_eq!(out, [1, 2, 3, 4, 5, 6, 2, 3, 4, 5, 6, 7]);
|
||||
// colors=3 bpc=8, 2 columns: Sub looks 3 bytes back.
|
||||
let enc = [1, 1, 2, 3, 1, 1, 1];
|
||||
assert_eq!(dec(&p(11, 3, 8, 2), &enc).unwrap(), [1, 2, 3, 2, 3, 4]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn png_partial_final_row() {
|
||||
let enc = [0, 1, 2, 3, 4, 2, 1, 1];
|
||||
assert_eq!(dec(&p(12, 1, 8, 4), &enc).unwrap(), [1, 2, 3, 4, 2, 3]);
|
||||
// A lone tag byte at the end yields nothing.
|
||||
assert_eq!(dec(&p(12, 1, 8, 4), &[0, 1, 2, 3, 4, 2]).unwrap(), [1, 2, 3, 4]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn png_invalid_tag() {
|
||||
let r = decode(&p(10, 1, 8, 2), vec![0, 1, 2, 5, 1, 1], "FlateDecode", None);
|
||||
let f = r.err().unwrap();
|
||||
assert!(matches!(f.error, CodecError::Corrupt { .. }));
|
||||
assert_eq!(f.partial, [1, 2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiff_8bit() {
|
||||
// 2 colors, 3 columns: differences per component.
|
||||
let enc = [10, 20, 1, 2, 3, 4, /* row 2 */ 5, 5, 250, 10, 1, 1];
|
||||
assert_eq!(dec(&p(2, 2, 8, 3), &enc).unwrap(), [10, 20, 11, 22, 14, 26, 5, 5, 255, 15, 0, 16]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiff_16bit() {
|
||||
let enc = [0x01, 0x00, 0xff, 0xff, 0x00, 0x02];
|
||||
assert_eq!(dec(&p(2, 1, 16, 3), &enc).unwrap(), [0x01, 0x00, 0x00, 0xff, 0x01, 0x01]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiff_sub_byte() {
|
||||
// 1 bpc, 1 color, 10 columns: bits 1 0 0 1 1 0 0 0 | 1 1 (pad 000000)
|
||||
// Cumulative XOR: 1 1 1 0 1 1 1 1 | 0 1
|
||||
let enc = [0b1001_1000, 0b1100_0000];
|
||||
assert_eq!(dec(&p(2, 1, 1, 10), &enc).unwrap(), [0b1110_1111, 0b0100_0000]);
|
||||
// 4 bpc, 1 color, 3 columns: samples 5, 15, 3 -> 5, 4, 7 (mod 16).
|
||||
assert_eq!(dec(&p(2, 1, 4, 3), &[0x5f, 0x30]).unwrap(), [0x54, 0x70]);
|
||||
// 2 bpc, 2 colors, 2 columns: samples 1 2 3 3 -> 1 2 0 1.
|
||||
assert_eq!(dec(&p(2, 2, 2, 2), &[0b0110_1111]).unwrap(), [0b0110_0001]);
|
||||
// Padding bits are left alone.
|
||||
assert_eq!(dec(&p(2, 1, 4, 1), &[0x5f]).unwrap(), [0x5f]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiff_partial_rows() {
|
||||
// 16-bit with an odd trailing byte: the lone byte is left as is.
|
||||
assert_eq!(dec(&p(2, 1, 16, 4), &[0, 1, 0, 1, 9]).unwrap(), [0, 1, 0, 2, 9]);
|
||||
assert_eq!(dec(&p(2, 1, 8, 4), &[1, 1, 1, 1, 2, 2]).unwrap(), [1, 2, 3, 4, 2, 4]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_params() {
|
||||
for bad in [p(2, 1, 3, 1), p(12, 0, 8, 1), p(12, 1, 8, 0), p(3, 1, 8, 1), p(12, 1, 8, i64::MAX), p(2, 300, 8, 1)] {
|
||||
assert!(matches!(dec(&bad, &[0, 1, 2]), Err(CodecError::Corrupt { .. })), "{bad:?}");
|
||||
assert!(encode(&bad, &[1, 2]).is_err());
|
||||
}
|
||||
// Huge but representable Columns does not allocate a huge row.
|
||||
assert_eq!(dec(&p(12, 1, 8, 1 << 40), &[2, 7, 8]).unwrap(), [7, 8]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encode_choices() {
|
||||
let data = [1u8, 2, 3, 4, 5, 6];
|
||||
let up = encode(&p(12, 1, 8, 3), &data).unwrap();
|
||||
assert_eq!(up, [2, 1, 2, 3, 2, 3, 3, 3]);
|
||||
let paeth = encode(&p(14, 1, 8, 3), &data).unwrap();
|
||||
assert_eq!(paeth[0], 4);
|
||||
assert_eq!(paeth[4], 4);
|
||||
// Per-row optimum picks Sub for a ramp.
|
||||
let ramp: Vec<u8> = (0..64).collect();
|
||||
let opt = encode(&p(15, 1, 8, 64), &ramp).unwrap();
|
||||
assert_eq!(opt[0], 1);
|
||||
for pred in [10, 11, 12, 13, 14, 15] {
|
||||
let e = encode(&p(pred, 1, 8, 3), &data).unwrap();
|
||||
assert_eq!(dec(&p(pred, 1, 8, 3), &e).unwrap(), data);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
//! RunLengthDecode (ISO 32000-2 §7.4.5): a length byte L of 0–127 copies the next
|
||||
//! L + 1 bytes, 129–255 repeats the next byte 257 − L times, 128 is EOD.
|
||||
//! A missing EOD is tolerated; a truncated run is corrupt.
|
||||
|
||||
use crate::{Failure, Step};
|
||||
|
||||
const NAME: &str = "RunLengthDecode";
|
||||
|
||||
pub fn decode(data: &[u8], max: usize) -> Step {
|
||||
let mut out = Vec::with_capacity(data.len().saturating_mul(2).min(max));
|
||||
let mut i = 0;
|
||||
while let Some(&l) = data.get(i) {
|
||||
i += 1;
|
||||
match l {
|
||||
128 => break,
|
||||
0..=127 => {
|
||||
let n = usize::from(l) + 1;
|
||||
let avail = n.min(data.len() - i);
|
||||
if out.len() + avail > max {
|
||||
return Err(Failure::limit(max));
|
||||
}
|
||||
out.extend_from_slice(&data[i..i + avail]);
|
||||
i += avail;
|
||||
if avail < n {
|
||||
return Err(Failure::corrupt(NAME, "truncated literal run", out));
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
let Some(&b) = data.get(i) else {
|
||||
return Err(Failure::corrupt(NAME, "truncated repeat run", out));
|
||||
};
|
||||
i += 1;
|
||||
let n = 257 - usize::from(l);
|
||||
if out.len() + n > max {
|
||||
return Err(Failure::limit(max));
|
||||
}
|
||||
out.resize(out.len() + n, b);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub fn encode(data: &[u8]) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(data.len() + data.len() / 128 + 2);
|
||||
let n = data.len();
|
||||
let mut i = 0;
|
||||
while i < n {
|
||||
let mut run = 1;
|
||||
while i + run < n && run < 128 && data[i + run] == data[i] {
|
||||
run += 1;
|
||||
}
|
||||
if run >= 2 {
|
||||
out.push((257 - run) as u8);
|
||||
out.push(data[i]);
|
||||
i += run;
|
||||
continue;
|
||||
}
|
||||
// Literal: extend until a repeat begins or 128 bytes.
|
||||
let start = i;
|
||||
let mut j = i + 1;
|
||||
while j < n && j - start < 128 && !(j + 1 < n && data[j] == data[j + 1]) {
|
||||
j += 1;
|
||||
}
|
||||
out.push((j - start - 1) as u8);
|
||||
out.extend_from_slice(&data[start..j]);
|
||||
i = j;
|
||||
}
|
||||
out.push(128);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::CodecError;
|
||||
|
||||
fn dec(d: &[u8]) -> Result<Vec<u8>, CodecError> {
|
||||
decode(d, 10_000).map_err(|f| f.error)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vectors() {
|
||||
assert_eq!(dec(&[2, b'a', b'b', b'c', 254, b'x', 128]).unwrap(), b"abcxxx");
|
||||
assert_eq!(dec(&[129, 7, 128]).unwrap(), [7; 128]);
|
||||
assert!(dec(&[127]).is_err());
|
||||
assert_eq!(dec(&[0, b'q']).unwrap(), b"q"); // missing EOD
|
||||
assert_eq!(dec(&[128, 0, b'q']).unwrap(), b""); // data after EOD ignored
|
||||
assert_eq!(dec(&[]).unwrap(), b"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated() {
|
||||
let f = decode(&[4, b'a', b'b'], 100).err().unwrap();
|
||||
assert!(matches!(f.error, CodecError::Corrupt { .. }));
|
||||
assert_eq!(f.partial, b"ab");
|
||||
assert!(matches!(dec(&[0, b'a', 200]), Err(CodecError::Corrupt { .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limit() {
|
||||
assert!(matches!(decode(&[129, 0, 129, 0], 200).map_err(|f| f.error), Err(CodecError::LimitExceeded(200))));
|
||||
assert!(matches!(decode(&[3, 1, 2, 3, 4], 3).map_err(|f| f.error), Err(CodecError::LimitExceeded(3))));
|
||||
assert_eq!(decode(&[129, 0, 129, 0], 256).ok().unwrap().len(), 256);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoding() {
|
||||
assert_eq!(encode(b"abcxxx"), [2, b'a', b'b', b'c', 254, b'x', 128]);
|
||||
assert_eq!(encode(b""), [128]);
|
||||
assert_eq!(encode(&[5; 300]), [129, 5, 129, 5, 213, 5, 128]);
|
||||
let lit: Vec<u8> = (0..200u8).collect();
|
||||
let e = encode(&lit);
|
||||
assert_eq!((e[0], e[129]), (127, 71));
|
||||
assert_eq!(dec(&e).unwrap(), lit);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user