diff --git a/vendor/README.craft-reader.md b/vendor/README.craft-reader.md index f9f281ab..b8271db1 100644 --- a/vendor/README.craft-reader.md +++ b/vendor/README.craft-reader.md @@ -401,3 +401,26 @@ of 85c10051 for hayro-syntax was already carried here. PdfCraft's raster tests a `jbig2_placements_at_the_end_of_an_i32_draw_nothing`); the unit tests in `lib.rs`, `bitmap.rs` and `decode/symbol.rs` come with the patches. PdfCraft's later JBIG2 follow-ups (b7ec3669, 72e763ad, e3bc4e85, patch 13 and later) are not part of this port. + +JBIG2 follow-ups, ported from PdfCraft after the decode-bound patches above, each marked +`PdfCraft patch` in `hayro-jbig2`: [72e763ad](https://github.com/storytold/pdfcraft/commit/72e763ad1) +(a symbol with no rows costs a step; a refined instance is charged first), +[b7ec3669](https://github.com/storytold/pdfcraft/commit/b7ec3669c) (what an item reads is charged), +[e3bc4e85](https://github.com/storytold/pdfcraft/commit/e3bc4e855) (a height class of no symbols +decodes no bitmap), [0ed33990](https://github.com/storytold/pdfcraft/commit/0ed339901) (only its +`hayro-jbig2` part: `Image::decode_within` takes the work left, so callers can share one allowance), +[01d3479f](https://github.com/storytold/pdfcraft/commit/01d3479f2), [b9e068aa](https://github.com/storytold/pdfcraft/commit/b9e068aa3), +[858ce960](https://github.com/storytold/pdfcraft/commit/858ce9608), +[07f44038](https://github.com/storytold/pdfcraft/commit/07f440388), +[e38b9d18](https://github.com/storytold/pdfcraft/commit/e38b9d182) and +[8717a085](https://github.com/storytold/pdfcraft/commit/8717a085f) (a region past `i32::MAX` is off the page; a +refinement refines the bitmap it is placed on, the page only in its area, and reads 0 past its +reference's width and a reference right of it from its first column), +[58c30af3](https://github.com/storytold/pdfcraft/commit/58c30af31) (a stream that starts with a JBIG2 +file header draws) and the `hayro-jbig2` part of +[e2464e14](https://github.com/storytold/pdfcraft/commit/e2464e144) (a dictionary may hold 2^28 symbol +pixels, as a real 14,642 x 8,879 scan needs; `OverflowError::BitmapLimit` and `DecodeError::is_limit` tell +a decoder's limit from damage). Not ported from those commits: the page-wide allowance and the +refused-mask handling in `hayro-syntax` and `hayro-interpret` (0ed33990, 4aa732ea, e2464e14), which rest on +the decoded-image cache of 558ffd7f (a performance change). Tests: those that came with the patches in +`hayro-jbig2`, and PdfCraft's raster tests adapted in `hayro-jbig2/src/pdfcraft_tests.rs`. diff --git a/vendor/hayro-jbig2/SOURCE.json b/vendor/hayro-jbig2/SOURCE.json index 5efee12a..0568ac87 100644 --- a/vendor/hayro-jbig2/SOURCE.json +++ b/vendor/hayro-jbig2/SOURCE.json @@ -62,7 +62,7 @@ "path": "src/bitmap.rs", "source_sha256": "59ed83d8bac6c39e40270a9856ca3faee628de7d9efae1b01f714e140e2b6b7e", "source_git_blob": "35ed97561597d30a70fe819cf8cfbd3780538f75", - "sha256": "dd330bda095350b91eeb15b029bdae209e1ed3d5c711dff279b79abd7832f847" + "sha256": "f5fa055ed19f6113d517875479ed4d4d732e06a0010067883a4c38820d55ff97" }, { "path": "src/decode/generic.rs", @@ -74,7 +74,7 @@ "path": "src/decode/generic_refinement.rs", "source_sha256": "625075671e510e8ff1d7c3cade857792413c091d57a35561ad0591d7f04c2ea5", "source_git_blob": "a566b8de58c078dec874d5d1dea36c0f557f1a16", - "sha256": "93e2ce1a40c35fed24d4d611cfc1763518e00167b4e3c0a0671a18afa37c7064" + "sha256": "7a4e36aa8b371f532e33163aad59133f8afa946935570b2312d057cd2a8b3837" }, { "path": "src/decode/halftone.rs", @@ -98,31 +98,31 @@ "path": "src/decode/symbol.rs", "source_sha256": "aa57e88cdc6fd1647d7996875c253a643fedfc4438d5137e326e00d17d0d29ee", "source_git_blob": "f1aa8a6639e53dd350057d8d078198d363daa918", - "sha256": "627de1c83267597efef6d6c0d49aec004f43fd6e63490f117f744ed9210b47f0" + "sha256": "793e9943183b5ef119aa281dc13e782083fdac16e9b4c598083af1a95209256f" }, { "path": "src/decode/text.rs", "source_sha256": "01863cdcd48ab68b5230948d12212a815d293c7c9d5382a140772088bfb1c405", "source_git_blob": "bbc2ac55c7d7d5b8aebffbecfe93fda8b88526e8", - "sha256": "32ae695fe181ba07181fa9915b6bce2ca333031c392fae66dd15ce955a8d7b95" + "sha256": "1f75a55d76111980d667713f841aa38a1fc20918e5bdcfef08a3a36842e2ca10" }, { "path": "src/error.rs", "source_sha256": "39b6ec8fa058bc5c1722892290882e123d193d3844ed5846b30b742b94ba72b0", "source_git_blob": "c579368b44b444566178e68a2705cf9561fad71f", - "sha256": "326dd926cc52b191d0c0398b3b2bb482ca267919e369045f129930cfa485e7fb" + "sha256": "7bc0f28cf64b0510819a50037566b6eeeba1cf2654d095417743cf0b00e2eae8" }, { "path": "src/file.rs", "source_sha256": "10467a415ab2e8b92a8951a86f4c02c5c74f5a005629a11865a20bc23b26ca55", "source_git_blob": "1440a22efa89f0deec0029968c3a20bad8526d63", - "sha256": "c96741babbfacfd78bd2b63f0f16541aaa235f14bb4344ecd4707e3ca5942df1" + "sha256": "7b85b531dbb37ddad69d620e54934a3f88cb88a5aaf1dcd4209c990b92fdbd69" }, { "path": "src/gray_scale.rs", "source_sha256": "46f12893f4573610789015ba7036f6afb1d30b1454c0c674fc7ccdd9337d7487", "source_git_blob": "2836ec77a7c8d3970b13dd46f576778c64bfe2d9", - "sha256": "4bb0963338bfc9e3773db822465d5acbc1267c3349f04b30bff8c52e254874bb" + "sha256": "f82a6760de85bab325cffa6c75a0fa4e1d55fb9c67617be0d0fa112f2dd6c15f" }, { "path": "src/huffman_table.rs", @@ -158,7 +158,7 @@ "path": "src/lib.rs", "source_sha256": "22463230cc82543ee93791b642566a308f0f4eb14d3089b24dc2e7f06d0f784c", "source_git_blob": "ad77a6dd17920b2db66f6bb5a0f2cb8efaafd41a", - "sha256": "d0431d1c57c552b250b7b48895f0d768bfc75496b30cd0146b77764b09759c0f" + "sha256": "ac97e81ca5ec61d5d40247b561d0c1b17d7ac80aa173ee1bde099eb90e52a82f" }, { "path": "src/page_info.rs", diff --git a/vendor/hayro-jbig2/src/bitmap.rs b/vendor/hayro-jbig2/src/bitmap.rs index a240ca43..a4008314 100644 --- a/vendor/hayro-jbig2/src/bitmap.rs +++ b/vendor/hayro-jbig2/src/bitmap.rs @@ -7,6 +7,7 @@ use crate::decode::CombinationOperator; use crate::error::{OverflowError, Result, bail}; +use alloc::vec; use alloc::vec::Vec; // To guard better against malicious files, we limit bitmap dimensions @@ -30,6 +31,12 @@ pub(crate) const MAX_GRID_CELLS: u64 = 1 << 25; /// halftone 265 s, and a 923-byte PDF's regions would have taken days. pub(crate) const MAX_WORK: u64 = 4 * MAX_PIXELS; +/// PdfCraft patch: the work of reading one integer of an item, what an arithmetic-coded one takes +/// at worst in decisions (see [`MAX_WORK`]), so that a decision costs no more than a pixel. An item +/// with no pixels still reads its integers: charged a step each, a 22,515-byte PDF of refined text +/// instances of no size stayed inside `MAX_WORK` and took 730 s. +pub(crate) const INTEGER_WORK: u64 = 38; + /// PdfCraft patch: the work of a line of a Huffman table built for symbols, about 50 ns (see /// [`MAX_WORK`]). pub(crate) const HUFFMAN_LINE_WORK: u64 = 10; @@ -63,10 +70,10 @@ pub(crate) const BITMAP_WORDS: usize = size_of::() / size_of::(); /// output from the page header in between, and a 743-byte PDF's Indexed image asked for 137 GB. pub(crate) fn check_page_size(width: u32, height: u32) -> Result<()> { if width > MAX_DIMENSION || height > MAX_DIMENSION { - return Err(OverflowError::BitmapDimension.into()); + return Err(OverflowError::BitmapLimit.into()); } if u64::from(width) * u64::from(height) > MAX_PIXELS { - return Err(OverflowError::BitmapDimension.into()); + return Err(OverflowError::BitmapLimit.into()); } Ok(()) } @@ -86,6 +93,11 @@ pub(crate) struct Bitmap { pub(crate) stride: u32, /// Packed pixel data, one bit per pixel, row-major order. /// Each row is padded to a word boundary. + /// + /// PdfCraft patch: the padding is 0s, as pixels past the edge are (6.2.5.2, 6.3.5.2). Generic + /// and refinement decoding read the rows above their own pixels a word at a time, padding + /// included, and a refinement reads its reference the same way: whatever writes whole words + /// keeps the padding 0 (see `clear_padding`). pub(crate) data: Vec, /// "This four-byte field gives the horizontal offset in pixels of the bitmap /// encoded in this segment relative to the page bitmap." (7.4.1.3) @@ -95,14 +107,6 @@ pub(crate) struct Bitmap { pub(crate) y_location: u32, } -/// PdfCraft patch: fallible replacement for `vec![word; len]` / `resize`. -fn fill_words(data: &mut Vec, len: usize, word: Word) -> Result<()> { - data.try_reserve_exact(len) - .map_err(|_| OverflowError::BitmapDimension)?; - data.resize(len, word); - Ok(()) -} - impl Bitmap { /// PdfCraft patch: the words this bitmap holds, its header included. pub(crate) fn words(&self) -> usize { @@ -125,26 +129,47 @@ impl Bitmap { default_pixel: bool, ) -> Result { if width > MAX_DIMENSION || height > MAX_DIMENSION { - bail!(OverflowError::BitmapDimension); + bail!(OverflowError::BitmapLimit); } // PdfCraft patch: see `MAX_PIXELS`. if u64::from(width) * u64::from(height) > MAX_PIXELS { - bail!(OverflowError::BitmapDimension); + bail!(OverflowError::BitmapLimit); } let stride = width.div_ceil(WORD_BITS); let default_word: Word = if default_pixel { !0 } else { 0 }; - // PdfCraft patch: reserve fallibly so hostile dimensions error instead of aborting. - let mut data = Vec::new(); - fill_words(&mut data, stride as usize * height as usize, default_word)?; - Ok(Self { + let data = vec![default_word; stride as usize * height as usize]; + let mut bitmap = Self { width, height, stride, data, x_location, y_location, - }) + }; + if default_pixel { + bitmap.clear_padding(); + } + Ok(bitmap) + } + + /// PdfCraft patch: clear the bits past `width` in each row's last word. "All pixels lying + /// outside the bounds of the actual bitmap or the reference bitmap have the value 0" (6.3.5.2), + /// and a refinement reads its reference a word at a time, those bits included: filled whole + /// with a default pixel of 1, they read as 1s, so a refinement of the page, or of an + /// intermediate text or halftone region, decoded to other pixels than were coded. + fn clear_padding(&mut self) { + let trailing = self.width % WORD_BITS; + // A width of 0 has no words to clear, and `chunks_exact_mut` would take a stride of 0. + if trailing == 0 { + return; + } + let mask: Word = !0 << (WORD_BITS - trailing); + for row in self.data.chunks_exact_mut(self.stride as usize) { + if let Some(last) = row.last_mut() { + *last &= mask; + } + } } pub(crate) fn reinitialize( @@ -157,15 +182,17 @@ impl Bitmap { let stride = width.div_ceil(WORD_BITS); let default_word: Word = if default_pixel { !0 } else { 0 }; - // Allocate first so a failed reservation leaves the bitmap untouched. - let mut data = Vec::new(); - fill_words(&mut data, stride as usize * height as usize, default_word)?; self.width = width; self.height = height; self.stride = stride; self.x_location = 0; self.y_location = 0; - self.data = data; + self.data.clear(); + self.data + .resize(stride as usize * height as usize, default_word); + if default_pixel { + self.clear_padding(); + } Ok(()) } @@ -194,6 +221,17 @@ impl Bitmap { self.data[word_idx] |= (value as Word) << bit_pos; } + /// PdfCraft patch: combine a decoded region into this page at the region's own location. + /// + /// "Unless otherwise specified, all multi-bit values shall be treated as unsigned values" + /// (5.4.4), and a region's location (7.4.1.3, 7.4.1.4) is not specified otherwise. One past + /// `i32::MAX` is past any page. Cast to a signed coordinate it read as left of the page or above + /// it, and one within a region's size of 2^32 came back onto it. + pub(crate) fn combine_region(&mut self, region: &Self, operator: CombinationOperator) { + let signed = |location: u32| i32::try_from(location).unwrap_or(i32::MAX); + self.combine(region, signed(region.x_location), signed(region.y_location), operator); + } + /// Combine another bitmap into this one at a specific location. /// /// "These operators describe how the segment's bitmap is to be combined with @@ -280,15 +318,74 @@ impl Bitmap { self.data[(row * self.stride + word_idx) as usize] } + + /// PdfCraft patch: the `width` × `height` area of this bitmap at `x`, `y`, as a bitmap there. + /// It is what a refinement of the page refines: "a bitmap containing the current contents of + /// the page buffer, restricted to the area of the page buffer specified by this segment's + /// region segment information field" (7.4.7.4). Off this bitmap the area is 0, past its width + /// too whatever the padding of its rows holds, and outside the area the refinement reads 0 + /// (6.3.5.2), where reading the page itself took in the pixels around the area. + pub(crate) fn area(&self, x: u32, y: u32, width: u32, height: u32) -> Result { + let mut area = Self::new_with(width, height, x, y, false)?; + for row in 0..height { + let Some(from_row) = y.checked_add(row).filter(|r| *r < self.height) else { + continue; + }; + for word in 0..area.stride { + // The first column of this word of the area, in this bitmap; past its width, + // or past where a column can be, the rest of the row is 0. + let Some(from) = x.checked_add(word * WORD_BITS).filter(|c| *c < self.width) else { + break; + }; + let (index, shift) = (from / WORD_BITS, from % WORD_BITS); + let mut bits = self.get_word(from_row, index) << shift; + if shift > 0 { + bits |= self.get_word(from_row, index + 1) >> (WORD_BITS - shift); + } + // Both are above 0: `from` is inside this bitmap, and `word` inside the area. + let ours = self.width - from; + let theirs = width - word * WORD_BITS; + let kept = ours.min(theirs); + if kept < WORD_BITS { + bits &= !0 << (WORD_BITS - kept); + } + if let Some(slot) = area.data.get_mut((row * area.stride + word) as usize) { + *slot = bits; + } + } + } + Ok(area) + } } #[cfg(test)] -mod tests { +mod pdfcraft_limit_tests { use super::*; + use crate::error::DecodeError; + /// PdfCraft patch: a bitmap past the bound on a side or on pixels is refused by a limit. #[test] - fn failed_reservation_is_an_error() { - let mut data = Vec::new(); - assert!(fill_words(&mut data, usize::MAX / 2, 0).is_err()); + fn a_bitmap_past_its_bounds_is_refused_by_a_limit() { + let limit = Err(DecodeError::Overflow(OverflowError::BitmapLimit)); + assert_eq!( + Bitmap::new(MAX_DIMENSION + 1, 1).map(|_| ()), + limit, + "a side" + ); + assert_eq!( + Bitmap::new(1, MAX_DIMENSION + 1).map(|_| ()), + limit, + "the other" + ); + // 2^28 + 2^14 pixels, each side within its bound. + assert_eq!( + Bitmap::new(1 << 14, (1 << 14) + 1).map(|_| ()), + limit, + "the pixels" + ); + assert!( + Bitmap::new(1 << 14, 1 << 14).is_ok(), + "2^28 pixels are within it" + ); } } diff --git a/vendor/hayro-jbig2/src/decode/generic_refinement.rs b/vendor/hayro-jbig2/src/decode/generic_refinement.rs index a6634439..cd730fd9 100644 --- a/vendor/hayro-jbig2/src/decode/generic_refinement.rs +++ b/vendor/hayro-jbig2/src/decode/generic_refinement.rs @@ -10,7 +10,7 @@ use crate::ScratchBuffers; use crate::arithmetic_decoder::{ArithmeticDecoder, ArithmeticDecoderContext}; use crate::bitmap::{Bitmap, WORD_BITS, WORD_SHIFT, Word}; use crate::decode::generic::ContextGatherer; -use crate::error::{OverflowError, ParseError, RegionError, Result, bail}; +use crate::error::{ParseError, RegionError, Result, bail}; use crate::reader::Reader; /// Generic refinement region decoding procedure (6.3). @@ -46,27 +46,18 @@ pub(crate) fn decode_into( // Validate that the region fits within the reference bitmap. // When referring to another segment, dimensions must match exactly (7.4.7.5). - // When using the page bitmap as reference, the region must fit within the page. + // PdfCraft patch: a refinement of the page refines the page's area of its own size (7.4.7.4, + // `Bitmap::area`), which it always fits. if header.region_info.width > reference.width || header.region_info.height > reference.height { bail!(RegionError::InvalidDimension); } - let reference_dx = i32::try_from(reference.x_location) - .ok() - .and_then(|r: i32| { - i32::try_from(header.region_info.x_location) - .ok() - .and_then(|h| r.checked_sub(h)) - }) - .ok_or(OverflowError::ReferenceOffset)?; - let reference_dy = i32::try_from(reference.y_location) - .ok() - .and_then(|r: i32| { - i32::try_from(header.region_info.y_location) - .ok() - .and_then(|h| r.checked_sub(h)) - }) - .ok_or(OverflowError::ReferenceOffset)?; + // PdfCraft patch: "GRREFERENCEDX 0, GRREFERENCEDY 0" (Table 35). The reference is another + // region, which a refinement refines where it is (their locations should be equal, 7.4.7.5), + // or the page restricted to the refinement's area (7.4.7.4, `Bitmap::area`). Offsetting it by + // the difference of the locations refused one past `i32::MAX` as an overflow, and read the + // page around a refinement of it. + let (reference_dx, reference_dy) = (0, 0); let mut decoder = ArithmeticDecoder::new(data); let num_context_bits = header.template.context_bits(); @@ -194,10 +185,24 @@ impl<'a> RefinementContextGatherer<'a> { }; self.reg_cur = 0; - self.ref_cur_x = 0; - self.ref_m1 = ContextGatherer::load_word(reference, ref_y.wrapping_sub(1) as u32, 0); - self.ref_cur = ContextGatherer::load_word(reference, ref_y as u32, 0); - self.ref_p1 = ContextGatherer::load_word(reference, ref_y.wrapping_add(1) as u32, 0); + // PdfCraft patch: the buffers start a column left of the reference column the row starts + // at, as `maybe_reload_buffers` would load them, or at column 0 where that is left of the + // reference. The row's first context takes in that column and the one left of it, which + // from column 32 on lay outside a word loaded from column 0 and read as 0: with the + // reference offset 32 left of the region its first column, and from 33 both (GRREFERENCEDX + // below −31, which a refined text instance has from a narrower bitmap or an RDX below −31, + // and a refined symbol from such an RDX). The region decoded to other pixels than were + // coded, or failed. + self.ref_cur_x = self + .reference_dx + .checked_neg() + .and_then(|column| u32::try_from(column).ok()) + .map_or(0, |column| column.saturating_sub(1)); + self.ref_m1 = + ContextGatherer::load_word(reference, ref_y.wrapping_sub(1) as u32, self.ref_cur_x); + self.ref_cur = ContextGatherer::load_word(reference, ref_y as u32, self.ref_cur_x); + self.ref_p1 = + ContextGatherer::load_word(reference, ref_y.wrapping_add(1) as u32, self.ref_cur_x); self.ctx = match self.template { RefinementTemplate::Template1 => { @@ -259,25 +264,25 @@ impl<'a> RefinementContextGatherer<'a> { self.reg_cur = ContextGatherer::load_word(region, self.reg_y, new_start); } + // PdfCraft patch: left of the reference the buffers keep what `start_row` loaded, its words + // from column 0. The context takes in the reference a column to the right of the current + // one (Figures 12 and 13), so at column −1 it reads column 0, and from 0 on the buffers are + // the reference's own. They were zeroed here and, at column 0, not loaded again: with the + // reference offset right of the region (GRREFERENCEDX above 0, which a refined text + // instance or symbol has from a wider bitmap or an RDX above 0), its first 32 columns read + // as 0, and the region decoded to other pixels than were coded, or as garbage that failed. let ref_x_signed = (x as i32).wrapping_sub(self.reference_dx); - if ref_x_signed < 0 { - self.ref_cur_x = 0; - self.ref_m1 = 0; - self.ref_cur = 0; - self.ref_p1 = 0; - } else { - let ref_x = ref_x_signed as u32; - - if ref_x + 1 >= self.ref_cur_x + WORD_BITS || ref_x < self.ref_cur_x { - let new_start = ref_x.saturating_sub(1); - let ref_y = (self.reg_y as i32).wrapping_sub(self.reference_dy); - self.ref_cur_x = new_start; - self.ref_m1 = - ContextGatherer::load_word(reference, ref_y.wrapping_sub(1) as u32, new_start); - self.ref_cur = ContextGatherer::load_word(reference, ref_y as u32, new_start); - self.ref_p1 = - ContextGatherer::load_word(reference, ref_y.wrapping_add(1) as u32, new_start); - } + if let Ok(ref_x) = u32::try_from(ref_x_signed) + && (ref_x + 1 >= self.ref_cur_x + WORD_BITS || ref_x < self.ref_cur_x) + { + let new_start = ref_x.saturating_sub(1); + let ref_y = (self.reg_y as i32).wrapping_sub(self.reference_dy); + self.ref_cur_x = new_start; + self.ref_m1 = + ContextGatherer::load_word(reference, ref_y.wrapping_sub(1) as u32, new_start); + self.ref_cur = ContextGatherer::load_word(reference, ref_y as u32, new_start); + self.ref_p1 = + ContextGatherer::load_word(reference, ref_y.wrapping_add(1) as u32, new_start); } } diff --git a/vendor/hayro-jbig2/src/decode/symbol.rs b/vendor/hayro-jbig2/src/decode/symbol.rs index 94250a3d..6eefa056 100644 --- a/vendor/hayro-jbig2/src/decode/symbol.rs +++ b/vendor/hayro-jbig2/src/decode/symbol.rs @@ -4,7 +4,7 @@ use alloc::vec; use alloc::vec::Vec; use crate::arithmetic_decoder::{ArithmeticDecoder, ArithmeticDecoderContext}; -use crate::bitmap::{BITMAP_WORDS, Bitmap, HUFFMAN_LINE_WORK, WORD_BITS, Word, decode_work}; +use crate::bitmap::{BITMAP_WORDS, Bitmap, HUFFMAN_LINE_WORK, INTEGER_WORK, WORD_BITS, Word, decode_work}; use crate::decode::generic::{decode_bitmap_mmr, parse_adaptive_template_pixels}; use crate::decode::text::{ DecodeContext, ReferenceCorner, TextRegionContexts, TextRegionFlags, TextRegionHeader, @@ -24,10 +24,14 @@ use crate::reader::Reader; const MAX_SYMBOLS: u32 = u16::MAX as u32; -/// PdfCraft patch: the most symbol pixels one dictionary may decode (2^26). Symbols are -/// decoded pixel by pixel, and up to 65535 of them of up to 65535 × 65535 each kept a fuzzed -/// file decoding for minutes; real dictionaries hold about a million pixels. -const MAX_DICTIONARY_PIXELS: u64 = 1 << 26; +/// PdfCraft patch: the most symbol pixels one dictionary may decode, as many as the largest page +/// bitmap holds (2^28, `bitmap::MAX_PIXELS`). Symbols are decoded pixel by pixel, and up to 65535 +/// of them of up to 65535 × 65535 each kept a fuzzed file decoding for minutes; each symbol now +/// pays for its pixels out of the stream's decoding work (`bitmap::MAX_WORK`), which bounds the +/// time, and holds them in the retained bitmaps (`room`), which bounds the memory. The bound was +/// 2^26, a quarter of this, which a real scan passes: the soft mask of a 14,642 × 8,879 drawing +/// with one dictionary of 7,559 symbols (#6722) needed more, and was refused. +const MAX_DICTIONARY_PIXELS: u64 = crate::bitmap::MAX_PIXELS; /// PdfCraft patch: decoding steps that decode nothing, allowed beyond what a valid dictionary can /// need. A height class without a symbol and an export run of length 0 don't move their loops @@ -101,6 +105,8 @@ pub(crate) fn decode( bail!(SymbolError::Invalid); } height_classes_left -= 1; + // PdfCraft patch: see `bitmap::INTEGER_WORK`. A class of no symbols still reads its delta. + *ctx.work_left = ctx.work_left.checked_sub(INTEGER_WORK).ok_or(OverflowError::DecodingWork)?; let height_class_delta = read_height_class_delta(&mut ctx)?.ok_or(SymbolError::UnexpectedOob)?; @@ -133,10 +139,13 @@ pub(crate) fn decode( .pixels .saturating_add(u64::from(symbol_width) * u64::from(ctx.height_class_height)); if ctx.pixels > MAX_DICTIONARY_PIXELS { - bail!(OverflowError::BitmapDimension); + bail!(OverflowError::BitmapLimit); } - // PdfCraft patch: see `bitmap::MAX_WORK`. - let work = decode_work(symbol_width, ctx.height_class_height); + // PdfCraft patch: see `bitmap::INTEGER_WORK`. Its width, and a stream can hold thousands + // of dictionaries of symbols of no pixels. An aggregated one reads its count, a symbol + // and two offsets as well. + let integers = if ctx.header.flags.use_refagg { 5 } else { 1 }; + let work = decode_work(symbol_width, ctx.height_class_height).saturating_add(integers * INTEGER_WORK); *ctx.work_left = ctx.work_left.checked_sub(work).ok_or(OverflowError::DecodingWork)?; // PdfCraft patch: see `room`. Rows take a word at least, so symbols of one column // held 270 MiB within the pixel bound, however few the dictionary exported. @@ -652,6 +661,17 @@ fn decode_height_class_collective_bitmap(ctx: &mut SymbolDecodeContext<'_>) -> R ctx.h_ctx.reader.align(); + // PdfCraft patch: a class of no symbols has nothing to cut out, and `bitmap::decode_work` charges + // rows by the symbol, so decoding this would be rows for nothing: 65,535 a class, and a dictionary + // can hold a thousand such classes. Its bytes still have to be stepped over. + if ctx.symbols_decoded_count == ctx.height_class_first_symbol { + ctx.h_ctx + .reader + .read_bytes(bitmap_size as usize) + .ok_or(ParseError::UnexpectedEof)?; + return Ok(()); + } + let collective_bitmap = if bitmap_size == 0 { // Bitmap is stored uncompressed, so we basically just copy-paste // each byte. Rows are padded to the byte boundary. @@ -759,6 +779,8 @@ fn export_symbols(ctx: &mut SymbolDecodeContext<'_>) -> Result> { bail!(SymbolError::Invalid); } runs_left -= 1; + // PdfCraft patch: see `bitmap::INTEGER_WORK`. A run of no symbols still reads its length. + *ctx.work_left = ctx.work_left.checked_sub(INTEGER_WORK).ok_or(OverflowError::DecodingWork)?; let run_length = read_run_length()?; let end_index = index.checked_add(run_length).ok_or(OverflowError::Index)?; diff --git a/vendor/hayro-jbig2/src/decode/text.rs b/vendor/hayro-jbig2/src/decode/text.rs index 71924acf..5406ffeb 100644 --- a/vendor/hayro-jbig2/src/decode/text.rs +++ b/vendor/hayro-jbig2/src/decode/text.rs @@ -11,7 +11,7 @@ use super::{ use super::{RegionBitmap, generic_refinement}; use crate::ScratchBuffers; use crate::arithmetic_decoder::{ArithmeticDecoder, ArithmeticDecoderContext}; -use crate::bitmap::{Bitmap, copy_work, decode_work}; +use crate::bitmap::{Bitmap, INTEGER_WORK, copy_work, decode_work}; use crate::error::{HuffmanError, OverflowError, ParseError, Result, SymbolError, bail}; use crate::huffman_table::{HuffmanTable, StandardHuffmanTables, TableLine}; use crate::integer_decoder::IntegerDecoder; @@ -151,7 +151,7 @@ pub(crate) fn decode_with( let symbol_id = ctx.read_symbol_id()?; let symbol_bitmap = - decode_symbol_instance_bitmap(&mut ctx, symbols, header, symbol_id)?; + decode_symbol_instance_bitmap(&mut ctx, symbols, header, symbol_id, work_left)?; let symbol_bitmap_ref: &Bitmap = match &symbol_bitmap { SymbolBitmap::Reference(idx) => symbols.get(*idx).ok_or(SymbolError::OutOfRange)?, @@ -161,14 +161,8 @@ pub(crate) fn decode_with( let symbol_height = symbol_bitmap_ref.height as i32; // PdfCraft patch: see `bitmap::MAX_WORK`. An instance of a large symbol costs its copy, - // and 10,000 of one took 64 s. A refined one was decoded as well. Even one with no - // pixels goes round this loop. - let (width, height) = (symbol_bitmap_ref.width, symbol_bitmap_ref.height); - let copied = copy_work(width, height).saturating_add(1); - let work = match &symbol_bitmap { - SymbolBitmap::Reference(_) => copied, - SymbolBitmap::Owned(_) => copied.saturating_add(decode_work(width, height)), - }; + // and 10,000 of one took 64 s. Even one with no pixels reads its four integers. + let work = copy_work(symbol_bitmap_ref.width, symbol_bitmap_ref.height).saturating_add(4 * INTEGER_WORK); *work_left = work_left.checked_sub(work).ok_or(OverflowError::DecodingWork)?; if !header.flags.transposed @@ -549,6 +543,7 @@ fn decode_symbol_instance_bitmap( symbols: &[&Bitmap], header: &TextRegionHeader<'_>, symbol_id: usize, + work_left: &mut u64, ) -> Result { if !header.flags.use_refinement || ctx.read_refinement_flag()? == 0 { return Ok(SymbolBitmap::Reference(symbol_id)); @@ -580,6 +575,11 @@ fn decode_symbol_instance_bitmap( .checked_add(rdy) .ok_or(OverflowError::ReferenceOffset)?; + // PdfCraft patch: see `bitmap::MAX_WORK`. Before it is decoded: charged after, one instance + // could still be 2^28 pixels past what was left. With the four integers that gave its size. + let work = decode_work(refined_width, refined_height).saturating_add(4 * INTEGER_WORK); + *work_left = work_left.checked_sub(work).ok_or(OverflowError::DecodingWork)?; + let mut refined_bitmap = Bitmap::new(refined_width, refined_height)?; ctx.decode_refinement_bitmap( diff --git a/vendor/hayro-jbig2/src/error.rs b/vendor/hayro-jbig2/src/error.rs index c8bcf8b8..a91074e5 100644 --- a/vendor/hayro-jbig2/src/error.rs +++ b/vendor/hayro-jbig2/src/error.rs @@ -141,6 +141,10 @@ pub enum OverflowError { SegmentCount, /// PdfCraft patch: a stream asks for more decoding work than it may. DecodingWork, + /// PdfCraft patch: a bitmap, gray-scale image or symbol dictionary is larger than the decoder + /// holds (see `bitmap::MAX_PIXELS`): one of its limits, where `BitmapDimension` is arithmetic + /// that overflows. + BitmapLimit, } impl fmt::Display for DecodeError { @@ -171,6 +175,7 @@ impl fmt::Display for OverflowError { Self::RetainedBitmaps => write!(f, "the segments retain too many bitmaps"), Self::SegmentCount => write!(f, "too many segments or references in a stream"), Self::DecodingWork => write!(f, "a stream asks for too much decoding work"), + Self::BitmapLimit => write!(f, "a bitmap is larger than the decoder holds"), } } } @@ -312,6 +317,59 @@ impl From for DecodeError { } } +impl DecodeError { + /// PdfCraft patch: whether one of the decoder's limits refused the stream (on a bitmap's size, + /// a dictionary's symbols, a region's instances, the bitmaps retained, the segments or the + /// decoding work) rather than its data failing to decode. A file within the format can reach + /// these, so a caller can tell a refusal of its own from a damaged file. A size or count the + /// file damaged can read as past one of them, and is a limit here too: it can't be told from a + /// hostile one. + pub fn is_limit(&self) -> bool { + matches!( + self, + Self::Overflow( + OverflowError::BitmapLimit + | OverflowError::GridDimension + | OverflowError::SymbolCount + | OverflowError::RetainedBitmaps + | OverflowError::SegmentCount + | OverflowError::DecodingWork + ) | Self::Symbol(SymbolError::TooManyInstances) + ) + } +} + +#[cfg(test)] +mod pdfcraft_limit_tests { + use super::*; + + /// PdfCraft patch: the decoder's own limits are limits, and arithmetic that overflows, + /// damage, is not. + #[test] + fn the_decoders_limits_are_limits_and_overflowing_arithmetic_is_not() { + let limits = [ + DecodeError::Overflow(OverflowError::BitmapLimit), + DecodeError::Overflow(OverflowError::GridDimension), + DecodeError::Overflow(OverflowError::SymbolCount), + DecodeError::Overflow(OverflowError::RetainedBitmaps), + DecodeError::Overflow(OverflowError::SegmentCount), + DecodeError::Overflow(OverflowError::DecodingWork), + DecodeError::Symbol(SymbolError::TooManyInstances), + ]; + for limit in limits { + assert!(limit.is_limit(), "{limit:?} is a limit"); + } + let damage = [ + DecodeError::Overflow(OverflowError::BitmapDimension), + DecodeError::Overflow(OverflowError::PlacementCoordinate), + DecodeError::Symbol(SymbolError::TooManySymbols), + ]; + for damage in damage { + assert!(!damage.is_limit(), "{damage:?} is damage"); + } + } +} + /// Result type for JBIG2 decoding operations. pub type Result = core::result::Result; diff --git a/vendor/hayro-jbig2/src/file.rs b/vendor/hayro-jbig2/src/file.rs index 5b8bbe98..c5b59b27 100644 --- a/vendor/hayro-jbig2/src/file.rs +++ b/vendor/hayro-jbig2/src/file.rs @@ -88,6 +88,19 @@ pub(crate) struct File<'a> { /// (D.4.1) const FILE_HEADER_ID: [u8; 8] = [0x97, 0x4A, 0x42, 0x32, 0x0D, 0x0A, 0x1A, 0x0A]; +/// PdfCraft patch (17): an embedded stream that starts with a file header (D.4), which some +/// producers write into a PDF's `JBIG2Decode` data although ISO 32000-1 7.4.7 has the embedded +/// organization (D.3) there. Returns the reader past the header and the organization the header +/// declares, or `None` when the data doesn't start with the header's ID string. +pub(crate) fn after_file_header(data: &[u8]) -> Result, FileOrganization)>> { + if !data.starts_with(&FILE_HEADER_ID) { + return Ok(None); + } + let mut reader = Reader::new(data); + let header = parse_file_header(&mut reader)?; + Ok(Some((reader, header.organization))) +} + /// Parse a standalone JBIG2 file. pub(crate) fn parse_file(data: &[u8]) -> Result> { let mut reader = Reader::new(data); @@ -170,7 +183,9 @@ fn parse_segments<'a>( FileOrganization::Sequential => { parse_segments_sequential(reader, &mut segments, &mut SegmentCount::default())?; } - FileOrganization::RandomAccess => parse_segments_random_access(reader, &mut segments)?, + FileOrganization::RandomAccess => { + parse_segments_random_access(reader, &mut segments, &mut SegmentCount::default())?; + } } Ok(segments) @@ -213,12 +228,12 @@ pub(crate) fn parse_segments_sequential<'a>( /// is followed by a sequence of segments headers; the last segment header is /// followed by the data for the first segment, then the data for the second /// segment, and so on." (D.2) -fn parse_segments_random_access<'a>( +pub(crate) fn parse_segments_random_access<'a>( reader: &mut Reader<'a>, segments: &mut Vec>, + count: &mut SegmentCount, ) -> Result<()> { let mut headers = Vec::new(); - let mut count = SegmentCount::default(); loop { if reader.at_end() { diff --git a/vendor/hayro-jbig2/src/gray_scale.rs b/vendor/hayro-jbig2/src/gray_scale.rs index 4aa4229a..1cd5016f 100644 --- a/vendor/hayro-jbig2/src/gray_scale.rs +++ b/vendor/hayro-jbig2/src/gray_scale.rs @@ -81,11 +81,11 @@ fn decode_arithmetic( let height = params.height; if width > MAX_DIMENSION || height > MAX_DIMENSION { - bail!(OverflowError::BitmapDimension); + bail!(OverflowError::BitmapLimit); } // PdfCraft patch: the product is bounded too (see `bitmap::MAX_GRID_CELLS`). if u64::from(width) * u64::from(height) > crate::bitmap::MAX_GRID_CELLS { - bail!(OverflowError::BitmapDimension); + bail!(OverflowError::BitmapLimit); } // `GSBPP` - The number of bits per gray-scale value. @@ -289,3 +289,34 @@ fn extract_bits( } } } + +#[cfg(test)] +mod pdfcraft_limit_tests { + use super::*; + use crate::error::DecodeError; + + /// PdfCraft patch: a gray-scale image past the bound on a side or on cells is refused by a + /// limit, before anything is decoded. A halftone region refuses such a grid first, so this is + /// reached only by calling it. + #[test] + fn a_gray_scale_image_past_its_bounds_is_refused_by_a_limit() { + let decode = |width, height| { + let params = GrayScaleParams { + use_mmr: false, + bits_per_pixel: 1, + width, + height, + template: Template::Template0, + skip_mask: None, + }; + decode_gray_scale_image(&[], ¶ms, &mut ScratchBuffers::default()).map(|_| ()) + }; + let limit = Err(DecodeError::Overflow(OverflowError::BitmapLimit)); + assert_eq!(decode(MAX_DIMENSION + 1, 1), limit, "a side"); + assert_eq!( + decode(1 << 13, (1 << 12) + 1), + limit, + "the cells, each side within its bound" + ); + } +} diff --git a/vendor/hayro-jbig2/src/lib.rs b/vendor/hayro-jbig2/src/lib.rs index 078370ed..6ec7fd83 100644 --- a/vendor/hayro-jbig2/src/lib.rs +++ b/vendor/hayro-jbig2/src/lib.rs @@ -78,7 +78,10 @@ pub use error::{ SegmentError, SymbolError, TemplateError, }; -use crate::file::{SegmentCount, parse_segments_sequential}; +use crate::file::{ + FileOrganization, SegmentCount, after_file_header, parse_segments_random_access, + parse_segments_sequential, +}; use bitmap::Bitmap; use decode::CombinationOperator; use decode::generic; @@ -120,7 +123,9 @@ impl<'a> Image<'a> { /// Parse an embedded JBIG2 image with optional global segments. /// /// The file is expected to use the embedded organization defined in - /// Annex D.3. + /// Annex D.3. PdfCraft patch (17): data that starts with a file header (D.4) + /// is read past it, in the organization it declares, as Acrobat and PDFBox + /// draw such streams; its segments still count together with the globals'. pub fn new_embedded(data: &'a [u8], globals: Option<&'a [u8]>) -> Result { let mut segments = Vec::new(); let mut count = SegmentCount::default(); @@ -129,8 +134,18 @@ impl<'a> Image<'a> { parse_segments_sequential(&mut reader, &mut segments, &mut count)?; }; - let mut reader = Reader::new(data); - parse_segments_sequential(&mut reader, &mut segments, &mut count)?; + match after_file_header(data)? { + Some((mut reader, FileOrganization::RandomAccess)) => { + parse_segments_random_access(&mut reader, &mut segments, &mut count)?; + } + Some((mut reader, FileOrganization::Sequential)) => { + parse_segments_sequential(&mut reader, &mut segments, &mut count)?; + } + None => { + let mut reader = Reader::new(data); + parse_segments_sequential(&mut reader, &mut segments, &mut count)?; + } + } segments.sort_by_key(|seg| seg.header.segment_number); @@ -198,11 +213,37 @@ impl<'a> Image<'a> { /// This is useful in case you want to convert multiple JBIG2 images, /// as it allows `hayro-jbig2` to reuse allocations during decoding. pub fn decode_with(&self, decoder: &mut D, ctx: &mut DecoderContext) -> Result<()> { - decode_segments(&self.segments, self.height_from_stripes, ctx)?; + let mut work_left = bitmap::MAX_WORK; + decode_segments(&self.segments, self.height_from_stripes, ctx, &mut work_left)?; emit_bitmap(&ctx.page_bitmap, decoder); Ok(()) } + + /// PdfCraft patch: [`Image::decode`], spending from `work`, what decoding may still cost + /// across a run of images. An image spends what it took, failing or not, starting with its + /// page's pixels, which whoever asked for them will go through, and its segments, so images + /// that decode nothing add up too. On top of that, it takes at most [`bitmap::MAX_WORK`] + /// whatever `work` says. One that `work` can't pay for spends all of it, so a run ends rather + /// than trying every image it has left. Reading the stream comes before this and is charged + /// by [`charge_reading`]. + pub fn decode_within(&self, decoder: &mut D, work: &mut u64) -> Result<()> { + let start = bitmap::decode_work(self.width, self.height).saturating_add(self.segments.len() as u64); + if charge_work(work, start).is_err() { + *work = 0; + bail!(OverflowError::DecodingWork); + } + let allowed = bitmap::MAX_WORK.min(*work); + let mut left = allowed; + let mut ctx = DecoderContext::default(); + let decoded = decode_segments(&self.segments, self.height_from_stripes, &mut ctx, &mut left); + let refused = decoded == Err(OverflowError::DecodingWork.into()); + *work = if refused && allowed < bitmap::MAX_WORK { 0 } else { *work - (allowed - left) }; + decoded?; + emit_bitmap(&ctx.page_bitmap, &mut *decoder); + + Ok(()) + } } fn emit_bitmap(bitmap: &Bitmap, decoder: &mut D) { @@ -262,22 +303,46 @@ fn emit_bitmap(bitmap: &Bitmap, decoder: &mut D) { } } -/// PdfCraft patch: count `work` more decoding work, refusing past [`bitmap::MAX_WORK`] before what -/// asks for it is decoded. +/// PdfCraft patch: count `work` more decoding work, refusing past what is `left` of +/// [`bitmap::MAX_WORK`] or a run's allowance before what asks for it is decoded. fn charge_work(left: &mut u64, work: u64) -> Result<()> { *left = left.checked_sub(work).ok_or(OverflowError::DecodingWork)?; Ok(()) } +/// PdfCraft patch: the steps of reading a byte of a stream or its globals: parsing and sorting +/// segments, and building their tables, measured at about 3 a byte at the worst. +const READ_WORK: u64 = 4; + +/// PdfCraft patch: charge `work`, what a run of images may still cost, for reading `bytes` of a stream +/// and its globals. This comes before anything is parsed, as it counts whether or not they turn out +/// to decode: an image that fails has still cost its reading, and it is read again for each use. What +/// `work` can't pay for spends all of it. +pub fn charge_reading(work: &mut u64, bytes: usize) -> Result<()> { + if charge_work(work, (bytes as u64).saturating_mul(READ_WORK)).is_err() { + *work = 0; + bail!(OverflowError::DecodingWork); + } + Ok(()) +} + /// PdfCraft patch: the work of a region decoded pixel by pixel. fn region_work(region: &decode::RegionSegmentInfo) -> u64 { bitmap::decode_work(region.width, region.height) } +/// PdfCraft patch: the page restricted to a refinement's area, its reference when it refers to no +/// other region (7.4.7.4). Copying it a word at a time costs less than the region's own decoding, +/// already charged, and it is held, as large as the region, only while the region decodes. +fn page_area(page: &Bitmap, region: &decode::RegionSegmentInfo) -> Result { + page.area(region.x_location, region.y_location, region.width, region.height) +} + fn decode_segments( segments: &[segment::Segment<'_>], height_from_stripes: Option, decoder_ctx: &mut DecoderContext, + work_left: &mut u64, ) -> Result<()> { // Find and parse page information segment first. if let Some(page_info) = segments @@ -298,7 +363,6 @@ fn decode_segments( let page_bitmap = &mut decoder_ctx.page_bitmap; let page_state = &mut decoder_ctx.page_state; let scratch_buffers = &mut decoder_ctx.scratch_buffers; - let mut work_left = bitmap::MAX_WORK; // Process all segments. for seg in segments { @@ -311,31 +375,26 @@ fn decode_segments( SegmentType::ImmediateGenericRegion | SegmentType::ImmediateLosslessGenericRegion => { let had_unknown_length = seg.header.data_length.is_none(); let header = generic::parse(&mut reader, had_unknown_length)?; - charge_work(&mut work_left, region_work(&header.region_info))?; + charge_work(work_left, region_work(&header.region_info))?; if page_state.can_decode_directly(page_bitmap, &header.region_info, false) { generic::decode_into(&header, page_bitmap, scratch_buffers)?; } else { let region = generic::decode(&header, scratch_buffers)?; - page_bitmap.combine( - ®ion.bitmap, - region.bitmap.x_location as i32, - region.bitmap.y_location as i32, - region.combination_operator, - ); + page_bitmap.combine_region(®ion.bitmap, region.combination_operator); } page_state.page_pristine = false; } SegmentType::IntermediateGenericRegion => { // Intermediate segments cannot have unknown length. let header = generic::parse(&mut reader, false)?; - charge_work(&mut work_left, region_work(&header.region_info))?; + charge_work(work_left, region_work(&header.region_info))?; let region = generic::decode(&header, scratch_buffers)?; page_state.store_region(seg.header.segment_number, region.bitmap)?; } SegmentType::PatternDictionary => { let header = pattern::parse(&mut reader)?; - charge_work(&mut work_left, header.work())?; + charge_work(work_left, header.work())?; let dictionary = pattern::decode(&header, scratch_buffers, page_state.retained_room())?; page_state.store_pattern_dictionary(seg.header.segment_number, dictionary)?; } @@ -344,7 +403,7 @@ fn decode_segments( // (6.5.5, step 1) // Collect references to avoid cloning; symbols are only cloned if re-exported. let input_symbols = page_state.referred_symbols(&seg.header.referred_to_segments)?; - charge_work(&mut work_left, input_symbols.len() as u64)?; + charge_work(work_left, input_symbols.len() as u64)?; // Collect Huffman tables from referred table segments. let referred_tables: Vec = seg @@ -371,14 +430,14 @@ fn decode_segments( &page_state.standard_tables, retained_contexts, page_state.retained_room(), - &mut work_left, + work_left, )?; page_state.store_symbol_dictionary(seg.header.segment_number, dictionary)?; } SegmentType::ImmediateTextRegion | SegmentType::ImmediateLosslessTextRegion => { // Collect symbols from referred symbol dictionaries (SBSYMS). let symbols = page_state.referred_symbols(&seg.header.referred_to_segments)?; - charge_work(&mut work_left, symbols.len() as u64)?; + charge_work(work_left, symbols.len() as u64)?; // Collect Huffman tables from referred table segments. // "These user-supplied Huffman decoding tables may be supplied either @@ -392,10 +451,10 @@ fn decode_segments( .collect(); let header = text::parse(&mut reader, symbols.len() as u32)?; - charge_work(&mut work_left, bitmap::copy_work(header.region_info.width, header.region_info.height))?; + charge_work(work_left, bitmap::copy_work(header.region_info.width, header.region_info.height))?; if header.flags.use_huffman { // Its symbol ID table has a line for each symbol. - charge_work(&mut work_left, (symbols.len() as u64).saturating_mul(bitmap::HUFFMAN_LINE_WORK))?; + charge_work(work_left, (symbols.len() as u64).saturating_mul(bitmap::HUFFMAN_LINE_WORK))?; } if page_state.can_decode_directly( @@ -410,7 +469,7 @@ fn decode_segments( &page_state.standard_tables, page_bitmap, scratch_buffers, - &mut work_left, + work_left, )?; } else { let region = text::decode( @@ -419,21 +478,16 @@ fn decode_segments( &referred_tables, &page_state.standard_tables, scratch_buffers, - &mut work_left, + work_left, )?; - page_bitmap.combine( - ®ion.bitmap, - region.bitmap.x_location as i32, - region.bitmap.y_location as i32, - region.combination_operator, - ); + page_bitmap.combine_region(®ion.bitmap, region.combination_operator); } page_state.page_pristine = false; } SegmentType::IntermediateTextRegion => { // Collect symbols from referred symbol dictionaries (SBSYMS). let symbols = page_state.referred_symbols(&seg.header.referred_to_segments)?; - charge_work(&mut work_left, symbols.len() as u64)?; + charge_work(work_left, symbols.len() as u64)?; // Collect Huffman tables from referred table segments. let referred_tables: Vec = seg @@ -445,10 +499,10 @@ fn decode_segments( .collect(); let header = text::parse(&mut reader, symbols.len() as u32)?; - charge_work(&mut work_left, bitmap::copy_work(header.region_info.width, header.region_info.height))?; + charge_work(work_left, bitmap::copy_work(header.region_info.width, header.region_info.height))?; if header.flags.use_huffman { // Its symbol ID table has a line for each symbol. - charge_work(&mut work_left, (symbols.len() as u64).saturating_mul(bitmap::HUFFMAN_LINE_WORK))?; + charge_work(work_left, (symbols.len() as u64).saturating_mul(bitmap::HUFFMAN_LINE_WORK))?; } let region = text::decode( &header, @@ -456,7 +510,7 @@ fn decode_segments( &referred_tables, &page_state.standard_tables, scratch_buffers, - &mut work_left, + work_left, )?; page_state.store_region(seg.header.segment_number, region.bitmap)?; } @@ -469,7 +523,7 @@ fn decode_segments( .ok_or(SegmentError::MissingPatternDictionary)?; let header = halftone::parse(&mut reader)?; - charge_work(&mut work_left, halftone::work(&header, pattern_dict))?; + charge_work(work_left, halftone::work(&header, pattern_dict))?; if page_state.can_decode_directly( page_bitmap, @@ -479,12 +533,7 @@ fn decode_segments( halftone::decode_into(&header, pattern_dict, page_bitmap, scratch_buffers)?; } else { let region = halftone::decode(&header, pattern_dict, scratch_buffers)?; - page_bitmap.combine( - ®ion.bitmap, - region.bitmap.x_location as i32, - region.bitmap.y_location as i32, - region.combination_operator, - ); + page_bitmap.combine_region(®ion.bitmap, region.combination_operator); } page_state.page_pristine = false; } @@ -497,21 +546,29 @@ fn decode_segments( .ok_or(SegmentError::MissingPatternDictionary)?; let header = halftone::parse(&mut reader)?; - charge_work(&mut work_left, halftone::work(&header, pattern_dict))?; + charge_work(work_left, halftone::work(&header, pattern_dict))?; let region = halftone::decode(&header, pattern_dict, scratch_buffers)?; page_state.store_region(seg.header.segment_number, region.bitmap)?; } SegmentType::IntermediateGenericRefinementRegion => { // Same logic as immediate refinement, but store result instead of combining. - let reference = seg + let referred_segment = seg .header .referred_to_segments .first() - .and_then(|&num| page_state.get_referred_segment(num)) - .unwrap_or(page_bitmap); + .and_then(|&num| page_state.get_referred_segment(num)); let header = generic_refinement::parse(&mut reader)?; - charge_work(&mut work_left, region_work(&header.region_info))?; + charge_work(work_left, region_work(&header.region_info))?; + // PdfCraft patch: a refinement of the page refines its area (`Bitmap::area`). + let area; + let reference = match referred_segment { + Some(region) => region, + None => { + area = page_area(page_bitmap, &header.region_info)?; + &area + } + }; let region = generic_refinement::decode(&header, reference, scratch_buffers)?; page_state.store_region(seg.header.segment_number, region.bitmap)?; } @@ -520,8 +577,8 @@ fn decode_segments( // "3) Determine the buffer associated with the region segment that // this segment refers to." (7.4.7.5) // - // "2) If there are no referred-to segments, then use the page - // bitmap as the reference buffer." (7.4.7.5) + // With no region segment referred to, the reference is the page restricted to + // this segment's area (7.4.7.4). let referred_segment = seg .header .referred_to_segments @@ -529,7 +586,7 @@ fn decode_segments( .and_then(|&num| page_state.get_referred_segment(num)); let header = generic_refinement::parse(&mut reader)?; - charge_work(&mut work_left, region_work(&header.region_info))?; + charge_work(work_left, region_work(&header.region_info))?; if let Some(referred_segment) = referred_segment && page_state.can_decode_directly(page_bitmap, &header.region_info, false) @@ -541,14 +598,17 @@ fn decode_segments( scratch_buffers, )?; } else { - let reference = referred_segment.unwrap_or(page_bitmap); + // PdfCraft patch: a refinement of the page refines its area (`Bitmap::area`). + let area; + let reference = match referred_segment { + Some(region) => region, + None => { + area = page_area(page_bitmap, &header.region_info)?; + &area + } + }; let region = generic_refinement::decode(&header, reference, scratch_buffers)?; - page_bitmap.combine( - ®ion.bitmap, - region.bitmap.x_location as i32, - region.bitmap.y_location as i32, - region.combination_operator, - ); + page_bitmap.combine_region(®ion.bitmap, region.combination_operator); } page_state.page_pristine = false; } @@ -794,6 +854,192 @@ mod tests { assert!(Image::new_embedded(&stream(1 << 20, 1 << 20), None).is_err()); } + /// Segment `number` of `kind` on page 1, referring to `refers_to`. + fn segment(number: u8, kind: u8, refers_to: &[u8], data: &[u8]) -> Vec { + let mut segment = vec![0, 0, 0, number, kind, (refers_to.len() as u8) << 5]; + segment.extend_from_slice(refers_to); + segment.push(1); + segment.extend_from_slice(&(data.len() as u32).to_be_bytes()); + segment.extend_from_slice(data); + segment + } + + /// A blank page of 8 × 8: resolution unknown, flags, not striped. + fn page_of_8() -> Vec { + segment(0, 48, &[], &[0, 0, 0, 8, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0x42, 0, 0]) + } + + /// Region information (7.4.1) of 4 × 4 at `x`, `y`, then the region's own `header`, then + /// arbitrary coded data. + fn region_of_4(x: u32, y: u32, operator: u8, header: &[u8]) -> Vec { + let mut data: Vec = [4u32, 4, x, y].iter().flat_map(|v| v.to_be_bytes()).collect(); + data.push(operator); + data.extend_from_slice(header); + data.extend_from_slice(&[0x5a; 16]); + data + } + + const GENERIC: &[u8] = &[0, 3, 0xFF, 0xFD, 0xFF, 2, 0xFE, 0xFE, 0xFE]; // template 0, AT + const REFINEMENT: &[u8] = &[0, 0xFF, 0xFF, 0xFF, 0xFF]; // template 0, AT + const OR: u8 = 0; + const REPLACE: u8 = 4; + + /// PdfCraft patch: a region's location is unsigned (5.4.4), so one past `i32::MAX` is off the + /// page. Taken for a signed one, a region at 0xFFFFFFFE drew back onto the page two pixels left + /// of it, and a refinement there was refused, and the rest of the stream with it. Each region + /// here is 4 × 4 on a blank page of 8 × 8, its coded data arbitrary: on the page it draws, off + /// it nothing. + #[test] + fn a_region_located_past_i32_max_is_off_the_page() { + struct Ink(u32); + impl Decoder for Ink { + fn push_pixel(&mut self, black: bool) { + self.0 += u32::from(black); + } + fn push_pixel_chunk(&mut self, black: bool, chunks: u32) { + self.0 += if black { 8 * chunks } else { 0 }; + } + fn next_line(&mut self) {} + } + let ink = |x: u32, y: u32| { + let forms = [ + // An immediate generic region. + vec![segment(1, 38, &[], ®ion_of_4(x, y, OR, GENERIC))], + // An immediate refinement of the page, which reads the page at its own location (7.4.7.4). + vec![segment(1, 42, &[], ®ion_of_4(x, y, REPLACE, REFINEMENT))], + // An immediate refinement of an intermediate region at the same place (7.4.7.5). + vec![ + segment(1, 36, &[], ®ion_of_4(x, y, OR, GENERIC)), + segment(2, 42, &[1], ®ion_of_4(x, y, OR, REFINEMENT)), + ], + ]; + forms.map(|form| { + let stream = [page_of_8(), form.concat()].concat(); + let mut ink = Ink(0); + Image::new_embedded(&stream, None).and_then(|image| image.decode(&mut ink)).map(|()| ink.0) + }) + }; + for (x, y) in [(0, 0), (2, 2)] { + assert!(ink(x, y).iter().all(|ink| matches!(ink, Ok(n) if *n > 0)), "{x}, {y}: {:?}", ink(x, y)); + } + for (x, y) in [(0xFFFF_FFFE, 0), (0, 0xFFFF_FFFE), (0x8000_0000, 0x8000_0000), (u32::MAX, u32::MAX)] { + assert_eq!(ink(x, y).map(|ink| ink.ok()), [Some(0); 3], "{x:#x}, {y:#x}"); + } + } + + /// PdfCraft patch: a refinement of a region refines it where it is, "GRREFERENCEDX 0, + /// GRREFERENCEDY 0" (Table 35), not the page at the refinement's location. A region and its + /// refinement, directly or through an intermediate refinement, decode otherwise than the same + /// refinements of the page, and at 2, 3 to the pixels they decode to at 0, 0, moved. + #[test] + fn a_refinement_of_a_region_decodes_the_same_wherever_they_are() { + #[derive(Default)] + struct Pixels(Vec); + impl Decoder for Pixels { + fn push_pixel(&mut self, black: bool) { + self.0.push(black); + } + fn push_pixel_chunk(&mut self, black: bool, chunks: u32) { + self.0.extend((0..8 * chunks).map(|_| black)); + } + fn next_line(&mut self) {} + } + // The first refinement refers to the region before it, or `of_page`, to nothing: the page. + let decode = |x: u32, y: u32, through_intermediate: bool, of_page: bool| { + let first: &[u8] = if of_page { &[] } else { &[1] }; + let mut form = vec![segment(1, 36, &[], ®ion_of_4(x, y, OR, GENERIC))]; + if through_intermediate { + form.push(segment(2, 40, first, ®ion_of_4(x, y, OR, REFINEMENT))); + form.push(segment(3, 42, &[2], ®ion_of_4(x, y, OR, REFINEMENT))); + } else { + form.push(segment(2, 42, first, ®ion_of_4(x, y, OR, REFINEMENT))); + } + let stream = [page_of_8(), form.concat()].concat(); + let mut pixels = Pixels::default(); + Image::new_embedded(&stream, None).unwrap().decode(&mut pixels).unwrap(); + pixels.0 + }; + for through_intermediate in [false, true] { + let at_origin = decode(0, 0, through_intermediate, false); + assert!(at_origin.contains(&true), "draws"); + assert_ne!(at_origin, decode(0, 0, through_intermediate, true), "the page refined instead"); + for (i, &black) in decode(2, 3, through_intermediate, false).iter().enumerate() { + let (x, y) = (i % 8, i / 8); + let moved = x >= 2 && y >= 3 && at_origin[(y - 3) * 8 + x - 2]; + assert_eq!(black, moved, "{x}, {y}, through an intermediate refinement: {through_intermediate}"); + } + } + } + + /// PdfCraft patch: a refinement of the page refines the page restricted to its area (7.4.7.4), + /// reading 0 around it, and a reference 32 or more left of a refinement is read from where each + /// row starts in it. So that a context read wrong changes what decodes, the refinements are + /// large, and their pictures come from the hash they were coded from: a 64 × 16 refinement at + /// 41, 4 of a 112 × 24 page, and a 70 × 12 text instance refined from a 140 × 12 symbol 35 + /// left of it. + #[test] + fn a_refinement_refines_the_page_in_its_area_and_any_reference_where_it_is() { + #[derive(Default)] + struct Rows(Vec>, Vec); + impl Decoder for Rows { + fn push_pixel(&mut self, black: bool) { + self.1.push(u8::from(black)); + } + fn push_pixel_chunk(&mut self, black: bool, chunks: u32) { + for _ in 0..8 * chunks { + self.push_pixel(black); + } + } + fn next_line(&mut self) { + self.0.push(core::mem::take(&mut self.1)); + } + } + fn bit(seed: u32, x: u32, y: u32) -> u8 { + let mut h = x.wrapping_mul(0x9E37_79B1).wrapping_add(y.wrapping_mul(0x85EB_CA77)).wrapping_add(seed.wrapping_mul(0xC2B2_AE3D)); + h ^= h >> 15; + h = h.wrapping_mul(0x2C1B_3C6D); + h ^= h >> 12; + h = h.wrapping_mul(0x297A_2D39); + h ^= h >> 15; + ((h >> 7) & 1) as u8 + } + // About one pixel in eight: where a refinement differs from its reference. + let flipped = |seed: u32, x: u32, y: u32| bit(seed, x, y) & bit(seed + 1, x, y) & bit(seed + 2, x, y); + // A page of bit(1, ..), and over it at 41, 4 the area refined from it. + const PAGE: &str = "000000003000010000001300000070000000180000000000000000420000000000012600010000016f00000070000000180000000000000000040003fffdff02fefefefb5d354db053c2a984c01af66c619b02c245055be23a2a431b04c60325c88676f6ff1b0a68c873ad9011a27a89fc99abc7db2e8f529ecf198ff8c6a726ddcb746dc51f3812e365e74ee668fa02f59182cae39f0ef43b3201da1bcd43a21f449889e15ba0ba1a9344da1bdb9dcb4015279a39d944509febc622a235fa2c623acb443f0c10dabf7fba5e9d6fef2f1289dfd0a6a3801d036f21a62045074f5d3158ab0373ad2334102ef867dc3132dc8d02fe517771c419cdbad47d67d409ec89d0425d1f51a48af6ba3bee6e0701185bfa5f46b347b0ec39e744b665d1e8aac868838d3db25162d65534ff4771d9c0d657bd48d5814fa7cc184d0e961defab2d75a79a88345cdae5546a54221a22ddfe9c8a0514f32ac9de806fbc5fb4f5275c79d679f0258f77900daaa273cee50e50c958f323414670d7761418639342fdb2082eac5e7934f1a06169991452b1a042eaee7871ffac000000022a000100000091000000400000001000000029000000040400ffffffffd24567306e52f57232cefa9800db00fb60c21b30b0e53d9fac8a1f863a580a7cc49ab401934ac8be3a18e2dc0f01774b76c3e9196283cb5504742c763a867bc1aae8c48d74d194dacda03ce7e91904897ab61add273d9188fda65dc6fefb995d85f48142e43c1970bedb9c5f4be3cb00d458e9f4ee9714a61fffac"; + let page: Vec> = (0..24) + .map(|y| (0..112).map(|x| bit(1, x, y) ^ if (41..105).contains(&x) && (4..20).contains(&y) { flipped(10, x - 41, y - 4) } else { 0 }).collect()) + .collect(); + // A symbol of bit(3, ..), and a text instance refined from it, 70 narrower with RDX 0. + const TEXT: &str = "0000000030000100000013000000460000000c000000000000000042000000000001000001000000ed000003fffdff02fefefe00000001000000013f4efc0ce48e6b5c00246a648400fb605a44d542ed29da2b9b55c01f4e5af9842dd70c88f4913c7f552503ba52a5d7b5d9fd08ff5794b4dbfb3df7bafcf9c43d04154c1cefa181ca1aba5da826a4e4229858bd0950442df69c098ab3b61a32f71b136974b96bcb5306f3e75af1a4ef51b8f56a355acb8b69915cfa623f5a21933be8c056a348783a116ea914e886af2c64bf3b23e5610726c981092220f05411857529861fbf4c630a15fd00b081d2351279e06602f4408b25dc1d01c223027ecb8c87c4b1d76e3882a759c1adaf749f0e20735f8bd5a7306fffac000000020620010100000085000000460000000c0000000000000000040012ffffffff00000001a967c0b129999147062a4766ce10bc3dae7f1daf43dda1427d31203a1bc9fa021e0504e0a4b01739952970b54ad11ac08c0684d2f556a1843eb01dc75b963be6237b7f02fdfe17ccf7f061de310a936fd08c5ddea47609789e55f7cbca81ae38219c4abc5f9eff47ffac"; + let text: Vec> = (0..12).map(|y| (0..70).map(|x| bit(3, x + 35, y) ^ flipped(30, x, y)).collect()).collect(); + for (stream, picture) in [(PAGE, page), (TEXT, text)] { + let stream: Vec = (0..stream.len() / 2).map(|i| u8::from_str_radix(&stream[i * 2..i * 2 + 2], 16).unwrap()).collect(); + let mut rows = Rows::default(); + Image::new_embedded(&stream, None).unwrap().decode(&mut rows).unwrap(); + assert_eq!(rows.0, picture); + } + } + + /// PdfCraft patch: the area a refinement of the page refines is the page inside it and 0 + /// outside: off the page's rows and columns, past where a column can be, and past the page's + /// width and the area's own whatever the padding of the page's rows holds. `new_with` clears + /// that padding (patch 13), so the page's words are set to all 1s, padding included. + #[test] + fn a_page_area_is_the_page_inside_it_and_0_outside() { + let mut page = Bitmap::new_with(40, 3, 0, 0, true).unwrap(); + page.data.fill(!0); + let area = |x, y, width, height| page.area(x, y, width, height).unwrap().data; + assert_eq!(area(30, 1, 20, 3), [0xFFC0_0000, 0xFFC0_0000, 0], "10 columns and 2 rows of the page"); + assert_eq!(area(0, 0, 33, 1), [!0, 0x8000_0000], "the whole of a word, and none of the area's padding"); + assert_eq!(area(36, 0, 8, 1), [0xF000_0000], "4 columns, and none of the page's padding"); + assert_eq!(area(5, 0, 8, 1), [0xFF00_0000], "inside the page"); + assert_eq!(area(40, 0, 8, 1), [0], "right of the page"); + assert_eq!(area(u32::MAX - 2, 0, 40, 1), [0, 0], "past where a column can be"); + assert_eq!(area(0, u32::MAX, 8, 2), [0, 0], "below the page"); + assert!(area(3, 0, 0, 2).is_empty(), "no width"); + } + /// A stream and its globals are one stream of segments, so they have one count. #[test] fn the_segments_of_a_stream_and_its_globals_count_together() { @@ -808,6 +1054,49 @@ mod tests { assert!(refused(&more), "one more isn't"); } + /// PdfCraft patch: a refinement whose reference is offset right of it reads the reference from + /// its first column, where it read the first 32 as 0. So that a context read wrong changes what + /// decodes, the refinement is large, and its picture comes from the hash it was coded from: a + /// 104 × 12 text instance refined from a 100 × 12 symbol 2 right of it. + #[test] + fn a_refinement_reads_a_reference_right_of_it_from_its_first_column() { + #[derive(Default)] + struct Rows(Vec>, Vec); + impl Decoder for Rows { + fn push_pixel(&mut self, black: bool) { + self.1.push(u8::from(black)); + } + fn push_pixel_chunk(&mut self, black: bool, chunks: u32) { + for _ in 0..8 * chunks { + self.push_pixel(black); + } + } + fn next_line(&mut self) { + self.0.push(core::mem::take(&mut self.1)); + } + } + fn bit(seed: u32, x: u32, y: u32) -> u8 { + let mut h = x.wrapping_mul(0x9E37_79B1).wrapping_add(y.wrapping_mul(0x85EB_CA77)).wrapping_add(seed.wrapping_mul(0xC2B2_AE3D)); + h ^= h >> 15; + h = h.wrapping_mul(0x2C1B_3C6D); + h ^= h >> 12; + h = h.wrapping_mul(0x297A_2D39); + h ^= h >> 15; + ((h >> 7) & 1) as u8 + } + // About one pixel in eight: where a refinement differs from its reference. + let flipped = |seed: u32, x: u32, y: u32| bit(seed, x, y) & bit(seed + 1, x, y) & bit(seed + 2, x, y); + // A symbol of bit(2, ..), and a text instance refined from it, 4 wider with RDX 0. + const TEXT: &str = "0000000030000100000013000000680000000c000000000000000042000000000001000001000000b1000003fffdff02fefefe00000001000000013f525ad7118f669a303f1b11efa5210e9765b834b1831c650f3a3d325dbd7317379ea4fb573b09a20c362f065d8f9301e74cbbc50c69910641c0e0eea7c2fd1e831ca1459a943be55ed8e074b426ad7ff1e8d8ff6f1aa44684de5da4b4c3483911604412e00e96a0db6ec7103feb2bc4301ae98d3f9c23078ae72dc46f76e9dfea209d39187f1194f7b9285cd6e7b2dc7abb5def43ff0abe2df554fc55ffac0000000206200101000000af000000680000000c0000000000000000040012ffffffff00000001a96b5febebec5a68693b51f5db25ddf176d4aae89904a71d1690f12888e10b038db723ac59941f76c2844b739ce524d9563f4005e1edbabf2e323e8e9ac1a3b9060587c3010e7d6bcf90c00dbf9000a2f17b1d9b598e01b9158dc16a63f4388a6f647b99dc7118aa09e08d59fba1bc24e3a0f1ebb0cae636b6bedbb65b01945228d3e5d2980a0ad6290c13d2f34f5c861415ffac"; + let picture: Vec> = (0..12) + .map(|y| (0..104).map(|x| (if (2..102).contains(&x) { bit(2, x - 2, y) } else { 0 }) ^ flipped(20, x, y)).collect()) + .collect(); + let stream: Vec = (0..TEXT.len() / 2).map(|i| u8::from_str_radix(&TEXT[i * 2..i * 2 + 2], 16).unwrap()).collect(); + let mut rows = Rows::default(); + Image::new_embedded(&stream, None).unwrap().decode(&mut rows).unwrap(); + assert_eq!(rows.0, picture); + } + /// A dictionary being decoded has what the page may still hold, not all of it: refused only /// once it is stored, its copies would already have been made. #[test] @@ -819,4 +1108,17 @@ mod tests { page.retained_words = MAX_RETAINED_WORDS; assert_eq!(page.retained_room(), 0); } + + /// Pixels past a bitmap's edge are 0 (6.3.5.2), and a refinement reads its reference a word at + /// a time, so a bitmap filled with 1s has none past its width: made, and a page made again for + /// the next image. + #[test] + fn a_bitmap_of_1s_has_nothing_past_its_width() { + assert_eq!(Bitmap::new_with(33, 2, 0, 0, true).unwrap().data, [!0, 1 << 31, !0, 1 << 31]); + let mut page = Bitmap::new_with(40, 1, 0, 0, false).unwrap(); + page.reinitialize(7, 2, true).unwrap(); + assert_eq!(page.data, [0xFE00_0000, 0xFE00_0000]); + assert_eq!(Bitmap::new_with(64, 1, 0, 0, true).unwrap().data, [!0, !0], "words filled whole"); + assert!(Bitmap::new_with(0, 3, 0, 0, true).unwrap().data.is_empty(), "no width"); + } } diff --git a/vendor/hayro-jbig2/src/pdfcraft_tests.rs b/vendor/hayro-jbig2/src/pdfcraft_tests.rs index e26d4e9c..cd96db6c 100644 --- a/vendor/hayro-jbig2/src/pdfcraft_tests.rs +++ b/vendor/hayro-jbig2/src/pdfcraft_tests.rs @@ -101,9 +101,14 @@ fn jbig2_segment(out: &mut Vec, number: u32, kind: u8, refs: &[u8], data: &[ /// The page information of a 16 × 2 JBIG2 page whose default pixel is black, so that the image, if /// it decodes, draws black whatever its other segments hold. fn black_jbig2_page() -> Vec { + black_jbig2_page_of(16, 2) +} + +/// [`black_jbig2_page`], `width` × `height`. +fn black_jbig2_page_of(width: u32, height: u32) -> Vec { let mut page = Vec::new(); - page.extend_from_slice(&16u32.to_be_bytes()); - page.extend_from_slice(&2u32.to_be_bytes()); + page.extend_from_slice(&width.to_be_bytes()); + page.extend_from_slice(&height.to_be_bytes()); page.extend_from_slice(&[0; 8]); // resolution page.extend_from_slice(&[4, 0, 0]); // flags (default pixel black), striping let mut stream = Vec::new(); @@ -120,6 +125,150 @@ fn jbig2_region_info(width: u32, height: u32) -> Vec { region } +/// A Huffman-coded symbol dictionary of `symbols` symbols of width 0 and height 1, exported or not: +/// each holds no pixels, and costs a bitmap's header all the same. +fn zero_width_jbig2_dictionary(symbols: u32, exported: bool) -> Vec { + let mut dictionary = vec![0x00, 0x01]; // Huffman, tables B.4, B.2, B.1, B.1 + dictionary.extend_from_slice(&(if exported { symbols } else { 0 }).to_be_bytes()); + dictionary.extend_from_slice(&symbols.to_be_bytes()); // new + let mut bits = Bits::default(); + bits.put(0, 1); // height class delta 1 (B.4) + for _ in 0..symbols { + bits.put(0, 1); // width delta 0 (B.2) + } + bits.put(0b111111, 6); // end of the class (B.2) + bits.put(0, 5); // BMSIZE 0, uncompressed (B.1) + bits.align(); + if exported { + bits.table_b1(0); // the first export run isn't exported + } + bits.table_b1(symbols); + dictionary.extend_from_slice(&bits.bytes); + dictionary +} + +/// A Huffman-coded symbol dictionary of `symbols` symbols of width 1 and no rows, none exported. +/// Only table B.5 has a negative height class delta, so it gets to no rows from one. +fn no_row_jbig2_dictionary(symbols: u32) -> Vec { + let mut bits = Bits::default(); + bits.put(0, 1); // height class delta 1 (B.5), a class of no symbols + bits.put(0b111111, 6); // end of the class (B.2) + bits.put(0, 5); // BMSIZE 0, uncompressed (B.1) + bits.align(); + bits.put(0b1111110, 7); // height class delta -1 (B.5): -255 and 254 + bits.put(254, 8); + bits.put(0b10, 2); // width delta 1 (B.2) + for _ in 1..symbols { + bits.put(0, 1); // width delta 0 (B.2) + } + bits.put(0b111111, 6); // end of the class (B.2) + bits.put(0, 5); // BMSIZE 0, uncompressed (B.1) + bits.align(); + bits.table_b1(symbols); // the first export run, not exported, is all of them + let mut dictionary = vec![0x00, 0x05]; // Huffman, tables B.5, B.2, B.1, B.1 + dictionary.extend_from_slice(&0u32.to_be_bytes()); // exported + dictionary.extend_from_slice(&symbols.to_be_bytes()); // new + dictionary.extend_from_slice(&bits.bytes); + dictionary +} + +/// A Huffman-coded symbol dictionary that climbs to `height`, stays about there for `classes` +/// classes of no symbols, each with a byte of MMR bitmap, and drops to one row for its one symbol, +/// not exported. +fn empty_class_jbig2_dictionary(classes: u32, height: u32) -> Vec { + let mut bits = Bits::default(); + bits.put(0b111110, 6); // height class delta, the upper line of table B.5 + bits.put(height - 76, 32); + let mut at = height; + for class in 0..classes { + // Table B.5 has no delta of 0, so it goes down one and up one. + if class % 2 == 1 { + bits.put(0b1111110, 7); // height class delta -1 (B.5): -255 and 254 + bits.put(254, 8); + at -= 1; + } else if class > 0 { + bits.put(0, 1); // height class delta 1 (B.5) + at += 1; + } + bits.put(0b111111, 6); // end of the class (B.2) + bits.put(1, 5); // BMSIZE 1 (B.1) + bits.align(); + bits.put(0xFF, 8); + } + bits.put(0b1111111, 7); // height class delta, the lower line of table B.5 + bits.put(at - 1 - 256, 32); + bits.put(0b10, 2); // width delta 1 (B.2) + bits.put(0b111111, 6); // end of the class (B.2) + bits.put(0, 5); // BMSIZE 0, uncompressed (B.1) + bits.align(); + bits.put(0, 8); // its row + bits.table_b1(1); // the first export run, not exported, is all of them + let mut dictionary = vec![0x00, 0x05]; // Huffman, tables B.5, B.2, B.1, B.1 + dictionary.extend_from_slice(&0u32.to_be_bytes()); // exported + dictionary.extend_from_slice(&1u32.to_be_bytes()); // new + dictionary.extend_from_slice(&bits.bytes); + dictionary +} + +/// A Huffman-coded symbol dictionary of `symbols` white symbols of width 1 and `height` rows, none +/// exported, for `height` from 76 up. +fn narrow_jbig2_dictionary(symbols: u32, height: u32) -> Vec { + let mut bits = Bits::default(); + bits.put(0b11111, 5); // height class delta, the upper line of table B.4 + bits.put(height - 76, 32); + bits.put(0b10, 2); // width delta 1 (B.2) + for _ in 1..symbols { + bits.put(0, 1); // width delta 0 (B.2) + } + bits.put(0b111111, 6); // end of the class (B.2) + let rows = height.div_ceil(8) + 1; // an MMR-coded white bitmap: one bit a row + bits.table_b1(rows); // BMSIZE (B.1) + bits.align(); + for _ in 0..rows { + bits.put(0xFF, 8); + } + bits.table_b1(symbols); // the first export run, not exported, is all of them + let mut dictionary = vec![0x00, 0x01]; // Huffman, tables B.4, B.2, B.1, B.1 + dictionary.extend_from_slice(&0u32.to_be_bytes()); // exported + dictionary.extend_from_slice(&symbols.to_be_bytes()); // new + dictionary.extend_from_slice(&bits.bytes); + dictionary +} + +/// A Huffman-coded symbol dictionary of one white symbol of `width` columns in each height class +/// of `heights` (rising), none exported: wide symbols hold many pixels in few retained words. +fn wide_jbig2_dictionary(width: u32, heights: &[u32]) -> Vec { + let mut bits = Bits::default(); + let mut previous = 0; + for &height in heights { + match height - previous { + // Height class delta (table B.4): 1, or the upper line. + 1 => bits.put(0, 1), + delta => { + bits.put(0b11111, 5); + bits.put(delta - 76, 32); + } + } + previous = height; + bits.put(0b111110, 6); // width delta, the upper line of table B.2 + bits.put(width - 75, 32); + bits.put(0b111111, 6); // end of the class (B.2) + let rows = height.div_ceil(8) + 1; // an MMR-coded white bitmap: one bit a row + bits.table_b1(rows); // BMSIZE (B.1) + bits.align(); + for _ in 0..rows { + bits.put(0xFF, 8); + } + } + let symbols = u32::try_from(heights.len()).unwrap(); + bits.table_b1(symbols); // the first export run, not exported, is all of them + let mut dictionary = vec![0x00, 0x01]; // Huffman, tables B.4, B.2, B.1, B.1 + dictionary.extend_from_slice(&0u32.to_be_bytes()); // exported + dictionary.extend_from_slice(&symbols.to_be_bytes()); // new + dictionary.extend_from_slice(&bits.bytes); + dictionary +} + /// An intermediate generic region, white, every row typical: it decodes in a few bytes however /// large it is. fn jbig2_white_region(width: u32, height: u32) -> Vec { @@ -154,28 +303,6 @@ fn jbig2_halftone_of(side: u32, width: u32, height: u32, mmr: bool) -> Vec { halftone } -/// A Huffman-coded symbol dictionary of `symbols` symbols of width 0 and height 1, exported or not: -/// each holds no pixels, and costs a bitmap's header all the same. -fn zero_width_jbig2_dictionary(symbols: u32, exported: bool) -> Vec { - let mut dictionary = vec![0x00, 0x01]; // Huffman, tables B.4, B.2, B.1, B.1 - dictionary.extend_from_slice(&(if exported { symbols } else { 0 }).to_be_bytes()); - dictionary.extend_from_slice(&symbols.to_be_bytes()); // new - let mut bits = Bits::default(); - bits.put(0, 1); // height class delta 1 (B.4) - for _ in 0..symbols { - bits.put(0, 1); // width delta 0 (B.2) - } - bits.put(0b111111, 6); // end of the class (B.2) - bits.put(0, 5); // BMSIZE 0, uncompressed (B.1) - bits.align(); - if exported { - bits.table_b1(0); // the first export run isn't exported - } - bits.table_b1(symbols); - dictionary.extend_from_slice(&bits.bytes); - dictionary -} - /// A Huffman-coded symbol dictionary of one white symbol, `side` from 76 up, exported or not. fn square_jbig2_dictionary(side: u32, exported: bool) -> Vec { let mut bits = Bits::default(); @@ -255,6 +382,25 @@ fn jbig2_refinement_region() -> Vec { region } +/// A class of no symbols has nothing to cut from its bitmap, and rows are charged by the symbol, so +/// decoding one would be rows for nothing: a thousand classes of 65,000 rows take a second, from a +/// dictionary of 4 KB. It isn't decoded, however many there are, and the symbol after them shows that +/// its bytes are stepped over. +#[test] +fn jbig2_classes_of_no_symbols_decode_no_bitmap() { + let mut stream = black_jbig2_page(); + for number in 0..1_000 { + jbig2_segment(&mut stream, number + 1, 0, &[], &empty_class_jbig2_dictionary(1024, 65_000)); + } + let (done, finished) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = done.send(jbig2_image_draws(&stream)); + }); + // This takes milliseconds now and took a quarter of an hour, so a margin this wide can't fail on + // a busy machine. + assert_eq!(finished.recv_timeout(std::time::Duration::from_secs(120)), Ok(true)); +} + /// Each item a JBIG2 stream decodes costs its work, and the stream may ask for 2^30: four of the /// largest pages, some 6 s to 17 s. Each bitmap is bounded on its own, but nothing bounded how many /// there were, so 1,140 bytes of generic regions took over 40 s, and a 795-byte PDF's one halftone @@ -363,45 +509,47 @@ fn jbig2_decoding_work_is_counted_item_by_item() { data: jbig2_pattern_dictionary(32, 2), cost: two_patterns(1, 32), }, - // Rows cost in a symbol of no pixels too. - Item { what: "symbols without pixels", before: vec![], spent: 0, kind: 0, refs: &[], data: zero_width_jbig2_dictionary(3, false), cost: 12 }, - Item { what: "a tall symbol of no width", before: vec![], spent: 0, kind: 0, refs: &[], data: zero(1000), cost: 4000 }, - Item { what: "new symbols", before: vec![], spent: 0, kind: 0, refs: &[], data: square_jbig2_dictionary(80, false), cost: 6720 }, - Item { what: "referred symbols", before: symbol(), spent: 4, kind: 0, refs: &[1], data: exporting_none, cost: 1 }, + // What an item reads counts, at 38 an integer: two classes, five widths and an export run. + Item { what: "symbols of no rows", before: vec![], spent: 0, kind: 0, refs: &[], data: no_row_jbig2_dictionary(5), cost: 304 }, + // Rows cost in a symbol of no pixels too, on top of a class, three widths and an export run. + Item { what: "symbols without pixels", before: vec![], spent: 0, kind: 0, refs: &[], data: zero_width_jbig2_dictionary(3, false), cost: 202 }, + Item { what: "a tall symbol of no width", before: vec![], spent: 0, kind: 0, refs: &[], data: zero(1000), cost: 4114 }, + Item { what: "new symbols", before: vec![], spent: 0, kind: 0, refs: &[], data: square_jbig2_dictionary(80, false), cost: 6834 }, + Item { what: "referred symbols", before: symbol(), spent: 156, kind: 0, refs: &[1], data: exporting_none, cost: 39 }, // Its symbol, and its one row and word; Huffman-coded, a table line for the symbol. - Item { what: "immediate text", before: symbol(), spent: 4, kind: 6, refs: &[1], data: jbig2_text_region(1, 1, 0), cost: 13 }, - Item { what: "intermediate text", before: symbol(), spent: 4, kind: 4, refs: &[1], data: jbig2_text_region(1, 1, 0), cost: 13 }, - Item { what: "a text region's pixels", before: symbol(), spent: 4, kind: 6, refs: &[1], data: jbig2_text_region(32, 32, 0), cost: 75 }, + Item { what: "immediate text", before: symbol(), spent: 156, kind: 6, refs: &[1], data: jbig2_text_region(1, 1, 0), cost: 13 }, + Item { what: "intermediate text", before: symbol(), spent: 156, kind: 4, refs: &[1], data: jbig2_text_region(1, 1, 0), cost: 13 }, + Item { what: "a text region's pixels", before: symbol(), spent: 156, kind: 6, refs: &[1], data: jbig2_text_region(32, 32, 0), cost: 75 }, Item { what: "an intermediate text region's pixels", before: symbol(), - spent: 4, + spent: 156, kind: 4, refs: &[1], data: jbig2_text_region(32, 32, 0), cost: 75, }, - // The region's 13, a copy of the symbol's row and the round of the loop. - Item { what: "an instance", before: symbol(), spent: 4, kind: 6, refs: &[1], data: jbig2_text_region(1, 1, 1), cost: 15 }, - // The region's 13, 80 rows of 4 words copied, the round of the loop. + // The region's 13, a copy of the symbol's row, and the instance's 4 integers. + Item { what: "an instance", before: symbol(), spent: 156, kind: 6, refs: &[1], data: jbig2_text_region(1, 1, 1), cost: 166 }, + // The region's 13, 80 rows of 4 words copied, and the instance's 4 integers. Item { what: "an instance of a large symbol", before: vec![(0, square_jbig2_dictionary(80, true))], - spent: 6720, + spent: 6872, kind: 6, refs: &[1], data: jbig2_text_region(1, 1, 1), - cost: 334, + cost: 485, }, - // And decoded as well as copied. + // And decoded as well as copied, with the 4 integers that gave its size. Item { what: "a refined instance", before: vec![(0, square_jbig2_dictionary(80, true))], - spent: 6720, + spent: 6872, kind: 6, refs: &[1], data: jbig2_refined_text_region(), - cost: 7054, + cost: 7357, }, // A bit plane, a cell's copy and the region's. Item { @@ -457,6 +605,54 @@ fn jbig2_decoding_work_is_counted_item_by_item() { } } +/// From `cargo xtask fuzz`: one changed byte in a symbol dictionary's arithmetic-coded data +/// hung the renderer. Two of its loops can stop advancing: an export run of length 0, and a +/// height class whose first width is already the end-of-class marker. Past the end of its +/// data the decoder can return those for ever. Vendored hayro-jbig2 patch: at most one height +/// class per new symbol and `2 × symbols + 1` export runs, each plus `EXTRA_EMPTY_STEPS`. +/// These synthetic dictionaries (two new symbols each, a few bytes of arbitrary data) decode +/// the same way. In the second every class after the first symbol adds 13 to the height, so it +/// only fails when the height overflows `u32`, after 330 million empty classes (about five +/// seconds; a delta of 0 would never end). The page draws it as twenty images, so that the +/// stall without the patch is well past the timeout on any machine. +#[test] +fn jbig2_symbol_dictionaries_that_never_advance_terminate() { + for (what, data, copies) in + [("zero-length export runs", &[0x05, 0x70, 0x05, 0x14, 0x02, 0x1d, 0x56, 0x3f][..], 1), ("empty height classes", &[0x06, 0x99][..], 20)] + { + // Embedded JBIG2 segments (ISO 14492 §7.2): page information, then a symbol dictionary. + let mut jbig2 = Vec::new(); + let mut segment = |number: u32, kind: u8, data: &[u8]| { + jbig2.extend_from_slice(&number.to_be_bytes()); + jbig2.extend_from_slice(&[kind, 0, 1]); + jbig2.extend_from_slice(&u32::try_from(data.len()).unwrap().to_be_bytes()); + jbig2.extend_from_slice(data); + }; + let mut page = Vec::new(); + page.extend_from_slice(&8u32.to_be_bytes()); + page.extend_from_slice(&8u32.to_be_bytes()); + page.extend_from_slice(&[0; 8]); // resolution + page.extend_from_slice(&[0, 0, 0]); // flags, striping + segment(0, 48, &page); + let mut dictionary = vec![0, 0]; // arithmetic coding, template 0, no refinement + dictionary.extend_from_slice(&[3, 0xff, 0xfd, 0xff, 2, 0xfe, 0xfe, 0xfe]); // AT pixels + dictionary.extend_from_slice(&1u32.to_be_bytes()); // exported symbols + dictionary.extend_from_slice(&2u32.to_be_bytes()); // new symbols + dictionary.extend_from_slice(data); + segment(1, 0, &dictionary); + // The stall without the patch is well past the timeout on any machine; `copies` decodes stand + // in for the images the page draws. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + for _ in 0..copies { + let _ = black_pixels(&jbig2); + } + let _ = tx.send(()); + }); + rx.recv_timeout(std::time::Duration::from_secs(20)).unwrap_or_else(|_| panic!("{what}: a JBIG2 symbol dictionary must not stall the decoder")); + } +} + /// A JBIG2 stream can put a region, a reference or a symbol at the very end of what an `i32` holds, /// where adding to it overflowed: a panic wherever overflow is checked, a build's own tests included. /// A region or a symbol out there is off the page, so nothing draws. Vendored hayro-jbig2 patch: the arithmetic wraps, as @@ -520,3 +716,4 @@ fn jbig2_placements_at_the_end_of_an_i32_draw_nothing() { assert_eq!(ink(hex(UPPER_LINE), 16, 12), 0, "an upper range line past an i32"); assert_eq!(ink(hex(LOWER_LINE), 16, 12), 0, "a lower range line past an i32"); } +