From f0ef6fd934dd29cf9ee0c9cb6b8b8860dfdd448a Mon Sep 17 00:00:00 2001 From: Jonathan Behrens Date: Sun, 25 Dec 2022 12:34:28 -0800 Subject: [PATCH 1/2] Faster decoding of fdeflate encoded PNGs --- src/decoder/zlib.rs | 82 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 62 insertions(+), 20 deletions(-) diff --git a/src/decoder/zlib.rs b/src/decoder/zlib.rs index f03b5b1f..9ade2ad2 100644 --- a/src/decoder/zlib.rs +++ b/src/decoder/zlib.rs @@ -3,10 +3,64 @@ use super::{stream::FormatErrorInner, DecodingError, CHUNCK_BUFFER_SIZE}; use miniz_oxide::inflate::core::{decompress, inflate_flags, DecompressorOxide}; use miniz_oxide::inflate::TINFLStatus; +enum Compressor { + FullZlib(DecompressorOxide), + FDeflate(fdeflate::Decompressor), +} +impl Compressor { + fn decompress( + &mut self, + input: &[u8], + output: &mut [u8], + output_position: usize, + end_of_input: bool, + ) -> (TINFLStatus, usize, usize) { + match self { + Compressor::FullZlib(decompressor) => decompress( + decompressor, + input, + output, + output_position, + inflate_flags::TINFL_FLAG_PARSE_ZLIB_HEADER + | inflate_flags::TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF + | if end_of_input { + 0 + } else { + inflate_flags::TINFL_FLAG_HAS_MORE_INPUT + }, + ), + Compressor::FDeflate(decompressor) => { + match decompressor.read(input, &mut output[output_position..], end_of_input) { + Ok((in_consumed, out_consumed)) if decompressor.done() => { + assert!(in_consumed <= input.len()); + (TINFLStatus::Done, in_consumed, out_consumed) + } + Ok((in_consumed, out_consumed)) => { + assert!(in_consumed <= input.len()); + (TINFLStatus::HasMoreOutput, in_consumed, out_consumed) + } + Err(fdeflate::DecompressionError::NotFDeflate) => { + *self = Compressor::FullZlib(DecompressorOxide::new()); + + assert_eq!(output_position, 0); + self.decompress(input, output, output_position, end_of_input) + } + Err(_) => (TINFLStatus::Failed, 0, 0), + } + } + } + } +} +impl Default for Compressor { + fn default() -> Self { + Compressor::FDeflate(fdeflate::Decompressor::new()) + } +} + /// Ergonomics wrapper around `miniz_oxide::inflate::stream` for zlib compressed data. pub(super) struct ZlibStream { /// Current decoding state. - state: Box, + state: Box, /// If there has been a call to decompress already. started: bool, /// A buffer of compressed data. @@ -35,7 +89,7 @@ pub(super) struct ZlibStream { impl ZlibStream { pub(crate) fn new() -> Self { ZlibStream { - state: Box::default(), + state: Default::default(), started: false, in_buffer: Vec::with_capacity(CHUNCK_BUFFER_SIZE), in_pos: 0, @@ -49,7 +103,7 @@ impl ZlibStream { self.in_buffer.clear(); self.out_buffer.clear(); self.out_pos = 0; - *self.state = DecompressorOxide::default(); + *self.state = Default::default(); } /// Fill the decoded buffer as far as possible from `data`. @@ -59,10 +113,6 @@ impl ZlibStream { data: &[u8], image_data: &mut Vec, ) -> Result { - const BASE_FLAGS: u32 = inflate_flags::TINFL_FLAG_PARSE_ZLIB_HEADER - | inflate_flags::TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF - | inflate_flags::TINFL_FLAG_HAS_MORE_INPUT; - self.prepare_vec_for_appending(); let (status, mut in_consumed, out_consumed) = { @@ -71,17 +121,13 @@ impl ZlibStream { } else { &self.in_buffer[self.in_pos..] }; - decompress( - &mut self.state, - in_data, - self.out_buffer.as_mut_slice(), - self.out_pos, - BASE_FLAGS, - ) + self.state + .decompress(in_data, self.out_buffer.as_mut_slice(), self.out_pos, false) }; if !self.in_buffer.is_empty() { self.in_pos += in_consumed; + in_consumed = 0; } if self.in_buffer.len() == self.in_pos { @@ -117,9 +163,6 @@ impl ZlibStream { &mut self, image_data: &mut Vec, ) -> Result<(), DecodingError> { - const BASE_FLAGS: u32 = inflate_flags::TINFL_FLAG_PARSE_ZLIB_HEADER - | inflate_flags::TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF; - if !self.started { return Ok(()); } @@ -135,12 +178,11 @@ impl ZlibStream { // TODO: we may be able to avoid the indirection through the buffer here. // First append all buffered data and then create a cursor on the image_data // instead. - decompress( - &mut self.state, + self.state.decompress( &tail[start..], self.out_buffer.as_mut_slice(), self.out_pos, - BASE_FLAGS, + true, ) }; From 1144b3dc5926da2ade185275b5239b213c390dde Mon Sep 17 00:00:00 2001 From: Jonathan Behrens Date: Fri, 30 Dec 2022 16:33:57 -0800 Subject: [PATCH 2/2] Add comment explaining fdeflate fallback --- Cargo.toml | 2 +- src/decoder/zlib.rs | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7cd89254..9bc030bb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ include = [ [dependencies] bitflags = "1.0" crc32fast = "1.2.0" -fdeflate = "0.2.0" +fdeflate = "0.2.1" flate2 = "1.0" miniz_oxide = "0.6.0" diff --git a/src/decoder/zlib.rs b/src/decoder/zlib.rs index 9ade2ad2..5b00f546 100644 --- a/src/decoder/zlib.rs +++ b/src/decoder/zlib.rs @@ -40,9 +40,12 @@ impl Compressor { (TINFLStatus::HasMoreOutput, in_consumed, out_consumed) } Err(fdeflate::DecompressionError::NotFDeflate) => { - *self = Compressor::FullZlib(DecompressorOxide::new()); - + // fdeflate guarantees that it will detect non-fdeflate streams before + // consuming any input. If that happens, sanity check that no output + // has been produced and feed the same input to a full zlib decoder. assert_eq!(output_position, 0); + + *self = Compressor::FullZlib(DecompressorOxide::new()); self.decompress(input, output, output_position, end_of_input) } Err(_) => (TINFLStatus::Failed, 0, 0),