Skip to content

Commit

Permalink
Faster decompression of non-fdeflate encoded PNGs
Browse files Browse the repository at this point in the history
  • Loading branch information
@fintelia
fintelia committed Mar 17, 2023
1 parent 290a015 commit 8f71cc0
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 134 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ include = [
[dependencies]
bitflags = "1.0"
crc32fast = "1.2.0"
fdeflate = "0.2.1"
fdeflate = "0.3.0"
flate2 = "1.0"
miniz_oxide = { version = "0.7.1", features = ["simd"] }

Expand Down
19 changes: 2 additions & 17 deletions src/decoder/stream.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ pub(crate) enum FormatErrorInner {
// Errors specific to the IDAT/fDAT chunks.
/// The compression of the data stream was faulty.
CorruptFlateStream {
err: miniz_oxide::inflate::TINFLStatus,
err: fdeflate::DecompressionError,
},
/// The image data chunk was too short for the expected pixel count.
NoMoreImageData,
Expand Down Expand Up @@ -291,22 +291,7 @@ impl fmt::Display for FormatError {
NoMoreImageData => write!(fmt, "IDAT or fDAT chunk is has not enough data for image."),
CorruptFlateStream { err } => {
write!(fmt, "Corrupt deflate stream. ")?;
use miniz_oxide::inflate::TINFLStatus;
match err {
TINFLStatus::Adler32Mismatch => write!(fmt, "Adler32 checksum failed."),
TINFLStatus::BadParam => write!(fmt, "Invalid input parameters."),
// The Done status should already have been handled.
TINFLStatus::Done => write!(fmt, "Unexpected done status."),
TINFLStatus::FailedCannotMakeProgress => {
write!(fmt, "Unexpected end of data.")
}
// The HasMoreOutput status should already have been handled.
TINFLStatus::HasMoreOutput => write!(fmt, "Has more output."),
// The HasMoreInput status should already have been handled.
TINFLStatus::NeedsMoreInput => write!(fmt, "Needs more input."),
// Write out the error number in case a new has been added.
_ => write!(fmt, "Error number {:?}.", err),
}
write!(fmt, "{:?}", err)
}
// TODO: Wrap more info in the enum variant
BadTextEncoding(tde) => {
Expand Down
152 changes: 36 additions & 116 deletions src/decoder/zlib.rs
View file
Original file line number Diff line number Diff line change
@@ -1,77 +1,11 @@
use super::{stream::FormatErrorInner, DecodingError, CHUNCK_BUFFER_SIZE};

use miniz_oxide::inflate::core::{decompress, inflate_flags, DecompressorOxide};
use miniz_oxide::inflate::TINFLStatus;

enum Compressor {
FullZlib(DecompressorOxide),
FDeflate(fdeflate::Decompressor),
}
impl Compressor {
fn decompress(
&mut self,
input: &[u8],
output: &mut [u8],
output_position: usize,
ignore_adler32: bool,
end_of_input: bool,
) -> (TINFLStatus, usize, usize) {
match self {
Compressor::FullZlib(decompressor) => {
const BASE_FLAGS: u32 = inflate_flags::TINFL_FLAG_PARSE_ZLIB_HEADER
| inflate_flags::TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF;

let mut flags = BASE_FLAGS;
if !end_of_input {
flags |= inflate_flags::TINFL_FLAG_HAS_MORE_INPUT;
}
if ignore_adler32 {
flags |= inflate_flags::TINFL_FLAG_IGNORE_ADLER32;
}

decompress(decompressor, input, output, output_position, flags)
}
Compressor::FDeflate(decompressor) => {
match decompressor.read(input, &mut output[output_position..], end_of_input) {
Ok((in_consumed, out_consumed)) if decompressor.done() => {
assert!(in_consumed <= input.len());
(TINFLStatus::Done, in_consumed, out_consumed)
}
Ok((in_consumed, out_consumed)) => {
assert!(in_consumed <= input.len());
(TINFLStatus::HasMoreOutput, in_consumed, out_consumed)
}
Err(fdeflate::DecompressionError::NotFDeflate) => {
// fdeflate guarantees that it will detect non-fdeflate streams before
// consuming any input. If that happens, sanity check that no output
// has been produced and feed the same input to a full zlib decoder.
assert_eq!(output_position, 0);

*self = Compressor::FullZlib(DecompressorOxide::new());
self.decompress(
input,
output,
output_position,
ignore_adler32,
end_of_input,
)
}
Err(_) => (TINFLStatus::Failed, 0, 0),
}
}
}
}
}
impl Default for Compressor {
fn default() -> Self {
Compressor::FDeflate(fdeflate::Decompressor::new())
}
}
use fdeflate::Decompressor;

/// Ergonomics wrapper around `miniz_oxide::inflate::stream` for zlib compressed data.
pub(super) struct ZlibStream {
/// Current decoding state.
state: Box<Compressor>,
state: Box<fdeflate::Decompressor>,
/// If there has been a call to decompress already.
started: bool,
/// A buffer of compressed data.
Expand Down Expand Up @@ -106,7 +40,7 @@ pub(super) struct ZlibStream {
impl ZlibStream {
pub(crate) fn new() -> Self {
ZlibStream {
state: Default::default(),
state: Box::new(Decompressor::new()),
started: false,
in_buffer: Vec::with_capacity(CHUNCK_BUFFER_SIZE),
in_pos: 0,
Expand All @@ -122,7 +56,7 @@ impl ZlibStream {
self.in_pos = 0;
self.out_buffer.clear();
self.out_pos = 0;
*self.state = Default::default();
*self.state = Decompressor::new();
}

/// Set the `ignore_adler32` flag and return `true` if the flag was
Expand Down Expand Up @@ -155,21 +89,23 @@ impl ZlibStream {
) -> Result<usize, DecodingError> {
self.prepare_vec_for_appending();

let (status, mut in_consumed, out_consumed) = {
let in_data = if self.in_buffer.is_empty() {
data
} else {
&self.in_buffer[self.in_pos..]
};
self.state.decompress(
in_data,
self.out_buffer.as_mut_slice(),
self.out_pos,
self.ignore_adler32,
false,
)
if !self.started && self.ignore_adler32 {
self.state.ignore_adler32();
}

let in_data = if self.in_buffer.is_empty() {
data
} else {
&self.in_buffer[self.in_pos..]
};

let (mut in_consumed, out_consumed) = self
.state
.read(in_data, self.out_buffer.as_mut_slice(), self.out_pos, false)
.map_err(|err| {
DecodingError::Format(FormatErrorInner::CorruptFlateStream { err }.into())
})?;

if !self.in_buffer.is_empty() {
self.in_pos += in_consumed;
in_consumed = 0;
Expand All @@ -189,14 +125,7 @@ impl ZlibStream {
self.out_pos += out_consumed;
self.transfer_finished_data(image_data);

match status {
TINFLStatus::Done | TINFLStatus::HasMoreOutput | TINFLStatus::NeedsMoreInput => {
Ok(in_consumed)
}
err => Err(DecodingError::Format(
FormatErrorInner::CorruptFlateStream { err }.into(),
)),
}
Ok(in_consumed)
}

/// Called after all consecutive IDAT chunks were handled.
Expand All @@ -219,40 +148,31 @@ impl ZlibStream {
loop {
self.prepare_vec_for_appending();

let (status, in_consumed, out_consumed) = {
// TODO: we may be able to avoid the indirection through the buffer here.
// First append all buffered data and then create a cursor on the image_data
// instead.
self.state.decompress(
let (in_consumed, out_consumed) = self
.state
.read(
&tail[start..],
self.out_buffer.as_mut_slice(),
self.out_pos,
self.ignore_adler32,
true,
)
};
.map_err(|err| {
DecodingError::Format(FormatErrorInner::CorruptFlateStream { err }.into())
})?;

start += in_consumed;
self.out_pos += out_consumed;

match status {
TINFLStatus::Done => {
self.out_buffer.truncate(self.out_pos);
image_data.append(&mut self.out_buffer);
return Ok(());
}
TINFLStatus::HasMoreOutput => {
let transferred = self.transfer_finished_data(image_data);
assert!(
transferred > 0 || in_consumed > 0 || out_consumed > 0,
"No more forward progress made in stream decoding."
);
}
err => {
return Err(DecodingError::Format(
FormatErrorInner::CorruptFlateStream { err }.into(),
));
}
if self.state.is_done() {
self.out_buffer.truncate(self.out_pos);
image_data.append(&mut self.out_buffer);
return Ok(());
} else {
let transferred = self.transfer_finished_data(image_data);
assert!(
transferred > 0 || in_consumed > 0 || out_consumed > 0,
"No more forward progress made in stream decoding."
);
}
}
}
Expand Down

0 comments on commit 8f71cc0

Please sign in to comment.