From ccfe2d55dfb79667d21c872e84263cc943bb27e5 Mon Sep 17 00:00:00 2001 From: "Jorge C. Leitao" Date: Thu, 2 Sep 2021 08:10:47 +0000 Subject: [PATCH] Fixed reading of dict-encoded, plain pages (fallback). --- src/io/parquet/read/binary/basic.rs | 4 ++-- src/io/parquet/read/fixed_size_binary.rs | 6 ++++-- src/io/parquet/read/primitive/basic.rs | 6 ++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/io/parquet/read/binary/basic.rs b/src/io/parquet/read/binary/basic.rs index bf07f06e0f6..0e97b77b529 100644 --- a/src/io/parquet/read/binary/basic.rs +++ b/src/io/parquet/read/binary/basic.rs @@ -231,7 +231,7 @@ fn extend_from_page( values, validity, ), - (Encoding::Plain, None, true) => read_plain_optional::( + (Encoding::Plain, _, true) => read_plain_optional::( validity_buffer, values_buffer, page.num_values(), @@ -239,7 +239,7 @@ fn extend_from_page( values, validity, ), - (Encoding::Plain, None, false) => { + (Encoding::Plain, _, false) => { read_plain_required::(page.buffer(), page.num_values(), offsets, values) } _ => { diff --git a/src/io/parquet/read/fixed_size_binary.rs b/src/io/parquet/read/fixed_size_binary.rs index 0957c651866..41f79e85d13 100644 --- a/src/io/parquet/read/fixed_size_binary.rs +++ b/src/io/parquet/read/fixed_size_binary.rs @@ -210,7 +210,7 @@ pub(crate) fn extend_from_page( values, validity, ), - (Encoding::Plain, None, true) => read_optional( + (Encoding::Plain, _, true) => read_optional( validity_buffer, values_buffer, page.num_values() as u32, @@ -218,7 +218,9 @@ pub(crate) fn extend_from_page( values, validity, ), - (Encoding::Plain, None, false) => { + // it can happen that there is a dictionary but the encoding is plain because + // it falled back. + (Encoding::Plain, _, false) => { read_required(page.buffer(), page.num_values() as u32, size, values) } _ => { diff --git a/src/io/parquet/read/primitive/basic.rs b/src/io/parquet/read/primitive/basic.rs index 4d5be5332b5..9f370f34ad4 100644 --- a/src/io/parquet/read/primitive/basic.rs +++ b/src/io/parquet/read/primitive/basic.rs @@ -168,7 +168,9 @@ where op, ) } - (Encoding::Plain, None, true) => read_nullable( + // it can happen that there is a dictionary but the encoding is plain because + // it falled back. + (Encoding::Plain, _, true) => read_nullable( validity_buffer, values_buffer, additional, @@ -176,7 +178,7 @@ where validity, op, ), - (Encoding::Plain, None, false) => read_required(page.buffer(), additional, values, op), + (Encoding::Plain, _, false) => read_required(page.buffer(), additional, values, op), _ => { return Err(other_utils::not_implemented( &page.encoding(),