decimal256 to FixedLenByteArray(32)

jorgecarleitao · Feb 22, 2023 · 6661c12 · 6661c12
1 parent 2540288
commit 6661c12
Show file tree

Hide file tree

Showing 5 changed files with 8 additions and 7 deletions.
diff --git a/src/io/parquet/read/deserialize/simple.rs b/src/io/parquet/read/deserialize/simple.rs
@@ -229,7 +229,7 @@ pub fn page_iter_to_arrays<'a, I: Pages + 'a>(
 
             Box::new(arrays) as _
         }
-        (PhysicalType::FixedLenByteArray(n), Decimal256(_, _)) if *n > 16 => {
+        (PhysicalType::FixedLenByteArray(n), Decimal256(_, _)) if *n > 32 => {
             return Err(Error::NotYetImplemented(format!(
                 "Can't decode Decimal256 type from Fixed Size Byte Array of len {n:?}"
             )))
@@ -239,7 +239,7 @@ pub fn page_iter_to_arrays<'a, I: Pages + 'a>(
 
             let pages = fixed_size_binary::Iter::new(
                 pages,
-                DataType::FixedSizeBinary(16),
+                DataType::FixedSizeBinary(n),
                 num_rows,
                 chunk_size,
             );
@@ -248,7 +248,7 @@ pub fn page_iter_to_arrays<'a, I: Pages + 'a>(
                 let array = maybe_array?;
                 let values = array
                     .values()
-                    .chunks_exact(16)
+                    .chunks_exact(n)
                     .map(|value: &[u8]| super::super::convert_i256(value, n))
                     .collect::<Vec<_>>();
                 let validity = array.validity().cloned();

diff --git a/src/io/parquet/read/statistics/mod.rs b/src/io/parquet/read/statistics/mod.rs
@@ -516,7 +516,7 @@ fn push(
             _ => unreachable!(),
         },
         Decimal256(_, _) => match physical_type {
-            ParquetPhysicalType::FixedLenByteArray(n) if *n > 16 => Err(Error::NotYetImplemented(
+            ParquetPhysicalType::FixedLenByteArray(n) if *n > 32 => Err(Error::NotYetImplemented(
                 format!("Can't decode Decimal256 type from Fixed Size Byte Array of len {n:?}"),
             )),
             ParquetPhysicalType::FixedLenByteArray(n) => fixlen::push_i256(from, *n, min, max),

diff --git a/src/io/parquet/write/mod.rs b/src/io/parquet/write/mod.rs
@@ -466,14 +466,14 @@ pub fn array_to_page_simple(
         }
         DataType::Decimal256(_, _) => {
             let type_ = type_;
-            let size = 16;
+            let size = 32;
             let array = array
                 .as_any()
                 .downcast_ref::<PrimitiveArray<i256>>()
                 .unwrap();
             let mut values = Vec::<u8>::with_capacity(size * array.len());
             array.values().iter().for_each(|x| {
-                let bytes = &x.to_be_bytes()[16 - size..];
+                let bytes = &x.to_be_bytes()[32 - size..];
                 values.extend_from_slice(bytes)
             });
             let array = FixedSizeBinaryArray::new(

diff --git a/src/io/parquet/write/schema.rs b/src/io/parquet/write/schema.rs
@@ -297,7 +297,7 @@ pub fn to_parquet_type(field: &Field) -> Result<ParquetType> {
         }
         DataType::Decimal256(_, _) => Ok(ParquetType::try_from_primitive(
             name,
-            PhysicalType::FixedLenByteArray(16),
+            PhysicalType::FixedLenByteArray(32),
             repetition,
             None,
             None,

diff --git a/tests/it/io/parquet/write.rs b/tests/it/io/parquet/write.rs
@@ -70,6 +70,7 @@ fn round_trip_opt_stats(
     let data = writer.into_inner().into_inner();
 
     let (result, stats) = read_column(&mut Cursor::new(data), "a1")?;
+
     assert_eq!(array.as_ref(), result.as_ref());
     if check_stats {
         assert_eq!(statistics, stats);