diff --git a/Cargo.toml b/Cargo.toml index f9bfca17..b2652edf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,17 +29,26 @@ categories.workspace = true [features] default = ["arrow-rs", "derive"] -arrow-rs = ["dep:arrow-array", "dep:arrow-buffer"] +arrow-rs = ["dep:arrow-array", "dep:arrow-buffer", "dep:arrow-schema"] derive = ["dep:narrow-derive"] [dependencies] -arrow-array = { version = "49.0.0", optional = true } -arrow-buffer = { version = "49.0.0", optional = true } +# arrow-array = { version = "49.0.0", optional = true } +# arrow-buffer = { version = "49.0.0", optional = true } +# arrow-schema = { version = "49.0.0", optional = true } +arrow-array = { git = "https://github.com/apache/arrow-rs", branch = "master", optional = true } +arrow-buffer = { git = "https://github.com/apache/arrow-rs", branch = "master", optional = true } +arrow-schema = { git = "https://github.com/apache/arrow-rs", branch = "master", optional = true } narrow-derive = { path = "narrow-derive", version = "^0.3.4", optional = true } [dev-dependencies] +# arrow-cast = { version = "49.0.0", default-features = false, features = ["prettyprint"] } +arrow-cast = { git = "https://github.com/apache/arrow-rs", branch = "master", default-features = false, features = ["prettyprint"] } +bytes = "1.5.0" criterion = { version = "0.5.1", default-features = false } rand = { version = "0.8.5", default-features = false, features = ["small_rng"] } +parquet = { git = "https://github.com/apache/arrow-rs", branch = "master", features = ["arrow"] } +# parquet = { version = "49.0.0", default-features = false, features = ["arrow"] } [profile.bench] lto = true @@ -48,3 +57,7 @@ codegen-units = 1 [[bench]] name = "narrow" harness = false + +# [[example]] +# name = "parquet" +# required-features = ["arrow-rs", "derive"] diff --git a/narrow-derive/src/lib.rs b/narrow-derive/src/lib.rs index 94679edc..fbed0840 100644 --- a/narrow-derive/src/lib.rs +++ b/narrow-derive/src/lib.rs @@ -12,7 +12,8 @@ const CRATE: &str = "narrow"; static NARROW: Lazy = Lazy::new(|| match proc_macro_crate::crate_name(CRATE) { Ok(found) => match found { - FoundCrate::Itself => "crate".to_string(), + // Requires `extern crate self as narrow` + FoundCrate::Itself => CRATE.to_string(), FoundCrate::Name(name) => name, }, _ => CRATE.to_string(), diff --git a/src/array/fixed_size_primitive.rs b/src/array/fixed_size_primitive.rs index 464308e7..a2d81eb9 100644 --- a/src/array/fixed_size_primitive.rs +++ b/src/array/fixed_size_primitive.rs @@ -3,7 +3,7 @@ use super::Array; use crate::{ bitmap::{Bitmap, BitmapRef, BitmapRefMut, ValidityBitmap}, - buffer::{BufferType, VecBuffer}, + buffer::{Buffer, BufferType, VecBuffer}, nullable::Nullable, validity::Validity, FixedSize, Index, Length, @@ -56,6 +56,13 @@ where { } +// todo(mbrobbel): buffer_ref traits? +impl AsRef<[T]> for FixedSizePrimitiveArray { + fn as_ref(&self) -> &[T] { + self.0.as_slice() + } +} + impl Default for FixedSizePrimitiveArray where diff --git a/src/array/string.rs b/src/array/string.rs index 7a30ec26..e2fe8d05 100644 --- a/src/array/string.rs +++ b/src/array/string.rs @@ -56,6 +56,16 @@ where } } +impl<'a, OffsetItem: OffsetElement, Buffer: BufferType> Extend<&'a &'a str> + for StringArray +where + VariableSizeBinaryArray: Extend<&'a [u8]>, +{ + fn extend>(&mut self, iter: I) { + self.0.extend(iter.into_iter().map(|str| str.as_bytes())); + } +} + impl<'a, OffsetItem: OffsetElement, Buffer: BufferType> Extend> for StringArray where diff --git a/src/array/struct.rs b/src/array/struct.rs index b3c1cc08..be41ebbf 100644 --- a/src/array/struct.rs +++ b/src/array/struct.rs @@ -14,7 +14,7 @@ pub trait StructArrayType: ArrayType { /// The array type that stores items of this struct. Note this differs from /// the [`ArrayType`] array because that wraps this array. Also note that this /// has no [`Array`] bound. - type Array; + type Array; // into this then requires all arraytype impls to provide a field } /// Array for product types. diff --git a/src/arrow/array/boolean.rs b/src/arrow/array/boolean.rs index 001c645e..67dbe4f8 100644 --- a/src/arrow/array/boolean.rs +++ b/src/arrow/array/boolean.rs @@ -1,7 +1,35 @@ //! Interop with `arrow-rs` boolean array. -use crate::{array::BooleanArray, bitmap::Bitmap, buffer::BufferType}; +use std::sync::Arc; + +use crate::{ + array::BooleanArray, arrow::ArrowArray, bitmap::Bitmap, buffer::BufferType, nullable::Nullable, + validity::Validity, +}; use arrow_buffer::{BooleanBuffer, NullBuffer}; +use arrow_schema::{DataType, Field}; + +impl ArrowArray for BooleanArray +where + Bitmap: Validity, +{ + type Array = arrow_array::BooleanArray; + + fn as_field(&self, name: &str) -> arrow_schema::Field { + Field::new(name, DataType::Boolean, NULLABLE) + } +} + +impl From> + for BooleanArray +where + Bitmap: Validity, + Self: From, +{ + fn from(value: Arc) -> Self { + Self::from(arrow_array::BooleanArray::from(value.to_data())) + } +} impl From> for arrow_array::BooleanArray where @@ -21,27 +49,118 @@ where } } +/// Panics when there are nulls +impl From for BooleanArray +where + Bitmap: From, +{ + fn from(value: arrow_array::BooleanArray) -> Self { + let (boolean_buffer, nulls_opt) = value.into_parts(); + match nulls_opt { + Some(_) => panic!("expected array without a null buffer"), + None => BooleanArray(boolean_buffer.into()), + } + } +} + +/// Panics when there are no nulls +// OR allocate one instead and use `TryFrom` conversion? +impl From for BooleanArray +where + Bitmap: From + From, +{ + fn from(value: arrow_array::BooleanArray) -> Self { + let (boolean_buffer, nulls_opt) = value.into_parts(); + match nulls_opt { + Some(null_buffer) => BooleanArray(Nullable { + data: boolean_buffer.into(), + validity: null_buffer.into(), + }), + None => panic!("expected array with a null buffer"), + } + } +} + #[cfg(test)] mod tests { - use arrow_array::Array; + use crate::{array::BooleanArray, buffer::ArcBuffer}; - use super::*; - use crate::{bitmap::ValidityBitmap, Length}; + const INPUT: [bool; 4] = [true, true, false, true]; + const INPUT_NULLABLE: [Option; 4] = [Some(true), None, Some(false), Some(true)]; #[test] - fn convert() { - let input = [true, false, true, true]; - let array = input.into_iter().collect::(); - assert_eq!(array.len(), 4); - let array_arrow: arrow_array::BooleanArray = array.into(); - assert_eq!(array_arrow.len(), 4); - - let input_nullable = [Some(true), None, Some(false)]; - let array_nullable = input_nullable.into_iter().collect::>(); - assert_eq!(array_nullable.len(), 3); - assert_eq!(array_nullable.null_count(), 1); - let array_arrow_nullable: arrow_array::BooleanArray = array_nullable.into(); - assert_eq!(array_arrow_nullable.len(), 3); - assert_eq!(array_arrow_nullable.null_count(), 1); + fn from() { + let boolean_array = INPUT.into_iter().collect::(); + assert_eq!( + arrow_array::BooleanArray::from(boolean_array) + .into_iter() + .flatten() + .collect::>(), + INPUT + ); + + let boolean_array_arc = INPUT + .into_iter() + .collect::>(); + assert_eq!( + arrow_array::BooleanArray::from(boolean_array_arc) + .into_iter() + .flatten() + .collect::>(), + INPUT + ); + + let boolean_array_nullable = INPUT_NULLABLE.into_iter().collect::>(); + assert_eq!( + arrow_array::BooleanArray::from(boolean_array_nullable) + .into_iter() + .collect::>(), + INPUT_NULLABLE + ); + } + + #[test] + #[should_panic(expected = "expected array with a null buffer")] + fn into_nullable() { + let boolean_array = arrow_array::BooleanArray::from(INPUT.into_iter().collect::>()); + let _ = BooleanArray::::from( + boolean_array, + ); + } + + #[test] + #[should_panic(expected = "expected array without a null buffer")] + fn into_non_nullable() { + let boolean_array_nullable = INPUT_NULLABLE + .into_iter() + .collect::(); + let _ = BooleanArray::::from( + boolean_array_nullable, + ); + } + + #[test] + fn into() { + let boolean_array = arrow_array::BooleanArray::from(INPUT.into_iter().collect::>()); + assert_eq!( + BooleanArray::::from( + boolean_array + ) + .into_iter() + .collect::>(), + INPUT + ); + + let boolean_array_nullable = INPUT_NULLABLE + .into_iter() + .collect::(); + assert_eq!( + BooleanArray::::from( + boolean_array_nullable + ) + .into_iter() + .collect::>(), + INPUT_NULLABLE + ); } } diff --git a/src/arrow/array/fixed_size_primitive.rs b/src/arrow/array/fixed_size_primitive.rs index 60e1b3bf..5d8b18df 100644 --- a/src/arrow/array/fixed_size_primitive.rs +++ b/src/arrow/array/fixed_size_primitive.rs @@ -1,104 +1,214 @@ //! Interop with `arrow-rs` fixed-sized primitive array. +use std::sync::Arc; + +use arrow_array::types::{ + ArrowPrimitiveType, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, + UInt16Type, UInt32Type, UInt64Type, UInt8Type, +}; +use arrow_buffer::{NullBuffer, ScalarBuffer}; +use arrow_schema::{DataType, Field}; + use crate::{ - array::FixedSizePrimitiveArray, - arrow::{buffer::ArrowBufferBuilder, ArrowBuffer}, - bitmap::Bitmap, - buffer::BufferType, - nullable::Nullable, - FixedSize, Length, + array::FixedSizePrimitiveArray, arrow::ArrowArray, bitmap::Bitmap, buffer::BufferType, + nullable::Nullable, validity::Validity, FixedSize, }; -use arrow_array::{types::ArrowPrimitiveType, PrimitiveArray}; -use arrow_buffer::{BooleanBuffer, NullBuffer, ScalarBuffer}; + +/// Create the `ArrowArray` impl and required conversions. +macro_rules! arrow_array_convert { + ($ty:ty, $primitive_type:ident, $data_type:ident) => { + impl ArrowArray + for FixedSizePrimitiveArray<$ty, NULLABLE, Buffer> + where + ::Buffer<$ty>: Validity, + { + type Array = arrow_array::PrimitiveArray<$primitive_type>; + + fn as_field(&self, name: &str) -> arrow_schema::Field { + Field::new(name, DataType::$data_type, NULLABLE) + } + } + + impl From> + for FixedSizePrimitiveArray<$ty, NULLABLE, Buffer> + where + ::Buffer<$ty>: Validity, + Self: From>, + { + fn from(value: Arc) -> Self { + Self::from(arrow_array::PrimitiveArray::<$primitive_type>::from( + value.to_data(), + )) + } + } + }; +} + +arrow_array_convert!(u8, UInt8Type, UInt8); +arrow_array_convert!(u16, UInt16Type, UInt16); +arrow_array_convert!(u32, UInt32Type, UInt32); +arrow_array_convert!(u64, UInt64Type, UInt64); + +arrow_array_convert!(i8, Int8Type, Int8); +arrow_array_convert!(i16, Int16Type, Int16); +arrow_array_convert!(i32, Int32Type, Int32); +arrow_array_convert!(i64, Int64Type, Int64); + +arrow_array_convert!(f32, Float32Type, Float32); +arrow_array_convert!(f64, Float64Type, Float64); impl, Buffer: BufferType> - From> for PrimitiveArray + From> for arrow_array::PrimitiveArray where - ::Buffer: Length + Into<::Buffer>, + ::Buffer: Into>, { fn from(value: FixedSizePrimitiveArray) -> Self { - let len = value.len(); - Self::new(ScalarBuffer::new(value.0.into().finish(), 0, len), None) + arrow_array::PrimitiveArray::new(value.0.into(), None) } } impl, Buffer: BufferType> - From> for PrimitiveArray + From> for arrow_array::PrimitiveArray where - ::Buffer: Length + Into<::Buffer>, - Bitmap: Into, + ::Buffer: Into>, + Bitmap: Into, { fn from(value: FixedSizePrimitiveArray) -> Self { - let len = value.len(); - Self::new( - ScalarBuffer::new(value.0.data.into().finish(), 0, len), - Some(NullBuffer::new(value.0.validity.into())), - ) + arrow_array::PrimitiveArray::new(value.0.data.into(), Some(value.0.validity.into())) + } +} + +/// Panics when there are nulls +impl, Buffer: BufferType> + From> for FixedSizePrimitiveArray +where + ::Buffer: From>, +{ + fn from(value: arrow_array::PrimitiveArray) -> Self { + let (_data_type, values, nulls_opt) = value.into_parts(); + match nulls_opt { + Some(_) => panic!("expected array without a null buffer"), + None => FixedSizePrimitiveArray(values.into()), + } } } -impl, U: FixedSize> From> - for FixedSizePrimitiveArray +/// Panics when there are no nulls +impl, Buffer: BufferType> + From> for FixedSizePrimitiveArray +where + ::Buffer: From>, + Bitmap: From, { - fn from(value: PrimitiveArray) -> Self { - let (_, scala_buffer, opt_null_buffer) = value.into_parts(); - if let Some(null_buffer) = opt_null_buffer { - Self(Nullable { - data: scala_buffer.into_inner(), + fn from(value: arrow_array::PrimitiveArray) -> Self { + let (_data_type, values, nulls_opt) = value.into_parts(); + match nulls_opt { + Some(null_buffer) => FixedSizePrimitiveArray(Nullable { + data: values.into(), validity: null_buffer.into(), - }) - } else { - Self(Nullable::from(scala_buffer.into_inner())) + }), + None => panic!("expected array with a null buffer"), } } } +impl From for FixedSizePrimitiveArray +where + ::Buffer: From, +{ + fn from(value: arrow_buffer::Buffer) -> Self { + Self(value.into()) + } +} + #[cfg(test)] -mod test { - use super::*; - use crate::array::Int8Array; +mod tests { + use arrow_array::types::{UInt16Type, UInt32Type}; + + use crate::array::FixedSizePrimitiveArray; + + const INPUT: [u32; 4] = [1, 2, 3, 4]; + const INPUT_NULLABLE: [Option; 4] = [Some(1), None, Some(3), Some(4)]; #[test] - fn arrow_array() { - use crate::{array::Int8Array, bitmap::ValidityBitmap}; - use arrow_array::{types::Int8Type, Array, PrimitiveArray}; + fn from() { + let fixed_size_primitive_array = INPUT.into_iter().collect::>(); + assert_eq!( + arrow_array::PrimitiveArray::::from(fixed_size_primitive_array) + .into_iter() + .flatten() + .collect::>(), + INPUT + ); - let input = [1, 2, 3, 4]; - let array_arrow_buffer = input + let fixed_size_primitive_array_nullable = INPUT_NULLABLE .into_iter() - .collect::>(); - let array_arrow_from = PrimitiveArray::::from(array_arrow_buffer); - assert_eq!(array_arrow_from.len(), 4); + .collect::>(); + assert_eq!( + arrow_array::PrimitiveArray::::from(fixed_size_primitive_array_nullable) + .into_iter() + .collect::>(), + INPUT_NULLABLE + ); + } - let array = - PrimitiveArray::::from(input.into_iter().collect::>()); - assert_eq!(array.len(), 4); + #[test] + #[should_panic(expected = "expected array with a null buffer")] + fn into_nullable() { + let primitive_array = INPUT + .into_iter() + .collect::>(); + let _ = FixedSizePrimitiveArray::< + u32, + true, + crate::arrow::buffer::scalar_buffer::ArrowScalarBuffer, + >::from(primitive_array); + } - let input_nullable = [Some(1), None, Some(3), Some(4)]; - let array_nullable = input_nullable + #[test] + #[should_panic(expected = "expected array without a null buffer")] + fn into_non_nullable() { + let primitive_array_nullable = INPUT_NULLABLE .into_iter() - .collect::>(); - assert_eq!(array_nullable.null_count(), 1); - let array_arrow = PrimitiveArray::::from(array_nullable); - assert_eq!(array_arrow.len(), 4); - assert_eq!(array_arrow.null_count(), 1); + .collect::>(); + let _ = FixedSizePrimitiveArray::< + u16, + false, + crate::arrow::buffer::scalar_buffer::ArrowScalarBuffer, + >::from(primitive_array_nullable); } #[test] - fn arrow_buffer() { - let input = [1, 2, 3, 4]; - let mut array = input + #[allow(clippy::redundant_closure_for_method_calls)] + fn into() { + let primitive_array = INPUT + .into_iter() + .collect::>(); + assert_eq!( + FixedSizePrimitiveArray::< + u32, + false, + crate::arrow::buffer::scalar_buffer::ArrowScalarBuffer, + >::from(primitive_array) + .into_iter() + .copied() + .collect::>(), + INPUT + ); + + let primitive_array_nullable = INPUT_NULLABLE .into_iter() - .collect::>(); - assert_eq!(array.len(), 4); - // Use arrow_buffer - array.0.append_n(5, 5); - assert_eq!(array.0.as_slice(), &[1, 2, 3, 4, 5, 5, 5, 5, 5]); - - let input_nullable = [Some(1), None, Some(3), Some(4)]; - let array_nullable = input_nullable + .collect::>(); + assert_eq!( + FixedSizePrimitiveArray::< + u16, + true, + crate::arrow::buffer::scalar_buffer::ArrowScalarBuffer, + >::from(primitive_array_nullable) .into_iter() - .collect::>(); - assert_eq!(array_nullable.len(), 4); + .map(|opt| opt.copied()) + .collect::>(), + INPUT_NULLABLE + ); } } diff --git a/src/arrow/array/mod.rs b/src/arrow/array/mod.rs index 2aa5b571..b2a1d970 100644 --- a/src/arrow/array/mod.rs +++ b/src/arrow/array/mod.rs @@ -2,3 +2,6 @@ mod boolean; mod fixed_size_primitive; +mod string; +mod r#struct; +mod variable_size_list; diff --git a/src/arrow/array/string.rs b/src/arrow/array/string.rs new file mode 100644 index 00000000..432f3ae8 --- /dev/null +++ b/src/arrow/array/string.rs @@ -0,0 +1,188 @@ +//! Interop with [`arrow-rs`] string array. + +use std::sync::Arc; + +use arrow_array::OffsetSizeTrait; +use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; +use arrow_schema::{DataType, Field}; + +use crate::{ + array::{FixedSizePrimitiveArray, StringArray, VariableSizeBinaryArray}, + arrow::ArrowArray, + bitmap::Bitmap, + buffer::BufferType, + nullable::Nullable, + offset::{Offset, OffsetElement}, + validity::Validity, +}; + +impl + ArrowArray for StringArray +where + ::Buffer: Validity, +{ + type Array = arrow_array::GenericStringArray; + + fn as_field(&self, name: &str) -> arrow_schema::Field { + Field::new(name, DataType::Utf8, NULLABLE) + } +} + +impl + From> for StringArray +where + ::Buffer: Validity, + Self: From>, +{ + fn from(value: Arc) -> Self { + Self::from(arrow_array::GenericStringArray::::from( + value.to_data(), + )) + } +} + +impl + From> for arrow_array::GenericStringArray +where + ::Buffer: Into>, + FixedSizePrimitiveArray: Into, +{ + fn from(value: StringArray) -> Self { + arrow_array::GenericStringArray::new( + // Safety: + // - The narrow offfset buffer contains valid offset data + unsafe { OffsetBuffer::new_unchecked(value.0 .0.offsets.into()) }, + value.0 .0.data.into(), + None, + ) + } +} + +impl + From> for arrow_array::GenericStringArray +where + ::Buffer: Into>, + FixedSizePrimitiveArray: Into, + Bitmap: Into, +{ + fn from(value: StringArray) -> Self { + arrow_array::GenericStringArray::new( + // Safety: + // - The narrow offfset buffer contains valid offset data + unsafe { OffsetBuffer::new_unchecked(value.0 .0.offsets.data.into()) }, + value.0 .0.data.into(), + Some(value.0 .0.offsets.validity.into()), + ) + } +} + +/// Panics when there are nulls +impl + From> for StringArray +where + FixedSizePrimitiveArray: From, + ::Buffer: From>, +{ + fn from(value: arrow_array::GenericStringArray) -> Self { + let (offsets, values, nulls_opt) = value.into_parts(); + match nulls_opt { + Some(_) => panic!("expected array without a null buffer"), + None => StringArray(VariableSizeBinaryArray(Offset { + data: values.into(), + offsets: offsets.into_inner().into(), + })), + } + } +} + +/// Panics when there are no nulls +impl + From> for StringArray +where + FixedSizePrimitiveArray: From, + ::Buffer: From>, + Bitmap: From, +{ + fn from(value: arrow_array::GenericStringArray) -> Self { + let (offsets, values, nulls_opt) = value.into_parts(); + match nulls_opt { + Some(null_buffer) => StringArray(VariableSizeBinaryArray(Offset { + data: values.into(), + offsets: Nullable { + data: offsets.into_inner().into(), + validity: null_buffer.into(), + }, + })), + None => panic!("expected array with a null buffer"), + } + } +} + +#[cfg(test)] +mod tests { + use std::i64; + + use crate::{array::StringArray, arrow::scalar_buffer::ArrowScalarBuffer}; + + const INPUT: [&str; 3] = ["hello", "world", "!"]; + const INPUT_NULLABLE: [Option<&str>; 3] = [Some("hello"), None, Some("!")]; + + #[test] + fn from() { + let string_array = INPUT.into_iter().collect::(); + assert_eq!( + arrow_array::StringArray::from(string_array) + .into_iter() + .flatten() + .collect::>(), + INPUT + ); + + let string_array_nullable = INPUT_NULLABLE + .into_iter() + .collect::>(); + assert_eq!( + arrow_array::GenericStringArray::::from(string_array_nullable) + .into_iter() + .collect::>(), + INPUT_NULLABLE + ); + } + + #[test] + #[should_panic(expected = "expected array with a null buffer")] + fn into_nullable() { + let string_array = INPUT + .into_iter() + .map(ToOwned::to_owned) + .map(Option::Some) + .collect::(); + let _: StringArray = string_array.into(); + } + + #[test] + #[should_panic(expected = "expected array without a null buffer")] + fn into_non_nullable() { + let string_array_nullable = INPUT_NULLABLE + .into_iter() + .collect::(); + let _: StringArray = string_array_nullable.into(); + } + + #[test] + fn into() { + let string_array = INPUT + .into_iter() + .map(ToOwned::to_owned) + .map(Option::Some) + .collect::(); + let _: StringArray = string_array.into(); + // todo(mbrobbel): intoiterator for stringarray + + let string_array_nullable = INPUT_NULLABLE + .into_iter() + .collect::(); + let _: StringArray = string_array_nullable.into(); + // todo(mbrobbel): intoiterator for stringarray + } +} diff --git a/src/arrow/array/struct.rs b/src/arrow/array/struct.rs new file mode 100644 index 00000000..b8492f96 --- /dev/null +++ b/src/arrow/array/struct.rs @@ -0,0 +1 @@ +//! Interop with [`arrow-rs`] struct arrays. diff --git a/src/arrow/array/variable_size_list.rs b/src/arrow/array/variable_size_list.rs new file mode 100644 index 00000000..5e8d1128 --- /dev/null +++ b/src/arrow/array/variable_size_list.rs @@ -0,0 +1,250 @@ +//! Interop with [`arrow-rs`] string array. + +use std::sync::Arc; + +use arrow_array::OffsetSizeTrait; +use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; +use arrow_schema::{DataType, Field}; + +use crate::{ + array::{Array, VariableSizeListArray}, + arrow::ArrowArray, + bitmap::Bitmap, + buffer::BufferType, + nullable::Nullable, + offset::{Offset, OffsetElement}, + validity::Validity, +}; + +impl< + T: ArrowArray, + const NULLABLE: bool, + OffsetItem: OffsetElement + OffsetSizeTrait, + Buffer: BufferType, + > ArrowArray for VariableSizeListArray +where + ::Buffer: Validity, +{ + type Array = arrow_array::GenericListArray; + + fn as_field(&self, name: &str) -> arrow_schema::Field { + Field::new( + name, + DataType::List(Arc::new(self.0.data.as_field("item"))), + NULLABLE, + ) + } +} + +impl< + T: Array, + const NULLABLE: bool, + OffsetItem: OffsetElement + OffsetSizeTrait, + Buffer: BufferType, + > From> for VariableSizeListArray +where + ::Buffer: Validity, + Self: From>, +{ + fn from(value: Arc) -> Self { + Self::from(arrow_array::GenericListArray::::from( + value.to_data(), + )) + } +} + +impl + From> + for arrow_array::GenericListArray +where + ::Buffer: Into>, + ::Array: From + 'static, +{ + fn from(value: VariableSizeListArray) -> Self { + arrow_array::GenericListArray::new( + Arc::new(value.0.data.as_field("item")), + // Safety: + // - The narrow offfset buffer contains valid offset data + unsafe { OffsetBuffer::new_unchecked(value.0.offsets.into()) }, + value.0.data.into_array_ref(), + None, + ) + } +} + +impl + From> + for arrow_array::GenericListArray +where + ::Buffer: Into>, + Bitmap: Into, + ::Array: From + 'static, +{ + fn from(value: VariableSizeListArray) -> Self { + arrow_array::GenericListArray::new( + Arc::new(value.0.data.as_field("item")), + // Safety: + // - The narrow offfset buffer contains valid offset data + unsafe { OffsetBuffer::new_unchecked(value.0.offsets.data.into()) }, + value.0.data.into_array_ref(), + Some(value.0.offsets.validity.into()), + ) + } +} + +/// Panics when there are nulls +impl + From> + for VariableSizeListArray +where + T: From>, + ::Buffer: From>, +{ + fn from(value: arrow_array::GenericListArray) -> Self { + let (_field, offsets, values, nulls_opt) = value.into_parts(); + match nulls_opt { + Some(_) => panic!("expected array without a null buffer"), + None => VariableSizeListArray(Offset { + data: values.into(), + offsets: offsets.into_inner().into(), + }), + } + } +} + +/// Panics when there are no nulls +impl + From> + for VariableSizeListArray +where + T: From>, + ::Buffer: From>, + Bitmap: From, +{ + fn from(value: arrow_array::GenericListArray) -> Self { + let (_field, offsets, values, nulls_opt) = value.into_parts(); + match nulls_opt { + Some(null_buffer) => VariableSizeListArray(Offset { + data: values.into(), + offsets: Nullable { + data: offsets.into_inner().into(), + validity: null_buffer.into(), + }, + }), + None => panic!("expected array with a null buffer"), + } + } +} + +#[cfg(test)] +mod tests { + use arrow_array::{ + builder::{ListBuilder, StringBuilder}, + types::UInt16Type, + Array as _, + }; + + use crate::{ + array::{StringArray, Uint16Array, VariableSizeListArray}, + arrow::scalar_buffer::ArrowScalarBuffer, + Length, + }; + + const INPUT: [&[u16]; 3] = [&[1, 2], &[3], &[4]]; + const INPUT_NULLABLE: [Option<&[&str]>; 3] = + [Some(&["hello", " "]), None, Some(&["world", "!"])]; + + #[test] + fn from() { + let variable_size_list_array = INPUT + .into_iter() + .collect::>(); + let list_array = arrow_array::ListArray::from(variable_size_list_array); + assert_eq!(list_array.len(), INPUT.len()); + + let variable_size_list_array_nullable = INPUT_NULLABLE + .into_iter() + .collect::>(); + let list_array_nullable = arrow_array::ListArray::from(variable_size_list_array_nullable); + assert_eq!(list_array_nullable.len(), INPUT_NULLABLE.len()); + } + + #[test] + #[should_panic(expected = "expected array with a null buffer")] + fn into_nullable() { + let list_array = arrow_array::ListArray::from_iter_primitive::( + INPUT + .into_iter() + .map(|opt| opt.iter().copied().map(Option::Some)) + .map(Option::Some), + ); + let _: VariableSizeListArray< + Uint16Array, + true, + i32, + ArrowScalarBuffer, + > = list_array.into(); + } + + #[test] + #[should_panic(expected = "expected array without a null buffer")] + fn into_non_nullable() { + let mut list_builder = + ListBuilder::with_capacity(StringBuilder::new(), INPUT_NULLABLE.len()); + INPUT_NULLABLE.into_iter().for_each(|opt| match opt { + Some(items) => { + for item in items { + list_builder.values().append_value(item); + } + list_builder.append(true); + } + None => { + list_builder.append(false); + } + }); + let list_array_nullable = list_builder.finish(); + let _: VariableSizeListArray< + StringArray, + false, + i32, + ArrowScalarBuffer, + > = list_array_nullable.into(); + } + + #[test] + fn into() { + let list_array = arrow_array::ListArray::from_iter_primitive::( + INPUT + .into_iter() + .map(|opt| opt.iter().copied().map(Option::Some)) + .map(Option::Some), + ); + let _: VariableSizeListArray< + Uint16Array, + false, + i32, + ArrowScalarBuffer, + > = list_array.into(); + + let mut list_builder = + ListBuilder::with_capacity(StringBuilder::new(), INPUT_NULLABLE.len()); + INPUT_NULLABLE.into_iter().for_each(|opt| match opt { + Some(items) => { + for item in items { + list_builder.values().append_value(item); + } + list_builder.append(true); + } + None => { + list_builder.append(false); + } + }); + let list_array_nullable = list_builder.finish(); + let _: VariableSizeListArray< + StringArray, + true, + i32, + ArrowScalarBuffer, + > = list_array_nullable.into(); + } +} diff --git a/src/arrow/bitmap.rs b/src/arrow/bitmap.rs deleted file mode 100644 index dc9cdfc1..00000000 --- a/src/arrow/bitmap.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! Interop with [`arrow-rs`]'s bitmap. - -use super::{buffer::ArrowBuffer, ArrowBufferBuilder}; -use crate::{bitmap::Bitmap, buffer::BufferType}; -use arrow_buffer::{BooleanBuffer, NullBuffer}; - -impl From> for BooleanBuffer -where - ::Buffer: Into<::Buffer>, -{ - fn from(value: Bitmap) -> Self { - BooleanBuffer::new(value.buffer.into().finish(), value.offset, value.bits) - } -} - -impl From> for NullBuffer -where - ::Buffer: Into<::Buffer>, -{ - fn from(value: Bitmap) -> Self { - Self::new(value.into()) - } -} - -impl From for Bitmap { - fn from(value: NullBuffer) -> Self { - let bits = value.len(); - let offset = value.offset(); - Bitmap { - buffer: value.into_inner().into_inner(), - bits, - offset, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Length; - - #[test] - fn convert() { - let input = vec![true, false, true]; - let bitmap_arrow = input.iter().collect::>(); - assert_eq!(bitmap_arrow.len(), 3); - let _: NullBuffer = bitmap_arrow.into(); - - let bitmap = input.into_iter().collect::(); - assert_eq!(bitmap.len(), 3); - let _: BooleanBuffer = bitmap.into(); - } -} diff --git a/src/arrow/buffer.rs b/src/arrow/buffer.rs deleted file mode 100644 index 62f4bcb5..00000000 --- a/src/arrow/buffer.rs +++ /dev/null @@ -1,108 +0,0 @@ -//! Interop with [`arrow-buffer`]. - -use crate::buffer::{Buffer, BufferMut, BufferType}; -use crate::{FixedSize, Index}; -use arrow_buffer::{BufferBuilder, ScalarBuffer}; - -/// A [`BufferType`] implementation for [`BufferBuilder`]. -#[derive(Clone, Copy)] -pub struct ArrowBufferBuilder; - -impl BufferType for ArrowBufferBuilder { - type Buffer = BufferBuilder; -} - -impl Buffer for BufferBuilder { - fn as_slice(&self) -> &[T] { - BufferBuilder::as_slice(self) - } -} - -impl BufferMut for BufferBuilder { - fn as_mut_slice(&mut self) -> &mut [T] { - BufferBuilder::as_slice_mut(self) - } -} - -impl Index for BufferBuilder { - type Item<'a> = &'a T - where - Self: 'a; - - unsafe fn index_unchecked(&self, index: usize) -> Self::Item<'_> { - self.as_slice().get_unchecked(index) - } -} - -/// A [`BufferType`] implementation for [`ScalarBuffer`]. -#[derive(Clone, Copy)] -pub struct ArrowScalarBuffer; - -impl BufferType for ArrowScalarBuffer { - type Buffer = ScalarBuffer; -} - -impl Buffer for ScalarBuffer { - fn as_slice(&self) -> &[T] { - self - } -} - -impl Index for ScalarBuffer { - type Item<'a> = &'a T - where - Self: 'a; - - unsafe fn index_unchecked(&self, index: usize) -> Self::Item<'_> { - self.get_unchecked(index) - } -} - -/// A [`BufferType`] implementation for [`arrow_buffer::Buffer`]. -#[derive(Clone, Copy)] -pub struct ArrowBuffer; - -impl BufferType for ArrowBuffer { - type Buffer = arrow_buffer::Buffer; -} - -impl Buffer for arrow_buffer::Buffer { - fn as_slice(&self) -> &[T] { - arrow_buffer::Buffer::typed_data(self) - } -} - -impl Index for arrow_buffer::Buffer { - type Item<'a> = &'a u8 - where - Self: 'a; - - unsafe fn index_unchecked(&self, index: usize) -> Self::Item<'_> { - self.get_unchecked(index) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn arrow() { - let buffer = arrow_buffer::BufferBuilder::from_iter([1, 2, 3, 4]); - assert_eq!(<_ as Buffer>::as_slice(&buffer), &[1, 2, 3, 4]); - - let mut buffer_builder = arrow_buffer::BufferBuilder::from_iter([1_u64, 2, 3, 4]); - assert_eq!( - <_ as BufferMut>::as_mut_slice(&mut buffer_builder), - &[1, 2, 3, 4] - ); - <_ as BufferMut>::as_mut_slice(&mut buffer_builder)[3] = 42; - assert_eq!( - <_ as Buffer>::as_slice(&buffer_builder), - &[1, 2, 3, 42] - ); - - let scalar_buffer = arrow_buffer::ScalarBuffer::from_iter([1, 2, 3, 4]); - assert_eq!(<_ as Buffer>::as_slice(&scalar_buffer), &[1, 2, 3, 4]); - } -} diff --git a/src/arrow/buffer/boolean_buffer.rs b/src/arrow/buffer/boolean_buffer.rs new file mode 100644 index 00000000..9ee17bc9 --- /dev/null +++ b/src/arrow/buffer/boolean_buffer.rs @@ -0,0 +1,78 @@ +//! Interop with [`arrow-rs`] boolean buffer. + +use arrow_buffer::BooleanBuffer; + +use crate::{bitmap::Bitmap, buffer::BufferType, Length}; + +impl Length for BooleanBuffer { + fn len(&self) -> usize { + BooleanBuffer::len(self) + } +} + +impl From> for BooleanBuffer +where + ::Buffer: Into, +{ + fn from(value: Bitmap) -> Self { + Self::new(value.buffer.into(), value.offset, value.bits) + } +} + +impl From for Bitmap +where + ::Buffer: From, +{ + fn from(value: BooleanBuffer) -> Self { + let bits = value.len(); + let offset = value.offset(); + Bitmap { + buffer: value.into_inner().into(), + bits, + offset, + } + } +} + +#[cfg(test)] +mod tests { + use crate::{arrow::buffer::scalar_buffer::ArrowScalarBuffer, buffer::ArcBuffer}; + + use super::*; + + const INPUT: [bool; 4] = [true, true, false, true]; + + #[test] + fn length() { + let boolean_buffer = INPUT.into_iter().collect::(); + assert_eq!(Length::len(&boolean_buffer), INPUT.len()); + } + + #[test] + fn from() { + let bitmap = INPUT.into_iter().collect::(); + assert_eq!( + BooleanBuffer::from(bitmap).into_iter().collect::>(), + INPUT + ); + + let bitmap_arc = INPUT.into_iter().collect::>(); + assert_eq!( + BooleanBuffer::from(bitmap_arc) + .into_iter() + .collect::>(), + INPUT + ); + } + + #[test] + fn into() { + let boolean_buffer = INPUT.into_iter().collect::(); + assert_eq!( + Bitmap::::from(boolean_buffer) + .into_iter() + .collect::>(), + INPUT + ); + } +} diff --git a/src/arrow/buffer/buffer_builder.rs b/src/arrow/buffer/buffer_builder.rs new file mode 100644 index 00000000..2270f242 --- /dev/null +++ b/src/arrow/buffer/buffer_builder.rs @@ -0,0 +1,110 @@ +//! Interop with [`arrow-rs`] buffer builder. + +use arrow_buffer::BufferBuilder; + +use crate::{ + array::FixedSizePrimitiveArray, + buffer::{Buffer, BufferMut, BufferType}, + FixedSize, Index, Length, +}; + +/// A [`BufferType`] implementation for [`BufferBuilder`]. +#[derive(Clone, Copy)] +pub struct ArrowBufferBuilder; + +impl BufferType for ArrowBufferBuilder { + type Buffer = BufferBuilder; +} + +impl Buffer for BufferBuilder { + fn as_slice(&self) -> &[T] { + BufferBuilder::as_slice(self) + } +} + +impl BufferMut for BufferBuilder { + fn as_mut_slice(&mut self) -> &mut [T] { + BufferBuilder::as_slice_mut(self) + } +} + +impl Index for BufferBuilder { + type Item<'a> = &'a T + where + Self: 'a; + + unsafe fn index_unchecked(&self, index: usize) -> Self::Item<'_> { + self.as_slice().get_unchecked(index) + } +} + +impl Length for BufferBuilder { + fn len(&self) -> usize { + BufferBuilder::len(self) + } +} + +impl From> + for BufferBuilder +{ + fn from(value: FixedSizePrimitiveArray) -> Self { + // Note: this makes a copy + let buffer = arrow_buffer::MutableBuffer::from(value.0.as_slice().to_vec()); + BufferBuilder::new_from_buffer(buffer) + } +} + +impl From> + for FixedSizePrimitiveArray +where + ::Buffer: From, +{ + fn from(mut value: BufferBuilder) -> Self { + FixedSizePrimitiveArray(value.finish().into()) + } +} + +#[cfg(test)] +mod tests { + use crate::{arrow::scalar_buffer::ArrowScalarBuffer, buffer::ArcBuffer}; + + use super::*; + + const INPUT: [u32; 4] = [1, 2, 3, 4]; + + #[test] + fn length() { + let buffer_builder = INPUT.into_iter().collect::>(); + assert_eq!(Length::len(&buffer_builder), INPUT.len()); + } + + #[test] + fn from() { + let fixed_size_primitive_array = INPUT.into_iter().collect::>(); + assert_eq!( + BufferBuilder::from(fixed_size_primitive_array).as_slice(), + INPUT + ); + + let fixed_size_primitive_array_arc = + INPUT + .into_iter() + .collect::>(); + assert_eq!( + BufferBuilder::from(fixed_size_primitive_array_arc).as_slice(), + INPUT + ); + } + + #[test] + fn into() { + let buffer_builder = INPUT.into_iter().collect::>(); + assert_eq!( + FixedSizePrimitiveArray::<_, false, ArrowScalarBuffer>::from(buffer_builder) + .into_iter() + .copied() + .collect::>(), + INPUT + ); + } +} diff --git a/src/arrow/buffer/mod.rs b/src/arrow/buffer/mod.rs new file mode 100644 index 00000000..d5a03bc6 --- /dev/null +++ b/src/arrow/buffer/mod.rs @@ -0,0 +1,39 @@ +//! Interop with [`arrow-rs`] buffer types. + +pub mod boolean_buffer; +pub mod buffer_builder; +pub mod null_buffer; +pub mod offset_buffer; +pub mod scalar_buffer; + +// /// A [`BufferType`] implementation for &'a arrow Buffer. +// /// +// /// Stores items `T` in an arrow `&Buffer`. +// #[derive(Clone, Copy, Debug)] +// pub struct ArrowRefBuffer<'a>(PhantomData<&'a ()>); + +// impl<'a> BufferType for ArrowRefBuffer<'a> { +// type Buffer = &'a [T]; +// } + +// impl Buffer for &arrow_buffer::Buffer { +// fn as_slice(&self) -> &[T] { +// self.typed_data() +// } +// } + +// impl Length for &arrow_buffer::Buffer { +// fn len(&self) -> usize { +// arrow_buffer::Buffer::len(self) +// } +// } + +// impl Index for &arrow_buffer::Buffer { +// type Item<'a> = &'a u8 +// where +// Self: 'a; + +// unsafe fn index_unchecked(&self, index: usize) -> Self::Item<'_> { +// self.get_unchecked(index) +// } +// } diff --git a/src/arrow/buffer/null_buffer.rs b/src/arrow/buffer/null_buffer.rs new file mode 100644 index 00000000..69a25404 --- /dev/null +++ b/src/arrow/buffer/null_buffer.rs @@ -0,0 +1,70 @@ +//! Interop with [`arrow-rs`] null buffer. + +use arrow_buffer::{BooleanBuffer, NullBuffer}; + +use crate::{bitmap::Bitmap, buffer::BufferType, Length}; + +impl Length for NullBuffer { + fn len(&self) -> usize { + NullBuffer::len(self) + } +} + +impl From> for NullBuffer +where + Bitmap: Into, +{ + fn from(value: Bitmap) -> Self { + Self::new(value.into()) + } +} + +impl From for Bitmap +where + Bitmap: From, +{ + fn from(value: NullBuffer) -> Self { + Bitmap::from(value.into_inner()) + } +} + +#[cfg(test)] +mod tests { + use crate::{arrow::buffer::scalar_buffer::ArrowScalarBuffer, buffer::ArcBuffer}; + + use super::*; + + const INPUT: [bool; 4] = [true, true, false, true]; + + #[test] + fn length() { + let null_buffer = INPUT.into_iter().collect::(); + assert_eq!(Length::len(&null_buffer), INPUT.len()); + } + + #[test] + fn from() { + let bitmap = INPUT.into_iter().collect::(); + assert_eq!( + NullBuffer::from(bitmap).into_iter().collect::>(), + INPUT + ); + + let bitmap_arc = INPUT.into_iter().collect::>(); + assert_eq!( + NullBuffer::from(bitmap_arc).into_iter().collect::>(), + INPUT + ); + } + + #[test] + fn into() { + let null_buffer = INPUT.into_iter().collect::(); + assert_eq!( + Bitmap::::from(null_buffer) + .into_iter() + .collect::>(), + INPUT + ); + } +} diff --git a/src/arrow/buffer/offset_buffer.rs b/src/arrow/buffer/offset_buffer.rs new file mode 100644 index 00000000..7cc80b33 --- /dev/null +++ b/src/arrow/buffer/offset_buffer.rs @@ -0,0 +1,85 @@ +//! Interop with [`arrow-rs`] offset buffer. + +//! Interop with [`arrow-rs`] null buffer. + +use arrow_buffer::{OffsetBuffer, ScalarBuffer}; + +use crate::{array::FixedSizePrimitiveArray, buffer::BufferType, offset::OffsetElement, Length}; + +impl Length for OffsetBuffer { + fn len(&self) -> usize { + self.as_ref().len() + } +} + +impl + From> for OffsetBuffer +where + FixedSizePrimitiveArray: Into>, +{ + fn from(value: FixedSizePrimitiveArray) -> Self { + Self::new(value.into()) + } +} + +impl From> + for FixedSizePrimitiveArray +where + ::Buffer: From>, +{ + fn from(value: OffsetBuffer) -> Self { + Self(value.into_inner().into()) + } +} + +#[cfg(test)] +mod tests { + + use crate::{ + array::FixedSizePrimitiveArray, arrow::buffer::scalar_buffer::ArrowScalarBuffer, + buffer::ArcBuffer, + }; + + use super::*; + + const INPUT: [usize; 4] = [1, 1, 2, 2]; + + #[test] + fn length() { + let offset_buffer = OffsetBuffer::::from_lengths(INPUT); + assert_eq!(Length::len(&offset_buffer), INPUT.len() + 1); + } + + #[test] + fn from() { + let fixed_size_primitive_array = INPUT + .into_iter() + .map(|x| x.try_into().expect("")) + .collect::>(); + assert_eq!( + OffsetBuffer::::from(fixed_size_primitive_array).as_ref(), + [1, 1, 2, 2] + ); + + let fixed_size_primitive_array_arc = INPUT + .into_iter() + .map(|x| x.try_into().expect("")) + .collect::>( + ); + assert_eq!( + OffsetBuffer::::from(fixed_size_primitive_array_arc).as_ref(), + [1, 1, 2, 2] + ); + } + + #[test] + fn into() { + let offset_buffer = OffsetBuffer::::from_lengths(INPUT); + assert_eq!( + FixedSizePrimitiveArray::::from(offset_buffer.clone()) + .into_iter() + .collect::>(), + offset_buffer.iter().collect::>() + ); + } +} diff --git a/src/arrow/buffer/scalar_buffer.rs b/src/arrow/buffer/scalar_buffer.rs new file mode 100644 index 00000000..8ea3e194 --- /dev/null +++ b/src/arrow/buffer/scalar_buffer.rs @@ -0,0 +1,113 @@ +//! Interop with [`arrow-rs`] scalar buffer. + +use arrow_buffer::ScalarBuffer; + +use crate::{ + array::FixedSizePrimitiveArray, + buffer::{Buffer, BufferType}, + FixedSize, Index, Length, +}; + +/// A [`BufferType`] implementation for [`ScalarBuffer`]. +#[derive(Clone, Copy)] +pub struct ArrowScalarBuffer; + +impl BufferType for ArrowScalarBuffer { + type Buffer = ScalarBuffer; +} + +impl Buffer for ScalarBuffer { + fn as_slice(&self) -> &[T] { + self + } +} + +impl Index for ScalarBuffer { + type Item<'a> = &'a T + where + Self: 'a; + + unsafe fn index_unchecked(&self, index: usize) -> Self::Item<'_> { + self.get_unchecked(index) + } +} + +impl Length for ScalarBuffer { + fn len(&self) -> usize { + self.as_ref().len() + } +} + +impl From> + for ScalarBuffer +where + ::Buffer: AsRef<[T]>, +{ + fn from(value: FixedSizePrimitiveArray) -> Self { + let len = value.len(); + // Note: this makes a copy + let buffer = arrow_buffer::Buffer::from_slice_ref(value.0.as_ref()); + ScalarBuffer::new(buffer, 0, len) + } +} + +impl From> + for FixedSizePrimitiveArray +where + ::Buffer: From, +{ + fn from(value: ScalarBuffer) -> Self { + FixedSizePrimitiveArray(value.into_inner().into()) + } +} + +#[cfg(test)] +mod tests { + use crate::buffer::ArcBuffer; + + use super::*; + + const INPUT: [u32; 4] = [1, 2, 3, 4]; + + #[test] + fn length() { + let scalar_buffer = INPUT.into_iter().collect::>(); + assert_eq!(Length::len(&scalar_buffer), INPUT.len()); + } + + #[test] + fn from() { + let fixed_size_primitive_array = INPUT.into_iter().collect::>(); + assert_eq!( + ScalarBuffer::from(fixed_size_primitive_array) + .into_iter() + .copied() + .collect::>(), + INPUT + ); + + let fixed_size_primitive_array_arc = + INPUT + .into_iter() + .collect::>(); + assert_eq!( + ScalarBuffer::from(fixed_size_primitive_array_arc) + .into_iter() + .copied() + .collect::>(), + INPUT + ); + } + + #[test] + fn into() { + let scalar_buffer = INPUT.into_iter().collect::>(); + assert_eq!( + FixedSizePrimitiveArray::<_, false, ArrowScalarBuffer>::from(scalar_buffer) + .into_iter() + .copied() + .collect::>(), + INPUT + ); + } +} diff --git a/src/arrow/length.rs b/src/arrow/length.rs deleted file mode 100644 index 50639134..00000000 --- a/src/arrow/length.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! Length implementations for [`arrow-rs`] items. - -use crate::Length; -use arrow_buffer::{ArrowNativeType, Buffer, BufferBuilder, ScalarBuffer}; - -impl Length for BufferBuilder { - fn len(&self) -> usize { - BufferBuilder::len(self) - } -} - -impl Length for ScalarBuffer { - fn len(&self) -> usize { - self.as_ref().len() - } -} - -impl Length for Buffer { - fn len(&self) -> usize { - Buffer::len(self) - } -} diff --git a/src/arrow/mod.rs b/src/arrow/mod.rs index 8293e572..42c32ccd 100644 --- a/src/arrow/mod.rs +++ b/src/arrow/mod.rs @@ -3,7 +3,29 @@ //! [`arrow-rs`]: https://crates.io/crates/arrow mod array; -mod bitmap; +// mod bitmap; + mod buffer; pub use buffer::*; -mod length; + +use crate::array::Array; +use arrow_array::ArrayRef; +use arrow_schema::Field; +use std::sync::Arc; + +/// Extension trait of [`Array`] for [`arrow-rs`] interop. +pub trait ArrowArray: Array + Sized { + /// The corresponding arrow array + type Array: arrow_array::Array; // + From + 'static; + + /// Returns as array ref + fn into_array_ref(self) -> ArrayRef + where + Self::Array: From + 'static, + { + Arc::::new(self.into()) + } + + /// Returns the field of this array. + fn as_field(&self, name: &str) -> Field; +}