Improved iterators

src/array/binary/iterator.rs

@@ -1,6 +1,6 @@
 use crate::{
     array::{ArrayAccessor, ArrayValuesIter, Offset},
-    bitmap::utils::ZipValidity,
+    bitmap::utils::{BitmapIter, ZipValidity},
 };
 
 use super::BinaryArray;
@@ -24,7 +24,7 @@ pub type BinaryValueIter<'a, O> = ArrayValuesIter<'a, BinaryArray<O>>;
 
 impl<'a, O: Offset> IntoIterator for &'a BinaryArray<O> {
     type Item = Option<&'a [u8]>;
-    type IntoIter = ZipValidity<'a, &'a [u8], BinaryValueIter<'a, O>>;
+    type IntoIter = ZipValidity<&'a [u8], BinaryValueIter<'a, O>, BitmapIter<'a>>;
 
     fn into_iter(self) -> Self::IntoIter {
         self.iter()

src/array/binary/mod.rs

@@ -1,6 +1,6 @@
 use crate::{
     bitmap::{
-        utils::{zip_validity, ZipValidity},
+        utils::{zip_validity, BitmapIter, ZipValidity},
         Bitmap,
     },
     buffer::Buffer,
@@ -116,7 +116,7 @@ impl<O: Offset> BinaryArray<O> {
     }
 
     /// Returns an iterator of `Option<&[u8]>` over every element of this array.
-    pub fn iter(&self) -> ZipValidity<&[u8], BinaryValueIter<O>> {
+    pub fn iter(&self) -> ZipValidity<&[u8], BinaryValueIter<O>, BitmapIter> {
         zip_validity(self.values_iter(), self.validity.as_ref().map(|x| x.iter()))
     }
 

src/array/boolean/iterator.rs

@@ -1,21 +1,35 @@
 use crate::bitmap::utils::{zip_validity, BitmapIter, ZipValidity};
+use crate::bitmap::IntoIter;
 
 use super::super::MutableArray;
 use super::{BooleanArray, MutableBooleanArray};
 
 impl<'a> IntoIterator for &'a BooleanArray {
     type Item = Option<bool>;
-    type IntoIter = ZipValidity<'a, bool, BitmapIter<'a>>;
+    type IntoIter = ZipValidity<bool, BitmapIter<'a>, BitmapIter<'a>>;
 
     #[inline]
     fn into_iter(self) -> Self::IntoIter {
         self.iter()
     }
 }
 
+impl IntoIterator for BooleanArray {
+    type Item = Option<bool>;
+    type IntoIter = ZipValidity<bool, IntoIter, IntoIter>;
+
+    #[inline]
+    fn into_iter(self) -> Self::IntoIter {
+        let (_, values, validity) = self.into_inner();
+        let values = values.into_iter();
+        let validity = validity.map(|x| x.into_iter());
+        ZipValidity::new(values, validity)
+    }
+}
+
 impl<'a> IntoIterator for &'a MutableBooleanArray {
     type Item = Option<bool>;
-    type IntoIter = ZipValidity<'a, bool, BitmapIter<'a>>;
+    type IntoIter = ZipValidity<bool, BitmapIter<'a>, BitmapIter<'a>>;
 
     #[inline]
     fn into_iter(self) -> Self::IntoIter {
@@ -26,7 +40,7 @@ impl<'a> IntoIterator for &'a MutableBooleanArray {
 impl<'a> MutableBooleanArray {
     /// Returns an iterator over the optional values of this [`MutableBooleanArray`].
     #[inline]
-    pub fn iter(&'a self) -> ZipValidity<'a, bool, BitmapIter<'a>> {
+    pub fn iter(&'a self) -> ZipValidity<bool, BitmapIter<'a>, BitmapIter<'a>> {
         zip_validity(
             self.values().iter(),
             self.validity().as_ref().map(|x| x.iter()),

src/array/boolean/mod.rs

@@ -87,7 +87,7 @@ impl BooleanArray {
 
     /// Returns an iterator over the optional values of this [`BooleanArray`].
     #[inline]
-    pub fn iter(&self) -> ZipValidity<bool, BitmapIter> {
+    pub fn iter(&self) -> ZipValidity<bool, BitmapIter, BitmapIter> {
         zip_validity(
             self.values().iter(),
             self.validity.as_ref().map(|x| x.iter()),
@@ -361,6 +361,17 @@ impl BooleanArray {
         std::sync::Arc::new(self)
     }
 
+    /// Returns its internal representation
+    #[must_use]
+    pub fn into_inner(self) -> (DataType, Bitmap, Option<Bitmap>) {
+        let Self {
+            data_type,
+            values,
+            validity,
+        } = self;
+        (data_type, values, validity)
+    }
+
     /// The canonical method to create a [`BooleanArray`]
     /// # Panics
     /// This function errors iff:

src/array/dictionary/iterator.rs

@@ -1,4 +1,4 @@
-use crate::bitmap::utils::ZipValidity;
+use crate::bitmap::utils::{BitmapIter, ZipValidity};
 use crate::scalar::Scalar;
 use crate::trusted_len::TrustedLen;
 
@@ -56,7 +56,7 @@ impl<'a, K: DictionaryKey> DoubleEndedIterator for DictionaryValuesIter<'a, K> {
 }
 
 type ValuesIter<'a, K> = DictionaryValuesIter<'a, K>;
-type ZipIter<'a, K> = ZipValidity<'a, Box<dyn Scalar>, ValuesIter<'a, K>>;
+type ZipIter<'a, K> = ZipValidity<Box<dyn Scalar>, ValuesIter<'a, K>, BitmapIter<'a>>;
 
 impl<'a, K: DictionaryKey> IntoIterator for &'a DictionaryArray<K> {
     type Item = Option<Box<dyn Scalar>>;

src/array/dictionary/mod.rs

@@ -2,7 +2,7 @@ use std::hint::unreachable_unchecked;
 
 use crate::{
     bitmap::{
-        utils::{zip_validity, ZipValidity},
+        utils::{zip_validity, BitmapIter, ZipValidity},
         Bitmap,
     },
     datatypes::{DataType, IntegerType},
@@ -190,7 +190,7 @@ impl<K: DictionaryKey> DictionaryArray<K> {
     /// # Implementation
     /// This function will allocate a new [`Scalar`] per item and is usually not performant.
     /// Consider calling `keys_iter` and `values`, downcasting `values`, and iterating over that.
-    pub fn iter(&self) -> ZipValidity<Box<dyn Scalar>, DictionaryValuesIter<K>> {
+    pub fn iter(&self) -> ZipValidity<Box<dyn Scalar>, DictionaryValuesIter<K>, BitmapIter> {
         zip_validity(
             DictionaryValuesIter::new(self),
             self.keys.validity().as_ref().map(|x| x.iter()),

src/array/fixed_size_binary/iterator.rs

@@ -1,5 +1,5 @@
 use crate::array::{ArrayAccessor, ArrayValuesIter, MutableArray};
-use crate::bitmap::utils::{zip_validity, ZipValidity};
+use crate::bitmap::utils::{zip_validity, BitmapIter, ZipValidity};
 
 use super::{FixedSizeBinaryArray, MutableFixedSizeBinaryArray};
 
@@ -37,7 +37,7 @@ pub type MutableFixedSizeBinaryValuesIter<'a> = ArrayValuesIter<'a, MutableFixed
 
 impl<'a> IntoIterator for &'a FixedSizeBinaryArray {
     type Item = Option<&'a [u8]>;
-    type IntoIter = ZipValidity<'a, &'a [u8], FixedSizeBinaryValuesIter<'a>>;
+    type IntoIter = ZipValidity<&'a [u8], FixedSizeBinaryValuesIter<'a>, BitmapIter<'a>>;
 
     fn into_iter(self) -> Self::IntoIter {
         self.iter()
@@ -46,7 +46,7 @@ impl<'a> IntoIterator for &'a FixedSizeBinaryArray {
 
 impl<'a> FixedSizeBinaryArray {
     /// constructs a new iterator
-    pub fn iter(&'a self) -> ZipValidity<'a, &'a [u8], FixedSizeBinaryValuesIter<'a>> {
+    pub fn iter(&'a self) -> ZipValidity<&'a [u8], FixedSizeBinaryValuesIter<'a>, BitmapIter<'a>> {
         zip_validity(self.values_iter(), self.validity.as_ref().map(|x| x.iter()))
     }
 
@@ -58,7 +58,7 @@ impl<'a> FixedSizeBinaryArray {
 
 impl<'a> IntoIterator for &'a MutableFixedSizeBinaryArray {
     type Item = Option<&'a [u8]>;
-    type IntoIter = ZipValidity<'a, &'a [u8], MutableFixedSizeBinaryValuesIter<'a>>;
+    type IntoIter = ZipValidity<&'a [u8], MutableFixedSizeBinaryValuesIter<'a>, BitmapIter<'a>>;
 
     fn into_iter(self) -> Self::IntoIter {
         self.iter()
@@ -67,7 +67,9 @@ impl<'a> IntoIterator for &'a MutableFixedSizeBinaryArray {
 
 impl<'a> MutableFixedSizeBinaryArray {
     /// constructs a new iterator
-    pub fn iter(&'a self) -> ZipValidity<'a, &'a [u8], MutableFixedSizeBinaryValuesIter<'a>> {
+    pub fn iter(
+        &'a self,
+    ) -> ZipValidity<&'a [u8], MutableFixedSizeBinaryValuesIter<'a>, BitmapIter<'a>> {
         zip_validity(
             self.iter_values(),
             self.validity().as_ref().map(|x| x.iter()),

src/array/fixed_size_list/iterator.rs

@@ -1,6 +1,6 @@
 use crate::{
     array::{Array, ArrayAccessor, ArrayValuesIter},
-    bitmap::utils::{zip_validity, ZipValidity},
+    bitmap::utils::{zip_validity, BitmapIter, ZipValidity},
 };
 
 use super::FixedSizeListArray;
@@ -22,7 +22,7 @@ unsafe impl<'a> ArrayAccessor<'a> for FixedSizeListArray {
 /// Iterator of values of a [`FixedSizeListArray`].
 pub type FixedSizeListValuesIter<'a> = ArrayValuesIter<'a, FixedSizeListArray>;
 
-type ZipIter<'a> = ZipValidity<'a, Box<dyn Array>, FixedSizeListValuesIter<'a>>;
+type ZipIter<'a> = ZipValidity<Box<dyn Array>, FixedSizeListValuesIter<'a>, BitmapIter<'a>>;
 
 impl<'a> IntoIterator for &'a FixedSizeListArray {
     type Item = Option<Box<dyn Array>>;

src/array/list/iterator.rs

@@ -1,6 +1,6 @@
 use crate::array::Offset;
 use crate::array::{Array, ArrayAccessor, ArrayValuesIter};
-use crate::bitmap::utils::{zip_validity, ZipValidity};
+use crate::bitmap::utils::{zip_validity, BitmapIter, ZipValidity};
 
 use super::ListArray;
 
@@ -21,7 +21,7 @@ unsafe impl<'a, O: Offset> ArrayAccessor<'a> for ListArray<O> {
 /// Iterator of values of a [`ListArray`].
 pub type ListValuesIter<'a, O> = ArrayValuesIter<'a, ListArray<O>>;
 
-type ZipIter<'a, O> = ZipValidity<'a, Box<dyn Array>, ListValuesIter<'a, O>>;
+type ZipIter<'a, O> = ZipValidity<Box<dyn Array>, ListValuesIter<'a, O>, BitmapIter<'a>>;
 
 impl<'a, O: Offset> IntoIterator for &'a ListArray<O> {
     type Item = Option<Box<dyn Array>>;

src/array/map/iterator.rs

@@ -1,5 +1,5 @@
 use crate::array::Array;
-use crate::bitmap::utils::{zip_validity, ZipValidity};
+use crate::bitmap::utils::{zip_validity, BitmapIter, ZipValidity};
 use crate::trusted_len::TrustedLen;
 
 use super::MapArray;
@@ -62,7 +62,7 @@ impl<'a> DoubleEndedIterator for MapValuesIter<'a> {
 
 impl<'a> IntoIterator for &'a MapArray {
     type Item = Option<Box<dyn Array>>;
-    type IntoIter = ZipValidity<'a, Box<dyn Array>, MapValuesIter<'a>>;
+    type IntoIter = ZipValidity<Box<dyn Array>, MapValuesIter<'a>, BitmapIter<'a>>;
 
     fn into_iter(self) -> Self::IntoIter {
         self.iter()
@@ -71,7 +71,7 @@ impl<'a> IntoIterator for &'a MapArray {
 
 impl<'a> MapArray {
     /// Returns an iterator of `Option<Box<dyn Array>>`
-    pub fn iter(&'a self) -> ZipValidity<'a, Box<dyn Array>, MapValuesIter<'a>> {
+    pub fn iter(&'a self) -> ZipValidity<Box<dyn Array>, MapValuesIter<'a>, BitmapIter<'a>> {
         zip_validity(
             MapValuesIter::new(self),
             self.validity.as_ref().map(|x| x.iter()),

src/array/physical_binary.rs

@@ -159,7 +159,6 @@ pub(crate) unsafe fn extend_from_trusted_len_values_iter<I, P, O>(
 
 // Populates `offsets` and `values` [`Vec`]s with information extracted
 // from the incoming `iterator`.
-
 // the return value indicates how many items were added.
 #[inline]
 pub(crate) fn extend_from_values_iter<I, P, O>(

src/array/primitive/iterator.rs

@@ -1,14 +1,29 @@
 use crate::{
-    bitmap::utils::{zip_validity, ZipValidity},
+    array::MutableArray,
+    bitmap::utils::{zip_validity, BitmapIter, ZipValidity},
+    bitmap::IntoIter as BitmapIntoIter,
+    buffer::IntoIter,
     types::NativeType,
 };
 
-use super::super::MutableArray;
 use super::{MutablePrimitiveArray, PrimitiveArray};
 
+impl<T: NativeType> IntoIterator for PrimitiveArray<T> {
+    type Item = Option<T>;
+    type IntoIter = ZipValidity<T, IntoIter<T>, BitmapIntoIter>;
+
+    #[inline]
+    fn into_iter(self) -> Self::IntoIter {
+        let (_, values, validity) = self.into_inner();
+        let values = values.into_iter();
+        let validity = validity.map(|x| x.into_iter());
+        ZipValidity::new(values, validity)
+    }
+}
+
 impl<'a, T: NativeType> IntoIterator for &'a PrimitiveArray<T> {
     type Item = Option<&'a T>;
-    type IntoIter = ZipValidity<'a, &'a T, std::slice::Iter<'a, T>>;
+    type IntoIter = ZipValidity<&'a T, std::slice::Iter<'a, T>, BitmapIter<'a>>;
 
     #[inline]
     fn into_iter(self) -> Self::IntoIter {
@@ -19,14 +34,14 @@ impl<'a, T: NativeType> IntoIterator for &'a PrimitiveArray<T> {
 impl<'a, T: NativeType> MutablePrimitiveArray<T> {
     /// Returns an iterator over `Option<T>`
     #[inline]
-    pub fn iter(&'a self) -> ZipValidity<'a, &'a T, std::slice::Iter<'a, T>> {
+    pub fn iter(&'a self) -> ZipValidity<&'a T, std::slice::Iter<'a, T>, BitmapIter<'a>> {
         zip_validity(
             self.values().iter(),
             self.validity().as_ref().map(|x| x.iter()),
         )
     }
 
-    /// Returns an iterator of `bool`
+    /// Returns an iterator of `T`
     #[inline]
     pub fn values_iter(&'a self) -> std::slice::Iter<'a, T> {
         self.values().iter()

src/array/primitive/mod.rs

@@ -1,6 +1,6 @@
 use crate::{
     bitmap::{
-        utils::{zip_validity, ZipValidity},
+        utils::{zip_validity, BitmapIter, ZipValidity},
         Bitmap,
     },
     buffer::Buffer,
@@ -140,7 +140,7 @@ impl<T: NativeType> PrimitiveArray<T> {
 
     /// Returns an iterator over the values and validity, `Option<&T>`.
     #[inline]
-    pub fn iter(&self) -> ZipValidity<&T, std::slice::Iter<T>> {
+    pub fn iter(&self) -> ZipValidity<&T, std::slice::Iter<T>, BitmapIter> {
         zip_validity(
             self.values().iter(),
             self.validity().as_ref().map(|x| x.iter()),
@@ -297,6 +297,17 @@ impl<T: NativeType> PrimitiveArray<T> {
         self.values.get_mut().map(|x| x.as_mut())
     }
 
+    /// Returns its internal representation
+    #[must_use]
+    pub fn into_inner(self) -> (DataType, Buffer<T>, Option<Bitmap>) {
+        let Self {
+            data_type,
+            values,
+            validity,
+        } = self;
+        (data_type, values, validity)
+    }
+
     /// Try to convert this [`PrimitiveArray`] to a [`MutablePrimitiveArray`] via copy-on-write semantics.
     ///
     /// A [`PrimitiveArray`] is backed by a [`Buffer`] and [`Bitmap`] which are essentially `Arc<Vec<_>>`.

src/array/struct_/iterator.rs

@@ -1,5 +1,5 @@
 use crate::{
-    bitmap::utils::{zip_validity, ZipValidity},
+    bitmap::utils::{zip_validity, BitmapIter, ZipValidity},
     scalar::{new_scalar, Scalar},
     trusted_len::TrustedLen,
 };
@@ -75,7 +75,7 @@ impl<'a> DoubleEndedIterator for StructValueIter<'a> {
 }
 
 type ValuesIter<'a> = StructValueIter<'a>;
-type ZipIter<'a> = ZipValidity<'a, Vec<Box<dyn Scalar>>, ValuesIter<'a>>;
+type ZipIter<'a> = ZipValidity<Vec<Box<dyn Scalar>>, ValuesIter<'a>, BitmapIter<'a>>;
 
 impl<'a> IntoIterator for &'a StructArray {
     type Item = Option<Vec<Box<dyn Scalar>>>;

src/array/utf8/iterator.rs

@@ -1,5 +1,5 @@
 use crate::array::{ArrayAccessor, ArrayValuesIter, Offset};
-use crate::bitmap::utils::ZipValidity;
+use crate::bitmap::utils::{BitmapIter, ZipValidity};
 
 use super::{MutableUtf8Array, MutableUtf8ValuesArray, Utf8Array};
 
@@ -22,7 +22,7 @@ pub type Utf8ValuesIter<'a, O> = ArrayValuesIter<'a, Utf8Array<O>>;
 
 impl<'a, O: Offset> IntoIterator for &'a Utf8Array<O> {
     type Item = Option<&'a str>;
-    type IntoIter = ZipValidity<'a, &'a str, Utf8ValuesIter<'a, O>>;
+    type IntoIter = ZipValidity<&'a str, Utf8ValuesIter<'a, O>, BitmapIter<'a>>;
 
     fn into_iter(self) -> Self::IntoIter {
         self.iter()
@@ -48,7 +48,7 @@ pub type MutableUtf8ValuesIter<'a, O> = ArrayValuesIter<'a, MutableUtf8ValuesArr
 
 impl<'a, O: Offset> IntoIterator for &'a MutableUtf8Array<O> {
     type Item = Option<&'a str>;
-    type IntoIter = ZipValidity<'a, &'a str, MutableUtf8ValuesIter<'a, O>>;
+    type IntoIter = ZipValidity<&'a str, MutableUtf8ValuesIter<'a, O>, BitmapIter<'a>>;
 
     fn into_iter(self) -> Self::IntoIter {
         self.iter()

src/array/utf8/mod.rs

@@ -1,6 +1,6 @@
 use crate::{
     bitmap::{
-        utils::{zip_validity, ZipValidity},
+        utils::{zip_validity, BitmapIter, ZipValidity},
         Bitmap,
     },
     buffer::Buffer,
@@ -132,7 +132,7 @@ impl<O: Offset> Utf8Array<O> {
     }
 
     /// Returns an iterator of `Option<&str>`
-    pub fn iter(&self) -> ZipValidity<&str, Utf8ValuesIter<O>> {
+    pub fn iter(&self) -> ZipValidity<&str, Utf8ValuesIter<O>, BitmapIter> {
         zip_validity(self.values_iter(), self.validity.as_ref().map(|x| x.iter()))
     }