From 64bf7bfad8e956d7a2da59e0f05be77192376bba Mon Sep 17 00:00:00 2001 From: Orson Peters Date: Thu, 7 Sep 2023 06:50:28 +0200 Subject: [PATCH] refactor: genericize PolarsDataType (#10952) --- crates/polars-core/src/chunked_array/from.rs | 13 +- .../src/chunked_array/list/iterator.rs | 3 +- .../src/chunked_array/ops/apply.rs | 27 +- .../src/chunked_array/ops/arity.rs | 92 ++---- .../src/chunked_array/ops/downcast.rs | 21 +- .../polars-core/src/chunked_array/ops/mod.rs | 3 +- crates/polars-core/src/datatypes/mod.rs | 287 +++++++----------- crates/polars-ops/src/series/ops/is_in.rs | 3 +- .../sinks/group_by/aggregates/mean.rs | 4 +- .../sinks/group_by/aggregates/min_max.rs | 2 +- .../sinks/group_by/aggregates/sum.rs | 4 +- 11 files changed, 157 insertions(+), 302 deletions(-) diff --git a/crates/polars-core/src/chunked_array/from.rs b/crates/polars-core/src/chunked_array/from.rs index 7e55c2d04355..b59ac906421e 100644 --- a/crates/polars-core/src/chunked_array/from.rs +++ b/crates/polars-core/src/chunked_array/from.rs @@ -72,8 +72,8 @@ fn from_chunks_list_dtype(chunks: &mut Vec, dtype: DataType) -> DataTy impl From for ChunkedArray where - T: PolarsDataType, - A: StaticallyMatchesPolarsType + Array, + T: PolarsDataType, + A: Array, { fn from(arr: A) -> Self { Self::with_chunk("", arr) @@ -86,7 +86,8 @@ where { pub fn with_chunk(name: &str, arr: A) -> Self where - A: StaticallyMatchesPolarsType + Array, + A: Array, + T: PolarsDataType, { unsafe { Self::from_chunks(name, vec![Box::new(arr)]) } } @@ -94,7 +95,8 @@ where pub fn from_chunk_iter(name: &str, iter: I) -> Self where I: IntoIterator, - ::Item: StaticallyMatchesPolarsType + Array, + T: PolarsDataType::Item>, + ::Item: Array, { let chunks = iter .into_iter() @@ -106,7 +108,8 @@ where pub fn try_from_chunk_iter(name: &str, iter: I) -> Result where I: IntoIterator>, - A: StaticallyMatchesPolarsType + Array, + T: PolarsDataType, + A: Array, { let chunks: Result<_, _> = iter .into_iter() diff --git a/crates/polars-core/src/chunked_array/list/iterator.rs b/crates/polars-core/src/chunked_array/list/iterator.rs index c22febe58a67..2363f18c1402 100644 --- a/crates/polars-core/src/chunked_array/list/iterator.rs +++ b/crates/polars-core/src/chunked_array/list/iterator.rs @@ -173,8 +173,7 @@ impl ListChunked { where V: PolarsDataType, F: FnMut(Option>) -> Option + Copy, - K: ArrayFromElementIter, - K::ArrayType: StaticallyMatchesPolarsType, + K: ArrayFromElementIter, { // TODO! make an amortized iter that does not flatten diff --git a/crates/polars-core/src/chunked_array/ops/apply.rs b/crates/polars-core/src/chunked_array/ops/apply.rs index a54321dcdc6b..b78cc9461bdc 100644 --- a/crates/polars-core/src/chunked_array/ops/apply.rs +++ b/crates/polars-core/src/chunked_array/ops/apply.rs @@ -17,14 +17,12 @@ use crate::utils::CustomIterTools; impl ChunkedArray where T: PolarsDataType, - Self: HasUnderlyingArray, { pub fn apply_values_generic<'a, U, K, F>(&'a self, op: F) -> ChunkedArray where U: PolarsDataType, - F: FnMut(<::ArrayT as StaticArray>::ValueT<'a>) -> K + Copy, - K: ArrayFromElementIter, - K::ArrayType: StaticallyMatchesPolarsType, + F: FnMut(T::Physical<'a>) -> K + Copy, + K: ArrayFromElementIter, { let iter = self.downcast_iter().map(|arr| { let element_iter = arr.values_iter().map(op); @@ -38,10 +36,8 @@ where pub fn try_apply_values_generic<'a, U, K, F, E>(&'a self, op: F) -> Result, E> where U: PolarsDataType, - F: FnMut(<::ArrayT as StaticArray>::ValueT<'a>) -> Result - + Copy, - K: ArrayFromElementIter, - K::ArrayType: StaticallyMatchesPolarsType, + F: FnMut(T::Physical<'a>) -> Result + Copy, + K: ArrayFromElementIter, E: Error, { let iter = self.downcast_iter().map(|arr| { @@ -56,12 +52,8 @@ where pub fn try_apply_generic<'a, U, K, F, E>(&'a self, op: F) -> Result, E> where U: PolarsDataType, - F: FnMut( - Option<<::ArrayT as StaticArray>::ValueT<'a>>, - ) -> Result, E> - + Copy, - K: ArrayFromElementIter, - K::ArrayType: StaticallyMatchesPolarsType, + F: FnMut(Option>) -> Result, E> + Copy, + K: ArrayFromElementIter, E: Error, { let iter = self.downcast_iter().map(|arr| { @@ -76,11 +68,8 @@ where pub fn apply_generic<'a, U, K, F>(&'a self, mut op: F) -> ChunkedArray where U: PolarsDataType, - F: FnMut( - Option<<::ArrayT as StaticArray>::ValueT<'a>>, - ) -> Option, - K: ArrayFromElementIter, - K::ArrayType: StaticallyMatchesPolarsType, + F: FnMut(Option>) -> Option, + K: ArrayFromElementIter, { if self.null_count() == 0 { let iter = self.downcast_iter().map(|arr| { diff --git a/crates/polars-core/src/chunked_array/ops/arity.rs b/crates/polars-core/src/chunked_array/ops/arity.rs index 4214c41deccc..2613c49b4008 100644 --- a/crates/polars-core/src/chunked_array/ops/arity.rs +++ b/crates/polars-core/src/chunked_array/ops/arity.rs @@ -3,10 +3,7 @@ use std::error::Error; use arrow::array::Array; use polars_arrow::utils::combine_validities_and; -use crate::datatypes::{ - ArrayFromElementIter, HasUnderlyingArray, PolarsNumericType, StaticArray, - StaticallyMatchesPolarsType, -}; +use crate::datatypes::{ArrayFromElementIter, PolarsNumericType, StaticArray}; use crate::prelude::{ChunkedArray, PolarsDataType}; use crate::utils::align_chunks_binary; @@ -20,14 +17,8 @@ where T: PolarsDataType, U: PolarsDataType, V: PolarsDataType, - ChunkedArray: HasUnderlyingArray, - ChunkedArray: HasUnderlyingArray, - F: for<'a> FnMut( - Option<< as HasUnderlyingArray>::ArrayT as StaticArray>::ValueT<'a>>, - Option<< as HasUnderlyingArray>::ArrayT as StaticArray>::ValueT<'a>>, - ) -> Option, - K: ArrayFromElementIter, - K::ArrayType: StaticallyMatchesPolarsType, + F: for<'a> FnMut(Option>, Option>) -> Option, + K: ArrayFromElementIter, { let (lhs, rhs) = align_chunks_binary(lhs, rhs); let iter = lhs @@ -53,14 +44,8 @@ where T: PolarsDataType, U: PolarsDataType, V: PolarsDataType, - ChunkedArray: HasUnderlyingArray, - ChunkedArray: HasUnderlyingArray, - F: for<'a> FnMut( - Option<< as HasUnderlyingArray>::ArrayT as StaticArray>::ValueT<'a>>, - Option<< as HasUnderlyingArray>::ArrayT as StaticArray>::ValueT<'a>>, - ) -> Result, E>, - K: ArrayFromElementIter, - K::ArrayType: StaticallyMatchesPolarsType, + F: for<'a> FnMut(Option>, Option>) -> Result, E>, + K: ArrayFromElementIter, E: Error, { let (lhs, rhs) = align_chunks_binary(lhs, rhs); @@ -87,14 +72,8 @@ where T: PolarsDataType, U: PolarsDataType, V: PolarsNumericType, - ChunkedArray: HasUnderlyingArray, - ChunkedArray: HasUnderlyingArray, - F: for<'a> FnMut( - < as HasUnderlyingArray>::ArrayT as StaticArray>::ValueT<'a>, - < as HasUnderlyingArray>::ArrayT as StaticArray>::ValueT<'a>, - ) -> K, - K: ArrayFromElementIter, - K::ArrayType: StaticallyMatchesPolarsType, + F: for<'a> FnMut(T::Physical<'a>, U::Physical<'a>) -> K, + K: ArrayFromElementIter, { let (lhs, rhs) = align_chunks_binary(lhs, rhs); let iter = lhs @@ -124,14 +103,8 @@ where T: PolarsDataType, U: PolarsDataType, V: PolarsNumericType, - ChunkedArray: HasUnderlyingArray, - ChunkedArray: HasUnderlyingArray, - F: for<'a> FnMut( - < as HasUnderlyingArray>::ArrayT as StaticArray>::ValueT<'a>, - < as HasUnderlyingArray>::ArrayT as StaticArray>::ValueT<'a>, - ) -> Result, - K: ArrayFromElementIter, - K::ArrayType: StaticallyMatchesPolarsType, + F: for<'a> FnMut(T::Physical<'a>, U::Physical<'a>) -> Result, + K: ArrayFromElementIter, E: Error, { let (lhs, rhs) = align_chunks_binary(lhs, rhs); @@ -163,14 +136,9 @@ pub fn binary_mut_with_options( where T: PolarsDataType, U: PolarsDataType, - V: PolarsDataType, - ChunkedArray: HasUnderlyingArray, - ChunkedArray: HasUnderlyingArray, - Arr: Array + StaticallyMatchesPolarsType, - F: FnMut( - & as HasUnderlyingArray>::ArrayT, - & as HasUnderlyingArray>::ArrayT, - ) -> Arr, + V: PolarsDataType, + Arr: Array, + F: FnMut(&T::Array, &U::Array) -> Arr, { let (lhs, rhs) = align_chunks_binary(lhs, rhs); let iter = lhs @@ -189,14 +157,9 @@ pub fn binary( where T: PolarsDataType, U: PolarsDataType, - V: PolarsDataType, - ChunkedArray: HasUnderlyingArray, - ChunkedArray: HasUnderlyingArray, - Arr: Array + StaticallyMatchesPolarsType, - F: FnMut( - & as HasUnderlyingArray>::ArrayT, - & as HasUnderlyingArray>::ArrayT, - ) -> Arr, + V: PolarsDataType, + Arr: Array, + F: FnMut(&T::Array, &U::Array) -> Arr, { binary_mut_with_options(lhs, rhs, op, lhs.name()) } @@ -210,14 +173,9 @@ pub fn try_binary( where T: PolarsDataType, U: PolarsDataType, - V: PolarsDataType, - ChunkedArray: HasUnderlyingArray, - ChunkedArray: HasUnderlyingArray, - Arr: Array + StaticallyMatchesPolarsType, - F: FnMut( - & as HasUnderlyingArray>::ArrayT, - & as HasUnderlyingArray>::ArrayT, - ) -> Result, + V: PolarsDataType, + Arr: Array, + F: FnMut(&T::Array, &U::Array) -> Result, E: Error, { let (lhs, rhs) = align_chunks_binary(lhs, rhs); @@ -243,12 +201,7 @@ pub unsafe fn binary_unchecked_same_type( where T: PolarsDataType, U: PolarsDataType, - ChunkedArray: HasUnderlyingArray, - ChunkedArray: HasUnderlyingArray, - F: FnMut( - & as HasUnderlyingArray>::ArrayT, - & as HasUnderlyingArray>::ArrayT, - ) -> Box, + F: FnMut(&T::Array, &U::Array) -> Box, { let (lhs, rhs) = align_chunks_binary(lhs, rhs); let chunks = lhs @@ -274,12 +227,7 @@ pub unsafe fn try_binary_unchecked_same_type( where T: PolarsDataType, U: PolarsDataType, - ChunkedArray: HasUnderlyingArray, - ChunkedArray: HasUnderlyingArray, - F: FnMut( - & as HasUnderlyingArray>::ArrayT, - & as HasUnderlyingArray>::ArrayT, - ) -> Result, E>, + F: FnMut(&T::Array, &U::Array) -> Result, E>, E: Error, { let (lhs, rhs) = align_chunks_binary(lhs, rhs); diff --git a/crates/polars-core/src/chunked_array/ops/downcast.rs b/crates/polars-core/src/chunked_array/ops/downcast.rs index 66197f51efb7..f2dfef3a6adc 100644 --- a/crates/polars-core/src/chunked_array/ops/downcast.rs +++ b/crates/polars-core/src/chunked_array/ops/downcast.rs @@ -47,18 +47,13 @@ impl<'a, T> Chunks<'a, T> { } #[doc(hidden)] -impl ChunkedArray -where - Self: HasUnderlyingArray, -{ +impl ChunkedArray { #[inline] - pub fn downcast_iter( - &self, - ) -> impl Iterator::ArrayT> + DoubleEndedIterator { + pub fn downcast_iter(&self) -> impl Iterator + DoubleEndedIterator { self.chunks.iter().map(|arr| { - // SAFETY: HasUnderlyingArray guarantees this is correct. + // SAFETY: T::Array guarantees this is correct. let arr = &**arr; - unsafe { &*(arr as *const dyn Array as *const ::ArrayT) } + unsafe { &*(arr as *const dyn Array as *const T::Array) } }) } @@ -69,16 +64,16 @@ where #[inline] pub unsafe fn downcast_iter_mut( &mut self, - ) -> impl Iterator::ArrayT> + DoubleEndedIterator { + ) -> impl Iterator + DoubleEndedIterator { self.chunks.iter_mut().map(|arr| { - // SAFETY: HasUnderlyingArray guarantees this is correct. + // SAFETY: T::Array guarantees this is correct. let arr = &mut **arr; - &mut *(arr as *mut dyn Array as *mut ::ArrayT) + &mut *(arr as *mut dyn Array as *mut T::Array) }) } #[inline] - pub fn downcast_chunks(&self) -> Chunks<'_, ::ArrayT> { + pub fn downcast_chunks(&self) -> Chunks<'_, T::Array> { Chunks::new(&self.chunks) } diff --git a/crates/polars-core/src/chunked_array/ops/mod.rs b/crates/polars-core/src/chunked_array/ops/mod.rs index 88817eed017c..fe509fd95c33 100644 --- a/crates/polars-core/src/chunked_array/ops/mod.rs +++ b/crates/polars-core/src/chunked_array/ops/mod.rs @@ -589,10 +589,9 @@ macro_rules! impl_chunk_expand { }}; } -impl ChunkExpandAtIndex for ChunkedArray +impl ChunkExpandAtIndex for ChunkedArray where ChunkedArray: ChunkFull + TakeRandom, - T: PolarsNumericType, { fn new_from_index(&self, index: usize, length: usize) -> ChunkedArray { let mut out = impl_chunk_expand!(self, length, index); diff --git a/crates/polars-core/src/datatypes/mod.rs b/crates/polars-core/src/datatypes/mod.rs index e4d592b4e3c5..c4279eb56ec2 100644 --- a/crates/polars-core/src/datatypes/mod.rs +++ b/crates/polars-core/src/datatypes/mod.rs @@ -47,124 +47,149 @@ pub use time_unit::*; use crate::chunked_array::arithmetic::ArrayArithmetics; pub use crate::chunked_array::logical::*; #[cfg(feature = "object")] +use crate::chunked_array::object::ObjectArray; +#[cfg(feature = "object")] use crate::chunked_array::object::PolarsObjectSafe; use crate::prelude::*; use crate::utils::Wrap; -pub struct Utf8Type {} - -pub struct BinaryType {} - -#[cfg(feature = "dtype-array")] -pub struct FixedSizeListType {} - -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -pub struct ListType {} +/// # Safety +/// +/// The StaticArray and dtype return must be correct. +pub unsafe trait PolarsDataType: Send + Sync + Sized { + type Physical<'a>; + type Array: for<'a> StaticArray = Self::Physical<'a>>; -pub trait PolarsDataType: Send + Sync { fn get_dtype() -> DataType where Self: Sized; } -macro_rules! impl_polars_datatype { - ($ca:ident, $variant:ident, $physical:ty) => { +pub trait PolarsNumericType: 'static +where + Self: for<'a> PolarsDataType = Self::Native, Array = PrimitiveArray>, +{ + type Native: NumericNative; +} + +pub trait PolarsIntegerType: PolarsNumericType {} +pub trait PolarsFloatType: PolarsNumericType {} + +macro_rules! impl_polars_num_datatype { + ($trait: ident, $ca:ident, $variant:ident, $physical:ty) => { #[derive(Clone, Copy)] pub struct $ca {} - impl PolarsDataType for $ca { + unsafe impl PolarsDataType for $ca { + type Physical<'a> = $physical; + type Array = PrimitiveArray<$physical>; + #[inline] fn get_dtype() -> DataType { DataType::$variant } } + + impl PolarsNumericType for $ca { + type Native = $physical; + } + + impl $trait for $ca {} }; } -impl_polars_datatype!(UInt8Type, UInt8, u8); -impl_polars_datatype!(UInt16Type, UInt16, u16); -impl_polars_datatype!(UInt32Type, UInt32, u32); -impl_polars_datatype!(UInt64Type, UInt64, u64); -impl_polars_datatype!(Int8Type, Int8, i8); -impl_polars_datatype!(Int16Type, Int16, i16); -impl_polars_datatype!(Int32Type, Int32, i32); -impl_polars_datatype!(Int64Type, Int64, i64); -impl_polars_datatype!(Float32Type, Float32, f32); -impl_polars_datatype!(Float64Type, Float64, f64); -impl_polars_datatype!(DateType, Date, i32); -#[cfg(feature = "dtype-decimal")] -impl_polars_datatype!(DecimalType, Unknown, i128); -impl_polars_datatype!(DatetimeType, Unknown, i64); -impl_polars_datatype!(DurationType, Unknown, i64); -impl_polars_datatype!(CategoricalType, Unknown, u32); -impl_polars_datatype!(TimeType, Time, i64); +macro_rules! impl_polars_datatype { + ($ca:ident, $variant:ident, $arr:ty, $lt:lifetime, $phys:ty) => { + #[derive(Clone, Copy)] + pub struct $ca {} -impl PolarsDataType for Utf8Type { - fn get_dtype() -> DataType { - DataType::Utf8 - } -} + unsafe impl PolarsDataType for $ca { + type Physical<$lt> = $phys; + type Array = $arr; -impl PolarsDataType for BinaryType { - fn get_dtype() -> DataType { - DataType::Binary - } + #[inline] + fn get_dtype() -> DataType { + DataType::$variant + } + } + }; } -pub struct BooleanType {} +impl_polars_num_datatype!(PolarsIntegerType, UInt8Type, UInt8, u8); +impl_polars_num_datatype!(PolarsIntegerType, UInt16Type, UInt16, u16); +impl_polars_num_datatype!(PolarsIntegerType, UInt32Type, UInt32, u32); +impl_polars_num_datatype!(PolarsIntegerType, UInt64Type, UInt64, u64); +impl_polars_num_datatype!(PolarsIntegerType, Int8Type, Int8, i8); +impl_polars_num_datatype!(PolarsIntegerType, Int16Type, Int16, i16); +impl_polars_num_datatype!(PolarsIntegerType, Int32Type, Int32, i32); +impl_polars_num_datatype!(PolarsIntegerType, Int64Type, Int64, i64); +impl_polars_num_datatype!(PolarsFloatType, Float32Type, Float32, f32); +impl_polars_num_datatype!(PolarsFloatType, Float64Type, Float64, f64); +impl_polars_datatype!(DateType, Date, PrimitiveArray, 'a, i32); +#[cfg(feature = "dtype-decimal")] +impl_polars_datatype!(DecimalType, Unknown, PrimitiveArray, 'a, i128); +impl_polars_datatype!(DatetimeType, Unknown, PrimitiveArray, 'a, i64); +impl_polars_datatype!(DurationType, Unknown, PrimitiveArray, 'a, i64); +impl_polars_datatype!(CategoricalType, Unknown, PrimitiveArray, 'a, u32); +impl_polars_datatype!(TimeType, Time, PrimitiveArray, 'a, i64); +impl_polars_datatype!(Utf8Type, Utf8, Utf8Array, 'a, &'a str); +impl_polars_datatype!(BinaryType, Binary, BinaryArray, 'a, &'a [u8]); +impl_polars_datatype!(BooleanType, Boolean, BooleanArray, 'a, bool); -impl PolarsDataType for BooleanType { - fn get_dtype() -> DataType { - DataType::Boolean - } -} +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct ListType {} +unsafe impl PolarsDataType for ListType { + type Physical<'a> = Box; + type Array = ListArray; -impl PolarsDataType for ListType { fn get_dtype() -> DataType { - // null as we cannot know anything without self. + // Null as we cannot know anything without self. DataType::List(Box::new(DataType::Null)) } } #[cfg(feature = "dtype-array")] -impl PolarsDataType for FixedSizeListType { +pub struct FixedSizeListType {} +#[cfg(feature = "dtype-array")] +unsafe impl PolarsDataType for FixedSizeListType { + type Physical<'a> = Box; + type Array = FixedSizeListArray; + fn get_dtype() -> DataType { - // null as we cannot know anything without self. + // Null as we cannot know anything without self. DataType::Array(Box::new(DataType::Null), 0) } } - #[cfg(feature = "dtype-decimal")] pub struct Int128Type {} - #[cfg(feature = "dtype-decimal")] -impl PolarsDataType for Int128Type { +unsafe impl PolarsDataType for Int128Type { + type Physical<'a> = i128; + type Array = PrimitiveArray; + fn get_dtype() -> DataType { - DataType::Decimal(None, Some(0)) // scale is not None to allow for get_any_value() to work + // Scale is not None to allow for get_any_value() to work. + DataType::Decimal(None, Some(0)) } } - +#[cfg(feature = "dtype-decimal")] +impl PolarsNumericType for Int128Type { + type Native = i128; +} +#[cfg(feature = "dtype-decimal")] +impl PolarsIntegerType for Int128Type {} #[cfg(feature = "object")] pub struct ObjectType(T); #[cfg(feature = "object")] -pub type ObjectChunked = ChunkedArray>; +unsafe impl PolarsDataType for ObjectType { + type Physical<'a> = &'a (); + type Array = ObjectArray; -#[cfg(feature = "object")] -impl PolarsDataType for ObjectType { fn get_dtype() -> DataType { DataType::Object(T::type_name()) } } -/// Any type that is not nested -pub trait PolarsSingleType: PolarsDataType {} - -impl PolarsSingleType for T where T: NativeType + PolarsDataType {} - -impl PolarsSingleType for Utf8Type {} - -impl PolarsSingleType for BinaryType {} - #[cfg(feature = "dtype-array")] pub type ArrayChunked = ChunkedArray; pub type ListChunked = ChunkedArray; @@ -183,6 +208,8 @@ pub type Float32Chunked = ChunkedArray; pub type Float64Chunked = ChunkedArray; pub type Utf8Chunked = ChunkedArray; pub type BinaryChunked = ChunkedArray; +#[cfg(feature = "object")] +pub type ObjectChunked = ChunkedArray>; pub trait NumericNative: PartialOrd @@ -206,147 +233,43 @@ pub trait NumericNative: + IsFloat + ArrayArithmetics { - type POLARSTYPE: PolarsNumericType; + type PolarsType: PolarsNumericType; } impl NumericNative for i8 { - type POLARSTYPE = Int8Type; + type PolarsType = Int8Type; } impl NumericNative for i16 { - type POLARSTYPE = Int16Type; + type PolarsType = Int16Type; } impl NumericNative for i32 { - type POLARSTYPE = Int32Type; + type PolarsType = Int32Type; } impl NumericNative for i64 { - type POLARSTYPE = Int64Type; + type PolarsType = Int64Type; } impl NumericNative for u8 { - type POLARSTYPE = UInt8Type; + type PolarsType = UInt8Type; } impl NumericNative for u16 { - type POLARSTYPE = UInt16Type; + type PolarsType = UInt16Type; } impl NumericNative for u32 { - type POLARSTYPE = UInt32Type; + type PolarsType = UInt32Type; } impl NumericNative for u64 { - type POLARSTYPE = UInt64Type; + type PolarsType = UInt64Type; } #[cfg(feature = "dtype-decimal")] impl NumericNative for i128 { - type POLARSTYPE = Int128Type; + type PolarsType = Int128Type; } impl NumericNative for f32 { - type POLARSTYPE = Float32Type; + type PolarsType = Float32Type; } impl NumericNative for f64 { - type POLARSTYPE = Float64Type; -} - -pub trait PolarsNumericType: Send + Sync + PolarsDataType + 'static { - type Native: NumericNative; -} -impl PolarsNumericType for UInt8Type { - type Native = u8; -} -impl PolarsNumericType for UInt16Type { - type Native = u16; + type PolarsType = Float64Type; } -impl PolarsNumericType for UInt32Type { - type Native = u32; -} -impl PolarsNumericType for UInt64Type { - type Native = u64; -} -impl PolarsNumericType for Int8Type { - type Native = i8; -} -impl PolarsNumericType for Int16Type { - type Native = i16; -} -impl PolarsNumericType for Int32Type { - type Native = i32; -} -impl PolarsNumericType for Int64Type { - type Native = i64; -} -#[cfg(feature = "dtype-decimal")] -impl PolarsNumericType for Int128Type { - type Native = i128; -} -impl PolarsNumericType for Float32Type { - type Native = f32; -} -impl PolarsNumericType for Float64Type { - type Native = f64; -} - -pub trait PolarsIntegerType: PolarsNumericType {} -impl PolarsIntegerType for UInt8Type {} -impl PolarsIntegerType for UInt16Type {} -impl PolarsIntegerType for UInt32Type {} -impl PolarsIntegerType for UInt64Type {} -impl PolarsIntegerType for Int8Type {} -impl PolarsIntegerType for Int16Type {} -impl PolarsIntegerType for Int32Type {} -impl PolarsIntegerType for Int64Type {} - -pub trait PolarsFloatType: PolarsNumericType {} -impl PolarsFloatType for Float32Type {} -impl PolarsFloatType for Float64Type {} // Provide options to cloud providers (credentials, region). pub type CloudOptions = PlHashMap; - -/// Used to safely match the underlying type of Polars data structures. -/// -/// # Safety -/// -/// The underlying physical type of the data structure on which this -/// is implemented must always match the given PolarsDataType. -pub unsafe trait StaticallyMatchesPolarsType {} - -unsafe impl StaticallyMatchesPolarsType for PrimitiveArray {} -unsafe impl StaticallyMatchesPolarsType for PrimitiveArray {} -unsafe impl StaticallyMatchesPolarsType for Utf8Array {} -unsafe impl StaticallyMatchesPolarsType for BinaryArray {} -unsafe impl StaticallyMatchesPolarsType for BooleanArray {} -unsafe impl StaticallyMatchesPolarsType for ListArray {} -#[cfg(feature = "dtype-array")] -unsafe impl StaticallyMatchesPolarsType for FixedSizeListArray {} - -#[doc(hidden)] -pub unsafe trait HasUnderlyingArray { - type ArrayT: StaticArray; -} - -unsafe impl HasUnderlyingArray for ChunkedArray { - type ArrayT = PrimitiveArray; -} - -unsafe impl HasUnderlyingArray for BooleanChunked { - type ArrayT = BooleanArray; -} - -unsafe impl HasUnderlyingArray for Utf8Chunked { - type ArrayT = Utf8Array; -} - -unsafe impl HasUnderlyingArray for BinaryChunked { - type ArrayT = BinaryArray; -} - -unsafe impl HasUnderlyingArray for ListChunked { - type ArrayT = ListArray; -} - -#[cfg(feature = "dtype-array")] -unsafe impl HasUnderlyingArray for ArrayChunked { - type ArrayT = FixedSizeListArray; -} - -#[cfg(feature = "object")] -unsafe impl HasUnderlyingArray for ObjectChunked { - type ArrayT = crate::chunked_array::object::ObjectArray; -} diff --git a/crates/polars-ops/src/series/ops/is_in.rs b/crates/polars-ops/src/series/ops/is_in.rs index ee52fd90b0a9..1a80742b6d33 100644 --- a/crates/polars-ops/src/series/ops/is_in.rs +++ b/crates/polars-ops/src/series/ops/is_in.rs @@ -7,8 +7,7 @@ use polars_core::with_match_physical_integer_polars_type; fn is_in_helper<'a, T>(ca: &'a ChunkedArray, other: &Series) -> PolarsResult where T: PolarsDataType, - ChunkedArray: HasUnderlyingArray, - < as HasUnderlyingArray>::ArrayT as StaticArray>::ValueT<'a>: Hash + Eq + Copy, + T::Physical<'a>: Hash + Eq + Copy, { let mut set = PlHashSet::with_capacity(other.len()); diff --git a/crates/polars-pipe/src/executors/sinks/group_by/aggregates/mean.rs b/crates/polars-pipe/src/executors/sinks/group_by/aggregates/mean.rs index 7898ee36d9cb..b5475767d875 100644 --- a/crates/polars-pipe/src/executors/sinks/group_by/aggregates/mean.rs +++ b/crates/polars-pipe/src/executors/sinks/group_by/aggregates/mean.rs @@ -42,7 +42,7 @@ impl MeanAgg { impl AggregateFn for MeanAgg where - K::POLARSTYPE: PolarsNumericType, + K::PolarsType: PolarsNumericType, K: NumericNative + Add, ::Simd: Add::Simd> + Sum, { @@ -107,7 +107,7 @@ where let arr = values.chunks().get_unchecked(0); arr.sliced_unchecked(offset as usize, length as usize) }; - let dtype = K::POLARSTYPE::get_dtype().to_arrow(); + let dtype = K::PolarsType::get_dtype().to_arrow(); let arr = polars_arrow::compute::cast::cast(arr.as_ref(), &dtype).unwrap(); let arr = unsafe { arr.as_any() diff --git a/crates/polars-pipe/src/executors/sinks/group_by/aggregates/min_max.rs b/crates/polars-pipe/src/executors/sinks/group_by/aggregates/min_max.rs index 486ea1631424..b44faa172ef1 100644 --- a/crates/polars-pipe/src/executors/sinks/group_by/aggregates/min_max.rs +++ b/crates/polars-pipe/src/executors/sinks/group_by/aggregates/min_max.rs @@ -89,7 +89,7 @@ where length: IdxSize, values: &Series, ) { - let ca: &ChunkedArray = values.as_ref().as_ref(); + let ca: &ChunkedArray = values.as_ref().as_ref(); let arr = ca.downcast_iter().next().unwrap(); let arr = unsafe { arr.slice_typed_unchecked(offset as usize, length as usize) }; // convince the compiler that K::POLARSTYPE::Native == K diff --git a/crates/polars-pipe/src/executors/sinks/group_by/aggregates/sum.rs b/crates/polars-pipe/src/executors/sinks/group_by/aggregates/sum.rs index 2133910065da..bb79272f21b2 100644 --- a/crates/polars-pipe/src/executors/sinks/group_by/aggregates/sum.rs +++ b/crates/polars-pipe/src/executors/sinks/group_by/aggregates/sum.rs @@ -34,7 +34,7 @@ impl + NumCast> SumAgg { impl AggregateFn for SumAgg where - K::POLARSTYPE: PolarsNumericType, + K::PolarsType: PolarsNumericType, K: NumericNative + Add, ::Simd: Add::Simd> + Sum, { @@ -89,7 +89,7 @@ where let arr = values.chunks().get_unchecked(0); arr.sliced_unchecked(offset as usize, length as usize) }; - let dtype = K::POLARSTYPE::get_dtype().to_arrow(); + let dtype = K::PolarsType::get_dtype().to_arrow(); let arr = polars_arrow::compute::cast::cast(arr.as_ref(), &dtype).unwrap(); let arr = unsafe { arr.as_any()