Skip to content
This repository has been archived by the owner on Feb 18, 2024. It is now read-only.

Macro natives and linting #6

Merged
merged 2 commits into from
Mar 11, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,046 changes: 1,046 additions & 0 deletions Cargo.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion src/array/binary/from.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ pub struct BinaryPrimitive<O: Offset> {

impl<O: Offset, P: AsRef<[u8]>> FromIterator<Option<P>> for BinaryPrimitive<O> {
fn from_iter<I: IntoIterator<Item = Option<P>>>(iter: I) -> Self {
Self::try_from_iter(iter.into_iter().map(|x| Ok(x))).unwrap()
Self::try_from_iter(iter.into_iter().map(Ok)).unwrap()
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/array/dictionary/from.rs
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ mod tests {
fn primitive() -> Result<()> {
let data = vec![Some("a"), Some("b"), Some("a")];

let data = data.into_iter().map(|x| Result::Ok(x));
let data = data.into_iter().map(Result::Ok);
let a = DictionaryPrimitive::<i32, Utf8Primitive<i32>, &str>::try_from_iter(data)?;
a.to(DataType::Dictionary(
Box::new(DataType::Int32),
Expand Down
2 changes: 1 addition & 1 deletion src/array/equal/fixed_size_list.rs
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ mod tests {
let data_type = FixedSizeListArray::default_datatype(DataType::Int32, 3);
let list = data
.as_ref()
.into_iter()
.iter()
.map(|x| {
Some(match x {
Some(x) => x.as_ref().iter().map(|x| Some(*x)).collect::<Vec<_>>(),
Expand Down
2 changes: 1 addition & 1 deletion src/array/equal/list.rs
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ mod tests {
let data_type = ListArray::<i32>::default_datatype(DataType::Int32);
let list = data
.as_ref()
.into_iter()
.iter()
.map(|x| {
x.as_ref()
.map(|x| x.as_ref().iter().map(|x| Some(*x)).collect::<Vec<_>>())
Expand Down
2 changes: 1 addition & 1 deletion src/array/fixed_size_binary/from.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ pub struct FixedSizeBinaryPrimitive {

impl<P: AsRef<[u8]>> FromIterator<Option<P>> for FixedSizeBinaryPrimitive {
fn from_iter<I: IntoIterator<Item = Option<P>>>(iter: I) -> Self {
Self::try_from_iter(iter.into_iter().map(|x| Ok(x))).unwrap()
Self::try_from_iter(iter.into_iter().map(Ok)).unwrap()
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/array/fixed_size_list/from.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ where
P: AsRef<[Option<T>]> + IntoIterator<Item = Option<T>>,
{
fn from_iter<I: IntoIterator<Item = Option<P>>>(iter: I) -> Self {
Self::try_from_iter(iter.into_iter().map(|x| Ok(x))).unwrap()
Self::try_from_iter(iter.into_iter().map(Ok)).unwrap()
}
}

Expand Down
4 changes: 2 additions & 2 deletions src/array/growable/list.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ use super::{
Growable,
};

fn extend_offset_values<'a, O: Offset>(
growable: &mut GrowableList<'a, O>,
fn extend_offset_values<O: Offset>(
growable: &mut GrowableList<'_, O>,
index: usize,
start: usize,
len: usize,
Expand Down
8 changes: 7 additions & 1 deletion src/array/growable/null.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,15 @@ pub struct GrowableNull {
length: usize,
}

impl Default for GrowableNull {
fn default() -> Self {
Self { length: 0 }
}
}

impl GrowableNull {
pub fn new() -> Self {
Self { length: 0 }
Self::default()
}
}

Expand Down
9 changes: 4 additions & 5 deletions src/array/growable/structure.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ impl<'a> GrowableStruct<'a> {
.collect();

let arrays = arrays
.into_iter()
.iter()
.map(|array| array.as_any().downcast_ref::<StructArray>().unwrap())
.collect::<Vec<_>>();

Expand Down Expand Up @@ -203,7 +203,7 @@ mod tests {
let result: StructArray = a.into();

let expected = StructArray::from_data(
fields.clone(),
fields,
vec![values[0].slice(2, 2).into(), values[1].slice(2, 2).into()],
None,
);
Expand All @@ -227,7 +227,7 @@ mod tests {
let result: StructArray = a.into();

let expected = StructArray::from_data(
fields.clone(),
fields,
vec![values[0].slice(1, 2).into(), values[1].slice(1, 2).into()],
Some(Bitmap::from((&[0b00000010], 5)).slice(1, 2)),
);
Expand Down Expand Up @@ -257,8 +257,7 @@ mod tests {
Primitive::<i32>::from(vec![Some(2), Some(3), Some(1), Some(2)]).to(DataType::Int32),
);

let expected =
StructArray::from_data(fields.clone(), vec![expected_string, expected_int], None);
let expected = StructArray::from_data(fields, vec![expected_string, expected_int], None);
assert_eq!(result, expected)
}
}
2 changes: 1 addition & 1 deletion src/array/list/from.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ where
P: AsRef<[Option<T>]> + IntoIterator<Item = Option<T>>,
{
fn from_iter<I: IntoIterator<Item = Option<P>>>(iter: I) -> Self {
Self::try_from_iter(iter.into_iter().map(|x| Ok(x))).unwrap()
Self::try_from_iter(iter.into_iter().map(Ok)).unwrap()
}
}

Expand Down
4 changes: 4 additions & 0 deletions src/array/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ pub trait Array: std::fmt::Debug + std::fmt::Display + Send + Sync + ToFFI {
/// The length of the array
fn len(&self) -> usize;

fn is_empty(&self) -> bool {
self.len() == 0
}

/// The [`DataType`] of the array
fn data_type(&self) -> &DataType;

Expand Down
10 changes: 2 additions & 8 deletions src/array/primitive/from.rs
Original file line number Diff line number Diff line change
Expand Up @@ -228,10 +228,7 @@ impl<T: NativeType, Ptr: std::borrow::Borrow<Option<T>>> FromIterator<Ptr> for P
})
.collect();

Self {
values: values,
validity,
}
Self { values, validity }
}
}

Expand All @@ -254,10 +251,7 @@ impl<T: NativeType, Ptr: std::borrow::Borrow<Option<T>>> TryFromIterator<Ptr> fo
})
.collect::<ArrowResult<_>>()?;

Ok(Self {
values: values,
validity,
})
Ok(Self { values, validity })
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/array/struct_.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ impl StructArray {
values: Vec<Arc<dyn Array>>,
validity: Option<Bitmap>,
) -> Self {
assert!(fields.len() > 0);
assert!(!fields.is_empty());
assert_eq!(fields.len(), values.len());
assert!(values.iter().all(|x| x.len() == values[0].len()));
if let Some(ref validity) = validity {
Expand Down
2 changes: 1 addition & 1 deletion src/array/utf8/from.rs
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ pub struct Utf8Primitive<O: Offset> {

impl<O: Offset, P: AsRef<str>> FromIterator<Option<P>> for Utf8Primitive<O> {
fn from_iter<I: IntoIterator<Item = Option<P>>>(iter: I) -> Self {
Self::try_from_iter(iter.into_iter().map(|x| Ok(x))).unwrap()
Self::try_from_iter(iter.into_iter().map(Ok)).unwrap()
}
}

Expand Down
35 changes: 16 additions & 19 deletions src/bits/chunk_iterator/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -176,28 +176,25 @@ impl<T: BitChunk> Iterator for BitChunkIterator<'_, T> {
// so when reading as u64 on a big-endian machine, the bytes need to be swapped
let combined = if self.offset == 0 {
unsafe { std::ptr::read_unaligned(chunk_data.add(self.chunk_index)) }
} else if self.chunk_index + 1 < self.chunk_len {
// on the middle we can use the whole next chunk
let current =
unsafe { std::ptr::read_unaligned(chunk_data.add(self.chunk_index)) }.to_le();
let chunk_data = self.buffer.as_ptr() as *const T;
let next = unsafe { std::ptr::read_unaligned(chunk_data.add(self.chunk_index + 1)) };

merge_reversed(current, next, self.offset)
} else {
if self.chunk_index + 1 < self.chunk_len {
// on the middle we can use the whole next chunk
let current =
unsafe { std::ptr::read_unaligned(chunk_data.add(self.chunk_index)) }.to_le();
let chunk_data = self.buffer.as_ptr() as *const T;
let next =
unsafe { std::ptr::read_unaligned(chunk_data.add(self.chunk_index + 1)) };

merge_reversed(current, next, self.offset)
} else {
// on the last chunk the "next" chunk may not be complete and thus we must create it from existing bytes.
let mut remainder = T::zero();
let dst = &mut remainder as *mut T as *mut u8;
// on the last chunk the "next" chunk may not be complete and thus we must create it from existing bytes.
let mut remainder = T::zero();
let dst = &mut remainder as *mut T as *mut u8;

let size_of = std::mem::size_of::<T>();
let remainder_bytes =
&self.buffer[self.chunk_index * size_of..(self.chunk_index + 1) * size_of];
let size_of = std::mem::size_of::<T>();
let remainder_bytes =
&self.buffer[self.chunk_index * size_of..(self.chunk_index + 1) * size_of];

unsafe { copy_with_merge(dst, remainder_bytes, self.offset) };
remainder
}
unsafe { copy_with_merge(dst, remainder_bytes, self.offset) };
remainder
};

self.chunk_index += 1;
Expand Down
8 changes: 7 additions & 1 deletion src/buffer/bitmap.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,17 @@ pub struct Bitmap {
null_count: usize,
}

impl Default for Bitmap {
fn default() -> Self {
MutableBitmap::new().into()
}
}

impl Bitmap {
/// Initializes an empty [`Bitmap`].
#[inline]
pub fn new() -> Self {
MutableBitmap::new().into()
Self::default()
}

#[inline]
Expand Down
5 changes: 2 additions & 3 deletions src/buffer/bitmap_ops.rs
Original file line number Diff line number Diff line change
Expand Up @@ -99,12 +99,11 @@ fn eq(lhs: &Bitmap, rhs: &Bitmap) -> bool {
// least-significant bit (LSB) numbering (also known as bit-endianness)
let lhs_remainder = &lhs_chunks.remainder().to_le_bytes()[0..remainder_bytes];
let rhs_remainder = &rhs_chunks.remainder().to_le_bytes()[0..remainder_bytes];
let remainder_equal = unsafe {
unsafe {
debug_assert!(lhs_remainder.len() * 8 >= lhs_chunks.remainder_len());
(0..lhs_chunks.remainder_len())
.all(|i| get_bit_unchecked(lhs_remainder, i) == get_bit_unchecked(rhs_remainder, i))
};
remainder_equal
}
}

impl PartialEq for Bitmap {
Expand Down
8 changes: 7 additions & 1 deletion src/buffer/immutable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,16 @@ pub struct Buffer<T: NativeType> {
length: usize,
}

impl<T: NativeType> Default for Buffer<T> {
fn default() -> Self {
MutableBuffer::new().into()
}
}

impl<T: NativeType> Buffer<T> {
#[inline]
pub fn new() -> Self {
MutableBuffer::new().into()
Self::default()
}

#[inline]
Expand Down
12 changes: 5 additions & 7 deletions src/buffer/mutable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,7 @@ impl<T: NativeType> MutableBuffer<T> {
let mut dst = unsafe { self.ptr.as_ptr().add(len.local_len) as *mut T };
let capacity = self.capacity;

while len.local_len + 1 <= capacity {
while len.local_len < capacity {
if let Some(item) = iterator.next() {
unsafe {
std::ptr::write(dst, item);
Expand Down Expand Up @@ -405,7 +405,7 @@ impl<T: NativeType> FromIterator<T> for MutableBuffer<T> {
};

buffer.extend_from_iter(iterator);
buffer.into()
buffer
}
}

Expand Down Expand Up @@ -462,7 +462,7 @@ impl Drop for SetLenOnDrop<'_> {
impl<T: NativeType, P: AsRef<[T]>> From<P> for MutableBuffer<T> {
#[inline]
fn from(slice: P) -> Self {
unsafe { MutableBuffer::from_trusted_len_iter(slice.as_ref().iter().map(|x| *x)) }
unsafe { MutableBuffer::from_trusted_len_iter(slice.as_ref().iter().copied()) }
}
}

Expand Down Expand Up @@ -508,9 +508,7 @@ impl MutableBuffer<u8> {
/// * The iterator must be TrustedLen
#[inline]
pub unsafe fn from_chunk_iter<I: Iterator<Item = u64>>(iter: I) -> Self {
let iterator = iter.into_iter();

let buffer = MutableBuffer::from_trusted_len_iter(iterator);
let buffer = MutableBuffer::from_trusted_len_iter(iter);
buffer.into()
}
}
Expand Down Expand Up @@ -541,7 +539,7 @@ mod tests {
let mut b = MutableBuffer::<f32>::with_capacity(16);
b.reserve(4);
assert_eq!(b.capacity(), 16);
b.extend_from_slice(&vec![0.1; 16]);
b.extend_from_slice(&[0.1; 16]);
b.reserve(4);
assert_eq!(b.capacity(), 32);
}
Expand Down
4 changes: 2 additions & 2 deletions src/compute/aggregate/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ where
}

match array.validity() {
None => Some(array.values().iter().map(|&x| x).sum()),
None => Some(array.values().iter().copied().sum()),
Some(buffer) => {
let values = array.values();
let validity_chunks = buffer.chunks::<u64>();
Expand Down Expand Up @@ -242,7 +242,7 @@ mod tests {
#[test]
fn test_primitive_array_float_sum() {
let a = Primitive::from_slice(&[1.1f64, 2.2, 3.3, 4.4, 5.5]).to(DataType::Float64);
assert!(16.5 - sum(&a).unwrap() < f64::EPSILON);
assert!((16.5 - sum(&a).unwrap()).abs() < f64::EPSILON);
}

#[test]
Expand Down
10 changes: 5 additions & 5 deletions src/compute/cast/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -728,11 +728,11 @@ mod tests {
let array = Primitive::<i32>::from_values(&[5, 6, 7, 8, 9]).to(DataType::Int32);
let b = cast(&array, &DataType::Float64).unwrap();
let c = b.as_any().downcast_ref::<Float64Array>().unwrap();
assert!(5.0 - c.value(0) < f64::EPSILON);
assert!(6.0 - c.value(1) < f64::EPSILON);
assert!(7.0 - c.value(2) < f64::EPSILON);
assert!(8.0 - c.value(3) < f64::EPSILON);
assert!(9.0 - c.value(4) < f64::EPSILON);
assert!((5.0 - c.value(0)).abs() < f64::EPSILON);
assert!((6.0 - c.value(1)).abs() < f64::EPSILON);
assert!((7.0 - c.value(2)).abs() < f64::EPSILON);
assert!((8.0 - c.value(3)).abs() < f64::EPSILON);
assert!((9.0 - c.value(4)).abs() < f64::EPSILON);
}

#[test]
Expand Down
2 changes: 1 addition & 1 deletion src/compute/cast/primitive_to.rs
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ pub fn primitive_to_dictionary<T: NativeType + Eq + Hash, K: DictionaryKey>(
let values = cast(array, to)?;
let values = values.as_any().downcast_ref::<PrimitiveArray<T>>().unwrap();

let iter = values.iter().map(|x| Result::Ok(x));
let iter = values.iter().map(Result::Ok);
let primitive = DictionaryPrimitive::<K, Primitive<T>, _>::try_from_iter(iter)?;

let array = primitive.to(DataType::Dictionary(
Expand Down
2 changes: 1 addition & 1 deletion src/compute/cast/string_to.rs
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ pub fn string_to_dictionary<O: Offset, K: DictionaryKey>(
let values = cast(array, &to)?;
let values = values.as_any().downcast_ref::<Utf8Array<O>>().unwrap();

let iter = values.iter().map(|x| Result::Ok(x));
let iter = values.iter().map(Result::Ok);
let primitive = DictionaryPrimitive::<K, Utf8Primitive<i32>, _>::try_from_iter(iter)?;

let array = primitive.to(DataType::Dictionary(Box::new(K::DATA_TYPE), Box::new(to)));
Expand Down
Loading