From fe55b2b6d4302e4484cbe455d632fcbf251a879c Mon Sep 17 00:00:00 2001 From: 1aguna Date: Sun, 3 Oct 2021 10:34:54 -0700 Subject: [PATCH 01/14] Add mod.rs for bitwise operations --- src/compute/bitwise/mod.rs | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/compute/bitwise/mod.rs diff --git a/src/compute/bitwise/mod.rs b/src/compute/bitwise/mod.rs new file mode 100644 index 00000000000..e69de29bb2d From 5664b657e5fb72d1227dac7e4dabcd93e29633cc Mon Sep 17 00:00:00 2001 From: 1aguna Date: Thu, 7 Oct 2021 20:29:18 -0700 Subject: [PATCH 02/14] Base commit: tests for bitwise operators --- tests/it/compute/arithmetics/bitwise/and.rs | 0 tests/it/compute/arithmetics/bitwise/mod.rs | 2 ++ tests/it/compute/arithmetics/bitwise/or.rs | 0 tests/it/compute/arithmetics/bitwise/xor.rs | 0 tests/it/compute/arithmetics/mod.rs | 1 + 5 files changed, 3 insertions(+) create mode 100644 tests/it/compute/arithmetics/bitwise/and.rs create mode 100644 tests/it/compute/arithmetics/bitwise/mod.rs create mode 100644 tests/it/compute/arithmetics/bitwise/or.rs create mode 100644 tests/it/compute/arithmetics/bitwise/xor.rs diff --git a/tests/it/compute/arithmetics/bitwise/and.rs b/tests/it/compute/arithmetics/bitwise/and.rs new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/it/compute/arithmetics/bitwise/mod.rs b/tests/it/compute/arithmetics/bitwise/mod.rs new file mode 100644 index 00000000000..abe8937f798 --- /dev/null +++ b/tests/it/compute/arithmetics/bitwise/mod.rs @@ -0,0 +1,2 @@ +mod xor; +mod and; \ No newline at end of file diff --git a/tests/it/compute/arithmetics/bitwise/or.rs b/tests/it/compute/arithmetics/bitwise/or.rs new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/it/compute/arithmetics/bitwise/xor.rs b/tests/it/compute/arithmetics/bitwise/xor.rs new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/it/compute/arithmetics/mod.rs b/tests/it/compute/arithmetics/mod.rs index 00b90bc0dd1..31ef290234e 100644 --- a/tests/it/compute/arithmetics/mod.rs +++ b/tests/it/compute/arithmetics/mod.rs @@ -1,6 +1,7 @@ mod basic; mod decimal; mod time; +mod bitwise; use arrow2::array::new_empty_array; use arrow2::compute::arithmetics::{arithmetic, can_arithmetic, Operator}; From 4bf646b3844d4e2405788a6a30ad6cc6ca584e59 Mon Sep 17 00:00:00 2001 From: 1aguna Date: Thu, 7 Oct 2021 21:59:16 -0700 Subject: [PATCH 03/14] Add binary bitwise operations --- src/compute/binary/bitwise.rs | 39 +++++++++++++++++++++++++++++++++++ src/compute/binary/mod.rs | 1 + src/compute/bitwise/mod.rs | 0 src/compute/boolean_kleene.rs | 2 +- src/compute/mod.rs | 2 ++ src/trusted_len.rs | 9 ++++++++ 6 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 src/compute/binary/bitwise.rs create mode 100644 src/compute/binary/mod.rs delete mode 100644 src/compute/bitwise/mod.rs diff --git a/src/compute/binary/bitwise.rs b/src/compute/binary/bitwise.rs new file mode 100644 index 00000000000..e28d067703d --- /dev/null +++ b/src/compute/binary/bitwise.rs @@ -0,0 +1,39 @@ +use crate::array::{Array, BooleanArray, PrimitiveArray}; +use crate::datatypes::DataType; +use crate::error::{ArrowError, Result}; +use crate::types::NativeType; +use crate::compute::arithmetics::basic::check_same_type; +use crate::compute::arity::binary; + +/// Performs `OR` operation on two arrays. +/// # Error +/// This function errors when the arrays have different lengths or are different types. +pub fn or(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Result> + where + T: NativeType + BitwiseOr, +{ + check_same_type(lhs, rhs)?; + binary(lhs, rhs, lhs.data_type().clone(), |a, b| a | b) +} + +/// Performs `XOR` operation on two arrays. +/// # Error +/// This function errors when the arrays have different lengths or are different types. +pub fn xor(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Result> + where + T: NativeType + BitAnd, +{ + check_same_type(lhs, rhs)?; + binary(lhs, rhs, lhs.data_type().clone(), |a, b| a ^ b) +} + +/// Performs `AND` operation on two arrays. +/// # Error +/// This function errors when the arrays have different lengths or are different types. +pub fn and(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Result> + where + T: NativeType + BitAnd, +{ + check_same_type(lhs, rhs)?; + binary(lhs, rhs, lhs.data_type().clone(), |a, b| a & b) +} diff --git a/src/compute/binary/mod.rs b/src/compute/binary/mod.rs new file mode 100644 index 00000000000..00bc8f63e8f --- /dev/null +++ b/src/compute/binary/mod.rs @@ -0,0 +1 @@ +pub mod bitwise; diff --git a/src/compute/bitwise/mod.rs b/src/compute/bitwise/mod.rs deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/compute/boolean_kleene.rs b/src/compute/boolean_kleene.rs index 4d65f449f22..896e030e20c 100644 --- a/src/compute/boolean_kleene.rs +++ b/src/compute/boolean_kleene.rs @@ -26,7 +26,7 @@ use crate::{ pub fn or(lhs: &BooleanArray, rhs: &BooleanArray) -> Result { if lhs.len() != rhs.len() { return Err(ArrowError::InvalidArgumentError( - "Cannot perform bitwise operation on arrays of different length".to_string(), + "Cannot perfobitwise operation on arrays of different length".to_string(), )); } diff --git a/src/compute/mod.rs b/src/compute/mod.rs index 51fcca52ef5..bfadd5577b3 100644 --- a/src/compute/mod.rs +++ b/src/compute/mod.rs @@ -14,6 +14,7 @@ pub mod aggregate; pub mod arithmetics; pub mod arity; +pub mod bitwise; pub mod boolean; pub mod boolean_kleene; pub mod cast; @@ -44,3 +45,4 @@ pub mod regex_match; #[cfg(feature = "merge_sort")] #[cfg_attr(docsrs, doc(cfg(feature = "merge_sort")))] pub mod merge_sort; + diff --git a/src/trusted_len.rs b/src/trusted_len.rs index 98ff458e56a..a7939d59fc8 100644 --- a/src/trusted_len.rs +++ b/src/trusted_len.rs @@ -1,3 +1,12 @@ +pub fn and(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Result> + where + T: NativeType + BitXor, +{ + check_same_type(lhs, rhs)?; + + binary(lhs, rhs, lhs.data_type().clone(), |a, b| a & b) +} + //! Declares [`TrustedLen`]. use std::slice::Iter; From 9f93fd920d4aad696a9b9f351a50b6d0ec6a3586 Mon Sep 17 00:00:00 2001 From: 1aguna Date: Sat, 9 Oct 2021 21:06:36 -0700 Subject: [PATCH 04/14] Add bitwise operator tests --- src/compute/binary/bitwise.rs | 16 +++++-- src/compute/mod.rs | 2 +- src/trusted_len.rs | 9 ---- tests/it/compute/arithmetics/bitwise/and.rs | 0 tests/it/compute/arithmetics/bitwise/mod.rs | 2 - tests/it/compute/arithmetics/bitwise/or.rs | 0 tests/it/compute/arithmetics/bitwise/xor.rs | 0 tests/it/compute/arithmetics/mod.rs | 1 - tests/it/compute/binary/bitwise.rs | 46 +++++++++++++++++++++ tests/it/compute/binary/mod.rs | 1 + tests/it/compute/mod.rs | 1 + 11 files changed, 62 insertions(+), 16 deletions(-) delete mode 100644 tests/it/compute/arithmetics/bitwise/and.rs delete mode 100644 tests/it/compute/arithmetics/bitwise/mod.rs delete mode 100644 tests/it/compute/arithmetics/bitwise/or.rs delete mode 100644 tests/it/compute/arithmetics/bitwise/xor.rs create mode 100644 tests/it/compute/binary/bitwise.rs create mode 100644 tests/it/compute/binary/mod.rs diff --git a/src/compute/binary/bitwise.rs b/src/compute/binary/bitwise.rs index e28d067703d..89219d36003 100644 --- a/src/compute/binary/bitwise.rs +++ b/src/compute/binary/bitwise.rs @@ -3,14 +3,15 @@ use crate::datatypes::DataType; use crate::error::{ArrowError, Result}; use crate::types::NativeType; use crate::compute::arithmetics::basic::check_same_type; -use crate::compute::arity::binary; +use crate::compute::arity::{binary, unary}; +use std::ops::{BitOr, BitAnd, BitXor, Not}; /// Performs `OR` operation on two arrays. /// # Error /// This function errors when the arrays have different lengths or are different types. pub fn or(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Result> where - T: NativeType + BitwiseOr, + T: NativeType + BitOr, { check_same_type(lhs, rhs)?; binary(lhs, rhs, lhs.data_type().clone(), |a, b| a | b) @@ -21,7 +22,7 @@ pub fn or(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Result(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Result> where - T: NativeType + BitAnd, + T: NativeType + BitXor, { check_same_type(lhs, rhs)?; binary(lhs, rhs, lhs.data_type().clone(), |a, b| a ^ b) @@ -37,3 +38,12 @@ pub fn and(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Result(arr: &PrimitiveArray) -> PrimitiveArray + where + T: NativeType + Not, +{ + let op = move |a: T| !a; + unary(arr, op, arr.data_type().clone()) +} diff --git a/src/compute/mod.rs b/src/compute/mod.rs index bfadd5577b3..1f0f3d8d061 100644 --- a/src/compute/mod.rs +++ b/src/compute/mod.rs @@ -14,7 +14,7 @@ pub mod aggregate; pub mod arithmetics; pub mod arity; -pub mod bitwise; +pub mod binary; pub mod boolean; pub mod boolean_kleene; pub mod cast; diff --git a/src/trusted_len.rs b/src/trusted_len.rs index a7939d59fc8..98ff458e56a 100644 --- a/src/trusted_len.rs +++ b/src/trusted_len.rs @@ -1,12 +1,3 @@ -pub fn and(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Result> - where - T: NativeType + BitXor, -{ - check_same_type(lhs, rhs)?; - - binary(lhs, rhs, lhs.data_type().clone(), |a, b| a & b) -} - //! Declares [`TrustedLen`]. use std::slice::Iter; diff --git a/tests/it/compute/arithmetics/bitwise/and.rs b/tests/it/compute/arithmetics/bitwise/and.rs deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/it/compute/arithmetics/bitwise/mod.rs b/tests/it/compute/arithmetics/bitwise/mod.rs deleted file mode 100644 index abe8937f798..00000000000 --- a/tests/it/compute/arithmetics/bitwise/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod xor; -mod and; \ No newline at end of file diff --git a/tests/it/compute/arithmetics/bitwise/or.rs b/tests/it/compute/arithmetics/bitwise/or.rs deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/it/compute/arithmetics/bitwise/xor.rs b/tests/it/compute/arithmetics/bitwise/xor.rs deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/it/compute/arithmetics/mod.rs b/tests/it/compute/arithmetics/mod.rs index 31ef290234e..00b90bc0dd1 100644 --- a/tests/it/compute/arithmetics/mod.rs +++ b/tests/it/compute/arithmetics/mod.rs @@ -1,7 +1,6 @@ mod basic; mod decimal; mod time; -mod bitwise; use arrow2::array::new_empty_array; use arrow2::compute::arithmetics::{arithmetic, can_arithmetic, Operator}; diff --git a/tests/it/compute/binary/bitwise.rs b/tests/it/compute/binary/bitwise.rs new file mode 100644 index 00000000000..e39576a4244 --- /dev/null +++ b/tests/it/compute/binary/bitwise.rs @@ -0,0 +1,46 @@ +#![allow(clippy::zero_prefixed_literal, clippy::inconsistent_digit_grouping)] + +use arrow2::array::*; +use arrow2::compute::arithmetics::decimal::add::*; +use arrow2::compute::binary::bitwise::*; +use arrow2::datatypes::DataType; + +#[test] +fn test_xor() { + let a = Int32Array::from(&[Some(2), Some(4), Some(6), Some(7)]); + let b = Int32Array::from(&[Some(3), Some(6), Some(9), Some(7)]); + let result = xor(&a, &b).unwrap(); + let expected = Int32Array::from(&[Some(1), Some(2), Some(15), Some(0)]); + + assert_eq!(result, expected); +} + +#[test] +fn test_and() { + let a = Int32Array::from(&[Some(1), Some(2), Some(15)]); + let b = Int32Array::from(&[Some(2), Some(2), Some(6)]); + let result = and( &a, &b).unwrap(); + let expected = Int32Array::from(&[Some(0), Some(2), Some(6)]); + + assert_eq!(result, expected); +} + +#[test] +fn test_or() { + let a = Int32Array::from(&[Some(1), Some(2), Some(0)]); + let b = Int32Array::from(&[Some(2), Some(2), Some(0)]); + let result = or(&a, &b).unwrap(); + let expected = Int32Array::from(&[Some(3), Some(2), Some(0)]); + + assert_eq!(result, expected); +} + + +#[test] +fn test_not() { + let a = Int8Array::from(&[Some(1i8), Some(-100i8)]); + let result = not(&a); + let expected = Int8Array::from(&[Some(-2), Some(99)]); + + assert_eq!(result, expected); +} \ No newline at end of file diff --git a/tests/it/compute/binary/mod.rs b/tests/it/compute/binary/mod.rs new file mode 100644 index 00000000000..e0282f29663 --- /dev/null +++ b/tests/it/compute/binary/mod.rs @@ -0,0 +1 @@ +mod bitwise; \ No newline at end of file diff --git a/tests/it/compute/mod.rs b/tests/it/compute/mod.rs index 78eced0b882..bf74f5d1b0f 100644 --- a/tests/it/compute/mod.rs +++ b/tests/it/compute/mod.rs @@ -1,5 +1,6 @@ mod aggregate; mod arithmetics; +mod binary; mod boolean; mod boolean_kleene; mod cast; From 519dca4c8cd7cbfd237ecf55daa3586907cef680 Mon Sep 17 00:00:00 2001 From: 1aguna Date: Wed, 20 Oct 2021 23:07:51 -0700 Subject: [PATCH 05/14] Change to pub use bitwise::* in mod.rs --- src/compute/binary/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/compute/binary/mod.rs b/src/compute/binary/mod.rs index 00bc8f63e8f..8b43e47b216 100644 --- a/src/compute/binary/mod.rs +++ b/src/compute/binary/mod.rs @@ -1 +1,2 @@ -pub mod bitwise; +mod bitwise; +pub use bitwise::*; \ No newline at end of file From f3b5375613de4df80047ac96e5cc8d76d13a098e Mon Sep 17 00:00:00 2001 From: 1aguna Date: Wed, 20 Oct 2021 23:08:16 -0700 Subject: [PATCH 06/14] Modify bitwise tests to include None --- tests/it/compute/binary/bitwise.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/it/compute/binary/bitwise.rs b/tests/it/compute/binary/bitwise.rs index e39576a4244..19ee9252cb3 100644 --- a/tests/it/compute/binary/bitwise.rs +++ b/tests/it/compute/binary/bitwise.rs @@ -2,15 +2,15 @@ use arrow2::array::*; use arrow2::compute::arithmetics::decimal::add::*; -use arrow2::compute::binary::bitwise::*; +use arrow2::compute::binary::*; use arrow2::datatypes::DataType; #[test] fn test_xor() { let a = Int32Array::from(&[Some(2), Some(4), Some(6), Some(7)]); - let b = Int32Array::from(&[Some(3), Some(6), Some(9), Some(7)]); + let b = Int32Array::from(&[None, Some(6), Some(9), Some(7)]); let result = xor(&a, &b).unwrap(); - let expected = Int32Array::from(&[Some(1), Some(2), Some(15), Some(0)]); + let expected = Int32Array::from(&[None, Some(2), Some(15), Some(0)]); assert_eq!(result, expected); } @@ -18,9 +18,9 @@ fn test_xor() { #[test] fn test_and() { let a = Int32Array::from(&[Some(1), Some(2), Some(15)]); - let b = Int32Array::from(&[Some(2), Some(2), Some(6)]); + let b = Int32Array::from(&[None, Some(2), Some(6)]); let result = and( &a, &b).unwrap(); - let expected = Int32Array::from(&[Some(0), Some(2), Some(6)]); + let expected = Int32Array::from(&[None, Some(2), Some(6)]); assert_eq!(result, expected); } @@ -28,9 +28,9 @@ fn test_and() { #[test] fn test_or() { let a = Int32Array::from(&[Some(1), Some(2), Some(0)]); - let b = Int32Array::from(&[Some(2), Some(2), Some(0)]); + let b = Int32Array::from(&[None, Some(2), Some(0)]); let result = or(&a, &b).unwrap(); - let expected = Int32Array::from(&[Some(3), Some(2), Some(0)]); + let expected = Int32Array::from(&[None, Some(2), Some(0)]); assert_eq!(result, expected); } @@ -38,9 +38,9 @@ fn test_or() { #[test] fn test_not() { - let a = Int8Array::from(&[Some(1i8), Some(-100i8)]); + let a = Int8Array::from(&[None, Some(1i8), Some(-100i8)]); let result = not(&a); - let expected = Int8Array::from(&[Some(-2), Some(99)]); + let expected = Int8Array::from(&[None, Some(-2), Some(99)]); assert_eq!(result, expected); } \ No newline at end of file From 7cf8824c9a1369042b78084f71d70d1b308df230 Mon Sep 17 00:00:00 2001 From: 1aguna Date: Wed, 20 Oct 2021 23:08:37 -0700 Subject: [PATCH 07/14] Add benchmarks for bitwise operators --- Cargo.toml | 4 +++ benches/bitwise.rs | 70 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 benches/bitwise.rs diff --git a/Cargo.toml b/Cargo.toml index 96c9d35a8c5..35a1aa7cc15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -237,3 +237,7 @@ harness = false [[bench]] name = "iter_list" harness = false + +[[bench]] +name = "bitwise" +harness = false \ No newline at end of file diff --git a/benches/bitwise.rs b/benches/bitwise.rs new file mode 100644 index 00000000000..4085821f616 --- /dev/null +++ b/benches/bitwise.rs @@ -0,0 +1,70 @@ +extern crate arrow2; + +use arrow2::{ + compute::binary, datatypes::DataType, + types::NativeType, + util::bench_util::{create_boolean_array, create_primitive_array}, +}; + +use criterion::{criterion_group, criterion_main, Criterion}; +use arrow2::array::PrimitiveArray; +use std::ops::{BitXor, BitOr, Not}; +use num_traits::NumCast; +use arrow2::util::bench_util::create_primitive_array_with_seed; +use flatbuffers::bitflags::_core::ops::BitAnd; + +fn bench_or(lhs: &PrimitiveArray, rhs: &PrimitiveArray) + where + T: NativeType + BitOr + NumCast, +{ + criterion::black_box(binary::or(lhs, rhs)).unwrap(); +} + +fn bench_xor(lhs: &PrimitiveArray, rhs: &PrimitiveArray) + where + T: NativeType + BitXor + NumCast, +{ + criterion::black_box(binary::xor(lhs, rhs)).unwrap(); +} + +fn bench_and(lhs: &PrimitiveArray, rhs: &PrimitiveArray) + where + T: NativeType + BitAnd + NumCast, +{ + criterion::black_box(binary::and(lhs, rhs)).unwrap(); +} + +fn bench_not(arr: &PrimitiveArray) + where + T: NativeType + Not + NumCast, +{ + criterion::black_box(binary::not(arr)); +} + +fn add_benchmark(c: &mut Criterion) { + (10..=20).step_by(2).for_each(|log2_size| { + let size = 2usize.pow(log2_size); + let arr_a = create_primitive_array_with_seed::(size, DataType::UInt64, 0.0, 43); + let arr_b = create_primitive_array_with_seed::(size, DataType::UInt64, 0.0, 42); + + c.bench_function(&format!("add 2^{}", log2_size), |b| { + b.iter(|| bench_or(&arr_a, &arr_b)) + }); + + c.bench_function(&format!("xor 2^{}", log2_size), |b| { + b.iter(|| bench_xor(&arr_a, &arr_b)) + }); + + c.bench_function(&format!("and 2^{}", log2_size), |b| { + b.iter(|| bench_and(&arr_a, &arr_b)) + }); + + c.bench_function(&format!("not 2^{}", log2_size), |b| { + b.iter(|| bench_not(&arr_a)) + }); + }); +} + + +criterion_group!(benches, add_benchmark); +criterion_main!(benches); From 6976a4e09f415e6bbfd4d669a9a1c1279f03149e Mon Sep 17 00:00:00 2001 From: 1aguna Date: Tue, 26 Oct 2021 19:43:53 -0700 Subject: [PATCH 08/14] Move bitwise operations to a single file in compute --- benches/bitwise.rs | 12 ++++++------ src/compute/binary/mod.rs | 2 -- src/compute/{binary => }/bitwise.rs | 0 src/compute/boolean_kleene.rs | 2 +- src/compute/mod.rs | 2 +- tests/it/compute/binary/mod.rs | 1 - tests/it/compute/{binary => }/bitwise.rs | 4 ++-- tests/it/compute/mod.rs | 2 +- 8 files changed, 11 insertions(+), 14 deletions(-) delete mode 100644 src/compute/binary/mod.rs rename src/compute/{binary => }/bitwise.rs (100%) delete mode 100644 tests/it/compute/binary/mod.rs rename tests/it/compute/{binary => }/bitwise.rs (93%) diff --git a/benches/bitwise.rs b/benches/bitwise.rs index 4085821f616..6c391316de8 100644 --- a/benches/bitwise.rs +++ b/benches/bitwise.rs @@ -1,7 +1,7 @@ extern crate arrow2; use arrow2::{ - compute::binary, datatypes::DataType, + compute::bitwise::*, datatypes::DataType, types::NativeType, util::bench_util::{create_boolean_array, create_primitive_array}, }; @@ -17,28 +17,28 @@ fn bench_or(lhs: &PrimitiveArray, rhs: &PrimitiveArray) where T: NativeType + BitOr + NumCast, { - criterion::black_box(binary::or(lhs, rhs)).unwrap(); + criterion::black_box(or(lhs, rhs)).unwrap(); } fn bench_xor(lhs: &PrimitiveArray, rhs: &PrimitiveArray) where T: NativeType + BitXor + NumCast, { - criterion::black_box(binary::xor(lhs, rhs)).unwrap(); + criterion::black_box(xor(lhs, rhs)).unwrap(); } fn bench_and(lhs: &PrimitiveArray, rhs: &PrimitiveArray) where T: NativeType + BitAnd + NumCast, { - criterion::black_box(binary::and(lhs, rhs)).unwrap(); + criterion::black_box(and(lhs, rhs)).unwrap(); } fn bench_not(arr: &PrimitiveArray) where T: NativeType + Not + NumCast, { - criterion::black_box(binary::not(arr)); + criterion::black_box(not(arr)); } fn add_benchmark(c: &mut Criterion) { @@ -47,7 +47,7 @@ fn add_benchmark(c: &mut Criterion) { let arr_a = create_primitive_array_with_seed::(size, DataType::UInt64, 0.0, 43); let arr_b = create_primitive_array_with_seed::(size, DataType::UInt64, 0.0, 42); - c.bench_function(&format!("add 2^{}", log2_size), |b| { + c.bench_function(&format!("or 2^{}", log2_size), |b| { b.iter(|| bench_or(&arr_a, &arr_b)) }); diff --git a/src/compute/binary/mod.rs b/src/compute/binary/mod.rs deleted file mode 100644 index 8b43e47b216..00000000000 --- a/src/compute/binary/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod bitwise; -pub use bitwise::*; \ No newline at end of file diff --git a/src/compute/binary/bitwise.rs b/src/compute/bitwise.rs similarity index 100% rename from src/compute/binary/bitwise.rs rename to src/compute/bitwise.rs diff --git a/src/compute/boolean_kleene.rs b/src/compute/boolean_kleene.rs index 896e030e20c..4d65f449f22 100644 --- a/src/compute/boolean_kleene.rs +++ b/src/compute/boolean_kleene.rs @@ -26,7 +26,7 @@ use crate::{ pub fn or(lhs: &BooleanArray, rhs: &BooleanArray) -> Result { if lhs.len() != rhs.len() { return Err(ArrowError::InvalidArgumentError( - "Cannot perfobitwise operation on arrays of different length".to_string(), + "Cannot perform bitwise operation on arrays of different length".to_string(), )); } diff --git a/src/compute/mod.rs b/src/compute/mod.rs index 1f0f3d8d061..bfadd5577b3 100644 --- a/src/compute/mod.rs +++ b/src/compute/mod.rs @@ -14,7 +14,7 @@ pub mod aggregate; pub mod arithmetics; pub mod arity; -pub mod binary; +pub mod bitwise; pub mod boolean; pub mod boolean_kleene; pub mod cast; diff --git a/tests/it/compute/binary/mod.rs b/tests/it/compute/binary/mod.rs deleted file mode 100644 index e0282f29663..00000000000 --- a/tests/it/compute/binary/mod.rs +++ /dev/null @@ -1 +0,0 @@ -mod bitwise; \ No newline at end of file diff --git a/tests/it/compute/binary/bitwise.rs b/tests/it/compute/bitwise.rs similarity index 93% rename from tests/it/compute/binary/bitwise.rs rename to tests/it/compute/bitwise.rs index 19ee9252cb3..5a0379eb9fc 100644 --- a/tests/it/compute/binary/bitwise.rs +++ b/tests/it/compute/bitwise.rs @@ -1,9 +1,9 @@ #![allow(clippy::zero_prefixed_literal, clippy::inconsistent_digit_grouping)] use arrow2::array::*; -use arrow2::compute::arithmetics::decimal::add::*; -use arrow2::compute::binary::*; +use arrow2::compute::bitwise::*; use arrow2::datatypes::DataType; +use itertools::assert_equal; #[test] fn test_xor() { diff --git a/tests/it/compute/mod.rs b/tests/it/compute/mod.rs index bf74f5d1b0f..732f1319303 100644 --- a/tests/it/compute/mod.rs +++ b/tests/it/compute/mod.rs @@ -1,6 +1,6 @@ mod aggregate; mod arithmetics; -mod binary; +mod bitwise; mod boolean; mod boolean_kleene; mod cast; From 5c47348c3ba17463b82e566dba4d09025a2151ea Mon Sep 17 00:00:00 2001 From: 1aguna Date: Wed, 27 Oct 2021 17:21:29 -0700 Subject: [PATCH 09/14] Ran cargo fmt and removed unused imports --- benches/bitwise.rs | 26 +++++++++++++------------- src/compute/bitwise.rs | 25 ++++++++++++------------- src/compute/mod.rs | 1 - tests/it/compute/bitwise.rs | 5 ++--- 4 files changed, 27 insertions(+), 30 deletions(-) diff --git a/benches/bitwise.rs b/benches/bitwise.rs index 6c391316de8..a0da4291c34 100644 --- a/benches/bitwise.rs +++ b/benches/bitwise.rs @@ -1,42 +1,43 @@ extern crate arrow2; use arrow2::{ - compute::bitwise::*, datatypes::DataType, + compute::bitwise::*, + datatypes::DataType, types::NativeType, util::bench_util::{create_boolean_array, create_primitive_array}, }; -use criterion::{criterion_group, criterion_main, Criterion}; use arrow2::array::PrimitiveArray; -use std::ops::{BitXor, BitOr, Not}; -use num_traits::NumCast; use arrow2::util::bench_util::create_primitive_array_with_seed; +use criterion::{criterion_group, criterion_main, Criterion}; use flatbuffers::bitflags::_core::ops::BitAnd; +use num_traits::NumCast; +use std::ops::{BitOr, BitXor, Not}; fn bench_or(lhs: &PrimitiveArray, rhs: &PrimitiveArray) - where - T: NativeType + BitOr + NumCast, +where + T: NativeType + BitOr + NumCast, { criterion::black_box(or(lhs, rhs)).unwrap(); } fn bench_xor(lhs: &PrimitiveArray, rhs: &PrimitiveArray) - where - T: NativeType + BitXor + NumCast, +where + T: NativeType + BitXor + NumCast, { criterion::black_box(xor(lhs, rhs)).unwrap(); } fn bench_and(lhs: &PrimitiveArray, rhs: &PrimitiveArray) - where - T: NativeType + BitAnd + NumCast, +where + T: NativeType + BitAnd + NumCast, { criterion::black_box(and(lhs, rhs)).unwrap(); } fn bench_not(arr: &PrimitiveArray) - where - T: NativeType + Not + NumCast, +where + T: NativeType + Not + NumCast, { criterion::black_box(not(arr)); } @@ -65,6 +66,5 @@ fn add_benchmark(c: &mut Criterion) { }); } - criterion_group!(benches, add_benchmark); criterion_main!(benches); diff --git a/src/compute/bitwise.rs b/src/compute/bitwise.rs index 89219d36003..723d5de5f70 100644 --- a/src/compute/bitwise.rs +++ b/src/compute/bitwise.rs @@ -1,17 +1,16 @@ -use crate::array::{Array, BooleanArray, PrimitiveArray}; -use crate::datatypes::DataType; -use crate::error::{ArrowError, Result}; -use crate::types::NativeType; +use crate::array::{Array, PrimitiveArray}; use crate::compute::arithmetics::basic::check_same_type; use crate::compute::arity::{binary, unary}; -use std::ops::{BitOr, BitAnd, BitXor, Not}; +use crate::error::Result; +use crate::types::NativeType; +use std::ops::{BitAnd, BitOr, BitXor, Not}; /// Performs `OR` operation on two arrays. /// # Error /// This function errors when the arrays have different lengths or are different types. pub fn or(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Result> - where - T: NativeType + BitOr, +where + T: NativeType + BitOr, { check_same_type(lhs, rhs)?; binary(lhs, rhs, lhs.data_type().clone(), |a, b| a | b) @@ -21,8 +20,8 @@ pub fn or(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Result(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Result> - where - T: NativeType + BitXor, +where + T: NativeType + BitXor, { check_same_type(lhs, rhs)?; binary(lhs, rhs, lhs.data_type().clone(), |a, b| a ^ b) @@ -32,8 +31,8 @@ pub fn xor(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Result(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Result> - where - T: NativeType + BitAnd, +where + T: NativeType + BitAnd, { check_same_type(lhs, rhs)?; binary(lhs, rhs, lhs.data_type().clone(), |a, b| a & b) @@ -41,8 +40,8 @@ pub fn and(lhs: &PrimitiveArray, rhs: &PrimitiveArray) -> Result(arr: &PrimitiveArray) -> PrimitiveArray - where - T: NativeType + Not, +where + T: NativeType + Not, { let op = move |a: T| !a; unary(arr, op, arr.data_type().clone()) diff --git a/src/compute/mod.rs b/src/compute/mod.rs index bfadd5577b3..a48b87d18e4 100644 --- a/src/compute/mod.rs +++ b/src/compute/mod.rs @@ -45,4 +45,3 @@ pub mod regex_match; #[cfg(feature = "merge_sort")] #[cfg_attr(docsrs, doc(cfg(feature = "merge_sort")))] pub mod merge_sort; - diff --git a/tests/it/compute/bitwise.rs b/tests/it/compute/bitwise.rs index 5a0379eb9fc..05e5541a03f 100644 --- a/tests/it/compute/bitwise.rs +++ b/tests/it/compute/bitwise.rs @@ -19,7 +19,7 @@ fn test_xor() { fn test_and() { let a = Int32Array::from(&[Some(1), Some(2), Some(15)]); let b = Int32Array::from(&[None, Some(2), Some(6)]); - let result = and( &a, &b).unwrap(); + let result = and(&a, &b).unwrap(); let expected = Int32Array::from(&[None, Some(2), Some(6)]); assert_eq!(result, expected); @@ -35,7 +35,6 @@ fn test_or() { assert_eq!(result, expected); } - #[test] fn test_not() { let a = Int8Array::from(&[None, Some(1i8), Some(-100i8)]); @@ -43,4 +42,4 @@ fn test_not() { let expected = Int8Array::from(&[None, Some(-2), Some(99)]); assert_eq!(result, expected); -} \ No newline at end of file +} From ce9ca778bbc2bc57a4bbeef52bd172b6bfbfb04a Mon Sep 17 00:00:00 2001 From: 1aguna Date: Thu, 28 Oct 2021 22:02:40 -0700 Subject: [PATCH 10/14] Update use declarations in src/compute/bitwise.rs Co-authored-by: Jorge Leitao --- src/compute/bitwise.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/compute/bitwise.rs b/src/compute/bitwise.rs index 723d5de5f70..127c3cc9344 100644 --- a/src/compute/bitwise.rs +++ b/src/compute/bitwise.rs @@ -1,9 +1,10 @@ +use std::ops::{BitAnd, BitOr, BitXor, Not}; + use crate::array::{Array, PrimitiveArray}; use crate::compute::arithmetics::basic::check_same_type; use crate::compute::arity::{binary, unary}; use crate::error::Result; use crate::types::NativeType; -use std::ops::{BitAnd, BitOr, BitXor, Not}; /// Performs `OR` operation on two arrays. /// # Error From b29598cbce571b156ce6c62e24eb0fcb701e386c Mon Sep 17 00:00:00 2001 From: 1aguna Date: Thu, 28 Oct 2021 22:03:10 -0700 Subject: [PATCH 11/14] Remove itertools from tests/it/compute/bitwise.rs Co-authored-by: Jorge Leitao --- tests/it/compute/bitwise.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/it/compute/bitwise.rs b/tests/it/compute/bitwise.rs index 05e5541a03f..7a6cbc46ffb 100644 --- a/tests/it/compute/bitwise.rs +++ b/tests/it/compute/bitwise.rs @@ -3,7 +3,6 @@ use arrow2::array::*; use arrow2::compute::bitwise::*; use arrow2::datatypes::DataType; -use itertools::assert_equal; #[test] fn test_xor() { From 54f9739df440eeb14df53cfecedc3067165aca7d Mon Sep 17 00:00:00 2001 From: 1aguna Date: Thu, 28 Oct 2021 22:03:23 -0700 Subject: [PATCH 12/14] Update use declarations in benches/bitwise.rs Co-authored-by: Jorge Leitao --- benches/bitwise.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/benches/bitwise.rs b/benches/bitwise.rs index a0da4291c34..cf49d8a9f7c 100644 --- a/benches/bitwise.rs +++ b/benches/bitwise.rs @@ -1,19 +1,16 @@ -extern crate arrow2; +use std::ops::{BitOr, BitXor, Not}; + +use criterion::{criterion_group, criterion_main, Criterion}; +use num_traits::NumCast; use arrow2::{ + array::PrimitiveArray, compute::bitwise::*, datatypes::DataType, types::NativeType, - util::bench_util::{create_boolean_array, create_primitive_array}, + util::bench_util::{create_boolean_array, create_primitive_array, create_primitive_array_with_seed}, }; -use arrow2::array::PrimitiveArray; -use arrow2::util::bench_util::create_primitive_array_with_seed; -use criterion::{criterion_group, criterion_main, Criterion}; -use flatbuffers::bitflags::_core::ops::BitAnd; -use num_traits::NumCast; -use std::ops::{BitOr, BitXor, Not}; - fn bench_or(lhs: &PrimitiveArray, rhs: &PrimitiveArray) where T: NativeType + BitOr + NumCast, From 3e9ca086c5897027853d409d77204eaa04c959e6 Mon Sep 17 00:00:00 2001 From: 1aguna Date: Thu, 28 Oct 2021 22:03:49 -0700 Subject: [PATCH 13/14] Update tests/it/compute/bitwise.rs Co-authored-by: Jorge Leitao --- tests/it/compute/bitwise.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/it/compute/bitwise.rs b/tests/it/compute/bitwise.rs index 7a6cbc46ffb..e7ae729780d 100644 --- a/tests/it/compute/bitwise.rs +++ b/tests/it/compute/bitwise.rs @@ -1,4 +1,3 @@ -#![allow(clippy::zero_prefixed_literal, clippy::inconsistent_digit_grouping)] use arrow2::array::*; use arrow2::compute::bitwise::*; From f2e1757f50c2b4f2181733694e5f8756c926400d Mon Sep 17 00:00:00 2001 From: 1aguna Date: Sat, 30 Oct 2021 15:05:54 -0700 Subject: [PATCH 14/14] Ran cargo fmt and removed unused imports from tests/it/compute/bitwise.rs --- benches/bitwise.rs | 4 +++- tests/it/compute/bitwise.rs | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benches/bitwise.rs b/benches/bitwise.rs index cf49d8a9f7c..c91b5aad394 100644 --- a/benches/bitwise.rs +++ b/benches/bitwise.rs @@ -8,7 +8,9 @@ use arrow2::{ compute::bitwise::*, datatypes::DataType, types::NativeType, - util::bench_util::{create_boolean_array, create_primitive_array, create_primitive_array_with_seed}, + util::bench_util::{ + create_boolean_array, create_primitive_array, create_primitive_array_with_seed, + }, }; fn bench_or(lhs: &PrimitiveArray, rhs: &PrimitiveArray) diff --git a/tests/it/compute/bitwise.rs b/tests/it/compute/bitwise.rs index e7ae729780d..92b8b47b4ab 100644 --- a/tests/it/compute/bitwise.rs +++ b/tests/it/compute/bitwise.rs @@ -1,7 +1,5 @@ - use arrow2::array::*; use arrow2::compute::bitwise::*; -use arrow2::datatypes::DataType; #[test] fn test_xor() {