Split ArrowField derive macro (#88)

* Split ArrowField derive macro into individual Field/Serialize/Deserialize derives for more composability. * Update README.md * fixes * Fix `struct_incorrect_type` test
DataEngineeringLabs · Jan 3, 2023 · 02db5dd · 02db5dd
1 parent 6c37e29
commit 02db5dd
Show file tree

Hide file tree

Showing 18 changed files with 817 additions and 749 deletions.
diff --git a/README.md b/README.md
@@ -14,9 +14,9 @@ Please see the [complex_example.rs](./arrow2_convert/tests/complex_example.rs) f
 /// Simple example
 
 use arrow2::array::Array;
-use arrow2_convert::{deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowField};
+use arrow2_convert::{deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowField, ArrowSerialize, ArrowDeserialize};
 
-#[derive(Debug, Clone, PartialEq, ArrowField)]
+#[derive(Debug, Clone, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
 pub struct Foo {
     name: String,
 }
@@ -47,7 +47,7 @@ fn main() {
 
 Types that implement the `ArrowField`, `ArrowSerialize` and `ArrowDeserialize` traits can be converted to/from Arrow via the `try_into_arrow` and the `try_into_collection` methods. 
 
-The `ArrowField` derive macro can be used to generate implementations of these traits for structs and enums. Custom implementations can also be defined for any type that needs to convert to/from Arrow by manually implementing the traits.
+The `ArrowField`, `ArrowSerialize` and `ArrowDeserialize` derive macros can be used to generate implementations of these traits for structs and enums. Custom implementations can also be defined for any type that needs to convert to/from Arrow by manually implementing the traits.
 
 For serializing to arrow, `TryIntoArrow::try_into_arrow` can be used to serialize any iterable into an `arrow2::Array` or a `arrow2::Chunk`.  `arrow2::Array` represents the in-memory Arrow layout. `arrow2::Chunk` represents a column group and can be used with `arrow2` API for other functionality such converting to parquet and arrow flight RPC.
 

diff --git a/arrow2_convert/src/lib.rs b/arrow2_convert/src/lib.rs
@@ -11,7 +11,7 @@ pub mod serialize;
 // macro_rules macro.
 #[cfg(feature = "arrow2_convert_derive")]
 #[doc(hidden)]
-pub use arrow2_convert_derive::ArrowField;
+pub use arrow2_convert_derive::{ArrowDeserialize, ArrowField, ArrowSerialize};
 
 // Test README with doctests
 #[doc = include_str!("../README.md")]

diff --git a/arrow2_convert/tests/complex_example.rs b/arrow2_convert/tests/complex_example.rs
@@ -5,10 +5,10 @@ use arrow2_convert::serialize::TryIntoArrow;
 ///
 /// - Deeply Nested structs and lists
 /// - Custom types
-use arrow2_convert::ArrowField;
+use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize};
 use std::borrow::Borrow;
 
-#[derive(Debug, Clone, PartialEq, ArrowField)]
+#[derive(Debug, Clone, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
 pub struct Root {
     name: Option<String>,
     is_deleted: bool,
@@ -55,15 +55,15 @@ pub struct Root {
     fixed_size_vec: Vec<i64>,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, ArrowField)]
+#[derive(Debug, Clone, PartialEq, Eq, ArrowField, ArrowSerialize, ArrowDeserialize)]
 pub struct Child {
     a1: i64,
     a2: String,
     // nested struct array
     child_array: Vec<ChildChild>,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, ArrowField)]
+#[derive(Debug, Clone, PartialEq, Eq, ArrowField, ArrowSerialize, ArrowDeserialize)]
 pub struct ChildChild {
     a1: i32,
     bool_array: Vec<bool>,
@@ -126,12 +126,12 @@ fn item1() -> Root {
         a1: Some(0.1),
         a2: 1,
         a3: Some(b"aa".to_vec()),
-        a4: NaiveDate::from_ymd(1970, 1, 2),
-        a5: NaiveDateTime::from_timestamp(10000, 0),
-        a6: Some(NaiveDateTime::from_timestamp(10001, 0)),
+        a4: NaiveDate::from_ymd_opt(1970, 1, 2).unwrap(),
+        a5: NaiveDateTime::from_timestamp_opt(10000, 0).unwrap(),
+        a6: Some(NaiveDateTime::from_timestamp_opt(10001, 0)).unwrap(),
         date_time_list: vec![
-            NaiveDateTime::from_timestamp(10000, 10),
-            NaiveDateTime::from_timestamp(10000, 11),
+            NaiveDateTime::from_timestamp_opt(10000, 10).unwrap(),
+            NaiveDateTime::from_timestamp_opt(10000, 11).unwrap(),
         ],
         nullable_list: Some(vec![Some("cc".to_string()), Some("dd".to_string())]),
         required_list: vec![Some("aa".to_string()), Some("bb".to_string())],
@@ -172,12 +172,12 @@ fn item2() -> Root {
         a1: Some(0.1),
         a2: 1,
         a3: Some(b"aa".to_vec()),
-        a4: NaiveDate::from_ymd(1970, 1, 2),
-        a5: NaiveDateTime::from_timestamp(10000, 0),
+        a4: NaiveDate::from_ymd_opt(1970, 1, 2).unwrap(),
+        a5: NaiveDateTime::from_timestamp_opt(10000, 0).unwrap(),
         a6: None,
         date_time_list: vec![
-            NaiveDateTime::from_timestamp(10000, 10),
-            NaiveDateTime::from_timestamp(10000, 11),
+            NaiveDateTime::from_timestamp_opt(10000, 10).unwrap(),
+            NaiveDateTime::from_timestamp_opt(10000, 11).unwrap(),
         ],
         nullable_list: None,
         required_list: vec![Some("ee".to_string()), Some("ff".to_string())],

diff --git a/arrow2_convert/tests/simple_example.rs b/arrow2_convert/tests/simple_example.rs
@@ -1,8 +1,11 @@
 /// Simple example
 use arrow2::array::Array;
-use arrow2_convert::{deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowField};
+use arrow2_convert::{
+    deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowDeserialize, ArrowField,
+    ArrowSerialize,
+};
 
-#[derive(Debug, Clone, PartialEq, Eq, ArrowField)]
+#[derive(Debug, Clone, PartialEq, Eq, ArrowField, ArrowSerialize, ArrowDeserialize)]
 pub struct Foo {
     name: String,
 }

diff --git a/arrow2_convert/tests/test_deserialize.rs b/arrow2_convert/tests/test_deserialize.rs
@@ -1,8 +1,6 @@
 use arrow2::array::*;
 use arrow2::error::Result;
-use arrow2_convert::deserialize::*;
-use arrow2_convert::serialize::*;
-use arrow2_convert::ArrowField;
+use arrow2_convert::{deserialize::*, serialize::*, ArrowDeserialize, ArrowField, ArrowSerialize};
 
 #[test]
 fn test_deserialize_iterator() {
@@ -11,7 +9,7 @@ fn test_deserialize_iterator() {
     use arrow2_convert::serialize::*;
     use std::borrow::Borrow;
 
-    #[derive(Debug, Clone, PartialEq, ArrowField)]
+    #[derive(Debug, Clone, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
     struct S {
         a1: i64,
     }
@@ -34,11 +32,11 @@ fn test_deserialize_iterator() {
 
 #[test]
 fn test_deserialize_schema_mismatch_error() {
-    #[derive(Debug, Clone, PartialEq, ArrowField)]
+    #[derive(Debug, Clone, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
     struct S1 {
         a: i64,
     }
-    #[derive(Debug, Clone, PartialEq, ArrowField)]
+    #[derive(Debug, Clone, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
     struct S2 {
         a: String,
     }
@@ -56,11 +54,11 @@ fn test_deserialize_schema_mismatch_error() {
 
 #[test]
 fn test_deserialize_large_types_schema_mismatch_error() {
-    #[derive(Debug, Clone, PartialEq, ArrowField)]
+    #[derive(Debug, Clone, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
     struct S1 {
         a: String,
     }
-    #[derive(Debug, Clone, PartialEq, ArrowField)]
+    #[derive(Debug, Clone, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
     struct S2 {
         #[arrow_field(type = "arrow2_convert::field::LargeString")]
         a: String,

diff --git a/arrow2_convert/tests/test_enum.rs b/arrow2_convert/tests/test_enum.rs
@@ -1,9 +1,12 @@
 use arrow2::array::*;
-use arrow2_convert::{deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowField};
+use arrow2_convert::{
+    deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowDeserialize, ArrowField,
+    ArrowSerialize,
+};
 
 #[test]
 fn test_dense_enum_unit_variant() {
-    #[derive(Debug, PartialEq, ArrowField)]
+    #[derive(Debug, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
     #[arrow_field(type = "dense")]
     enum TestEnum {
         VAL1,
@@ -25,7 +28,7 @@ fn test_dense_enum_unit_variant() {
 
 #[test]
 fn test_sparse_enum_unit_variant() {
-    #[derive(Debug, PartialEq, ArrowField)]
+    #[derive(Debug, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
     #[arrow_field(type = "sparse")]
     enum TestEnum {
         VAL1,
@@ -47,12 +50,12 @@ fn test_sparse_enum_unit_variant() {
 
 #[test]
 fn test_nested_unit_variant() {
-    #[derive(Debug, PartialEq, ArrowField)]
+    #[derive(Debug, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
     struct TestStruct {
         a1: i64,
     }
 
-    #[derive(Debug, PartialEq, ArrowField)]
+    #[derive(Debug, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
     #[arrow_field(type = "dense")]
     enum TestEnum {
         VAL1,
@@ -62,7 +65,7 @@ fn test_nested_unit_variant() {
         VAL5(ChildEnum),
     }
 
-    #[derive(Debug, PartialEq, ArrowField)]
+    #[derive(Debug, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
     #[arrow_field(type = "sparse")]
     enum ChildEnum {
         VAL1,
@@ -87,12 +90,12 @@ fn test_nested_unit_variant() {
 //#[test]
 #[allow(unused)]
 fn test_slice() {
-    #[derive(Debug, PartialEq, ArrowField)]
+    #[derive(Debug, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
     struct TestStruct {
         a1: i64,
     }
 
-    #[derive(Debug, PartialEq, ArrowField)]
+    #[derive(Debug, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
     #[arrow_field(type = "dense")]
     enum TestEnum {
         VAL1,
@@ -102,7 +105,7 @@ fn test_slice() {
         VAL5(ChildEnum),
     }
 
-    #[derive(Debug, PartialEq, ArrowField)]
+    #[derive(Debug, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
     #[arrow_field(type = "sparse")]
     enum ChildEnum {
         VAL1,

diff --git a/arrow2_convert/tests/test_flatten_chunk.rs b/arrow2_convert/tests/test_flatten_chunk.rs
@@ -1,12 +1,11 @@
 use arrow2::array::*;
 use arrow2::chunk::Chunk;
-use arrow2_convert::serialize::*;
-use arrow2_convert::ArrowField;
+use arrow2_convert::{serialize::*, ArrowField, ArrowSerialize};
 use std::sync::Arc;
 
 #[test]
 fn test_flatten_chunk() {
-    #[derive(Debug, Clone, ArrowField)]
+    #[derive(Debug, Clone, ArrowField, ArrowSerialize)]
     struct Struct {
         a: i64,
         b: i64,
@@ -35,7 +34,7 @@ fn test_flatten_chunk_empty_chunk_error() {
 
 #[test]
 fn test_flatten_chunk_no_single_struct_array_error() {
-    #[derive(Debug, Clone, ArrowField)]
+    #[derive(Debug, Clone, ArrowField, ArrowSerialize)]
     struct Struct {
         a: i64,
         b: String,

diff --git a/arrow2_convert/tests/test_hygeine.rs b/arrow2_convert/tests/test_hygeine.rs
@@ -2,6 +2,7 @@
 use arrow2_convert_derive::ArrowField;
 
 #[derive(ArrowField)]
+#[allow(dead_code)]
 struct S {
     int_field: i64,
 }
diff --git a/arrow2_convert/tests/test_round_trip.rs b/arrow2_convert/tests/test_round_trip.rs
@@ -6,18 +6,18 @@ use arrow2_convert::field::{LargeBinary, I128};
 use arrow2_convert::serialize::*;
 use arrow2_convert::{
     field::{FixedSizeBinary, FixedSizeVec, LargeString, LargeVec},
-    ArrowField,
+    ArrowDeserialize, ArrowField, ArrowSerialize,
 };
 use std::borrow::Borrow;
 use std::sync::Arc;
 
 #[test]
 fn test_nested_optional_struct_array() {
-    #[derive(Debug, Clone, ArrowField, PartialEq)]
+    #[derive(Debug, Clone, ArrowField, ArrowSerialize, ArrowDeserialize, PartialEq)]
     struct Top {
         child_array: Vec<Option<Child>>,
     }
-    #[derive(Debug, Clone, ArrowField, PartialEq)]
+    #[derive(Debug, Clone, ArrowField, ArrowSerialize, ArrowDeserialize, PartialEq)]
     struct Child {
         a1: i64,
     }
@@ -237,7 +237,7 @@ fn test_primitive_type_vec() {
 
 #[test]
 fn test_escaped_name() {
-    #[derive(ArrowField, Debug, Eq, PartialEq)]
+    #[derive(ArrowField, ArrowSerialize, ArrowDeserialize, Debug, Eq, PartialEq)]
     struct EscapedName {
         r#type: bool,
     }

diff --git a/arrow2_convert/tests/test_schema.rs b/arrow2_convert/tests/test_schema.rs
@@ -4,6 +4,7 @@ use arrow2_convert::ArrowField;
 #[test]
 fn test_schema_types() {
     #[derive(Debug, ArrowField)]
+    #[allow(dead_code)]
     struct Root {
         name: Option<String>,
         is_deleted: bool,
@@ -54,6 +55,7 @@ fn test_schema_types() {
     }
 
     #[derive(Debug, ArrowField)]
+    #[allow(dead_code)]
     struct Child {
         a1: i64,
         a2: String,
@@ -62,6 +64,7 @@ fn test_schema_types() {
     }
 
     #[derive(Debug, ArrowField)]
+    #[allow(dead_code)]
     pub struct ChildChild {
         a1: i32,
         bool_array: Vec<bool>,

diff --git a/arrow2_convert/tests/test_struct.rs b/arrow2_convert/tests/test_struct.rs
@@ -1,15 +1,13 @@
 use arrow2::array::*;
-use arrow2_convert::deserialize::*;
-use arrow2_convert::serialize::*;
-use arrow2_convert::ArrowField;
+use arrow2_convert::{deserialize::*, serialize::*, ArrowDeserialize, ArrowField, ArrowSerialize};
 
 #[test]
 fn test_nested_optional_struct_array() {
-    #[derive(Debug, Clone, ArrowField, PartialEq)]
+    #[derive(Debug, Clone, ArrowField, ArrowSerialize, ArrowDeserialize, PartialEq)]
     struct Top {
         child_array: Vec<Option<Child>>,
     }
-    #[derive(Debug, Clone, ArrowField, PartialEq)]
+    #[derive(Debug, Clone, ArrowField, ArrowSerialize, ArrowDeserialize, PartialEq)]
     struct Child {
         a1: i64,
     }
@@ -38,7 +36,7 @@ fn test_nested_optional_struct_array() {
 
 #[test]
 fn test_slice() {
-    #[derive(Debug, Clone, ArrowField, PartialEq)]
+    #[derive(Debug, Clone, ArrowField, ArrowSerialize, ArrowDeserialize, PartialEq)]
     struct T {
         a1: i64,
     }
@@ -57,11 +55,11 @@ fn test_slice() {
 
 #[test]
 fn test_nested_slice() {
-    #[derive(Debug, Clone, ArrowField, PartialEq)]
+    #[derive(Debug, Clone, ArrowField, ArrowSerialize, ArrowDeserialize, PartialEq)]
     struct Top {
         child_array: Vec<Option<Child>>,
     }
-    #[derive(Debug, Clone, ArrowField, PartialEq)]
+    #[derive(Debug, Clone, ArrowField, ArrowSerialize, ArrowDeserialize, PartialEq)]
     struct Child {
         a1: i64,
     }

diff --git a/arrow2_convert/tests/ui/struct_incorrect_type.rs b/arrow2_convert/tests/ui/struct_incorrect_type.rs
@@ -1,11 +1,10 @@
-use arrow2_convert::ArrowField;
 use arrow2_convert::field::LargeBinary;
+use arrow2_convert::{ArrowDeserialize, ArrowField, ArrowSerialize};
 
-#[derive(Debug, ArrowField)]
+#[derive(Debug, ArrowField, ArrowSerialize, ArrowDeserialize)]
 struct Test {
-    #[arrow_field(type="LargeBinary")]
-    s: String
+    #[arrow_field(type = "LargeBinary")]
+    s: String,
 }
 
-fn main() 
-{}
+fn main() {}