Factor out common datafusion types into another proto file (apache#10649

)

* Initial commit

* Minor changes

* Minor changes

* Minor changes

* Minor changes

* Move Column type

* Minor changes buggy

* Compiles

* Add new types to common

* Move scalarvalue

* Move new types to common

* Move additional fields

* Move common types

* Minor changes

* Change proto definition order

* Simplifications

* Minor changes

* REimport protobuf_common from proto

* Address reviews

Cargo.toml

@@ -32,6 +32,8 @@ members = [
     "datafusion/physical-plan",
     "datafusion/proto",
     "datafusion/proto/gen",
+    "datafusion/proto-common",
+    "datafusion/proto-common/gen",
     "datafusion/sql",
     "datafusion/sqllogictest",
     "datafusion/substrait",
@@ -89,6 +91,7 @@ datafusion-physical-expr = { path = "datafusion/physical-expr", version = "38.0.
 datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "38.0.0", default-features = false }
 datafusion-physical-plan = { path = "datafusion/physical-plan", version = "38.0.0" }
 datafusion-proto = { path = "datafusion/proto", version = "38.0.0" }
+datafusion-proto-common = { path = "datafusion/proto-common", version = "38.0.0" }
 datafusion-sql = { path = "datafusion/sql", version = "38.0.0" }
 datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "38.0.0" }
 datafusion-substrait = { path = "datafusion/substrait", version = "38.0.0" }

datafusion/proto-common/.gitignore

@@ -0,0 +1,4 @@
+# Files generated by regen.sh
+proto/proto_descriptor.bin
+src/datafusion_common.rs
+src/datafusion_common.serde.rs

datafusion/proto-common/Cargo.toml

@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "datafusion-proto-common"
+description = "Protobuf serialization of DataFusion common types"
+keywords = ["arrow", "query", "sql"]
+version = { workspace = true }
+edition = { workspace = true }
+readme = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+authors = { workspace = true }
+rust-version = "1.73"
+
+# Exclude proto files so crates.io consumers don't need protoc
+exclude = ["*.proto"]
+
+[lib]
+name = "datafusion_proto_common"
+path = "src/lib.rs"
+
+[features]
+default = []
+json = ["serde", "serde_json", "pbjson"]
+
+[dependencies]
+arrow = { workspace = true }
+chrono = { workspace = true }
+datafusion-common = { workspace = true }
+object_store = { version = "0.9.0" }
+pbjson = { version = "0.6.0", optional = true }
+prost = "0.12.0"
+serde = { version = "1.0", optional = true }
+serde_json = { version = "1.0", optional = true }
+
+[dev-dependencies]
+doc-comment = "0.3"
+tokio = "1.18"

datafusion/proto-common/README.md

@@ -0,0 +1,28 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# `datafusion-proto-common`: Apache DataFusion Protobuf Serialization / Deserialization
+
+This crate contains code to convert Apache [DataFusion] primitive types to and from
+bytes, which can be useful for sending data over the network.
+
+See [API Docs] for details and examples.
+
+[datafusion]: https://datafusion.apache.org
+[api docs]: http://docs.rs/datafusion-proto/latest

datafusion/proto-common/gen/Cargo.toml

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "gen-common"
+description = "Code generation for proto"
+version = "0.1.0"
+edition = { workspace = true }
+rust-version = "1.73"
+authors = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+publish = false
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[lints]
+workspace = true
+
+[dependencies]
+# Pin these dependencies so that the generated output is deterministic
+pbjson-build = "=0.6.2"
+prost-build = "=0.12.6"

datafusion/proto-common/gen/src/main.rs

@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::path::Path;
+
+type Error = Box<dyn std::error::Error>;
+type Result<T, E = Error> = std::result::Result<T, E>;
+
+fn main() -> Result<(), String> {
+    let proto_dir = Path::new("proto");
+    let proto_path = Path::new("proto/datafusion_common.proto");
+
+    // proto definitions has to be there
+    let descriptor_path = proto_dir.join("proto_descriptor.bin");
+
+    prost_build::Config::new()
+        .file_descriptor_set_path(&descriptor_path)
+        .out_dir("src")
+        .compile_well_known_types()
+        .extern_path(".google.protobuf", "::pbjson_types")
+        .compile_protos(&[proto_path], &["proto"])
+        .map_err(|e| format!("protobuf compilation failed: {e}"))?;
+
+    let descriptor_set = std::fs::read(&descriptor_path)
+        .unwrap_or_else(|e| panic!("Cannot read {:?}: {}", &descriptor_path, e));
+
+    pbjson_build::Builder::new()
+        .out_dir("src")
+        .register_descriptors(&descriptor_set)
+        .unwrap_or_else(|e| {
+            panic!("Cannot register descriptors {:?}: {}", &descriptor_set, e)
+        })
+        .build(&[".datafusion_common"])
+        .map_err(|e| format!("pbjson compilation failed: {e}"))?;
+
+    let prost = Path::new("src/datafusion_common.rs");
+    let pbjson = Path::new("src/datafusion_common.serde.rs");
+
+    std::fs::copy(prost, "src/generated/prost.rs").unwrap();
+    std::fs::copy(pbjson, "src/generated/pbjson.rs").unwrap();
+
+    Ok(())
+}