WIP.

Cargo.toml

@@ -18,13 +18,4 @@
 [workspace]
 members = [
     "datafusion",
-    "datafusion-cli",
-    "datafusion-examples",
-    "benchmarks",
-    "ballista/rust/client",
-    "ballista/rust/core",
-    "ballista/rust/executor",
-    "ballista/rust/scheduler",
 ]
-
-exclude = ["python"]

README.md

@@ -69,8 +69,8 @@ Run a SQL query against data stored in a CSV:
 
 ```rust
 use datafusion::prelude::*;
-use arrow::util::pretty::print_batches;
-use arrow::record_batch::RecordBatch;
+use arrow2::util::pretty::print_batches;
+use arrow2::record_batch::RecordBatch;
 
 #[tokio::main]
 async fn main() -> datafusion::error::Result<()> {
@@ -92,8 +92,8 @@ Use the DataFrame API to process data stored in a CSV:
 
 ```rust
 use datafusion::prelude::*;
-use arrow::util::pretty::print_batches;
-use arrow::record_batch::RecordBatch;
+use arrow2::util::pretty::print_batches;
+use arrow2::record_batch::RecordBatch;
 
 #[tokio::main]
 async fn main() -> datafusion::error::Result<()> {

datafusion-examples/examples/csv_sql.rs

@@ -27,7 +27,7 @@ async fn main() -> Result<()> {
     // create local execution context
     let mut ctx = ExecutionContext::new();
 
-    let testdata = datafusion::arrow::util::test_util::arrow_test_data();
+    let testdata = datafusion::crate::test::arrow_test_data();
 
     // register csv file with the execution context
     ctx.register_csv(

datafusion-examples/examples/dataframe.rs

@@ -27,7 +27,7 @@ async fn main() -> Result<()> {
     // create local execution context
     let mut ctx = ExecutionContext::new();
 
-    let testdata = datafusion::arrow::util::test_util::parquet_test_data();
+    let testdata = datafusion::crate::test::parquet_test_data();
 
     let filename = &format!("{}/alltypes_plain.parquet", testdata);
 

datafusion-examples/examples/flight_client.rs

@@ -31,7 +31,7 @@ use arrow_flight::{FlightDescriptor, Ticket};
 /// This example is run along-side the example `flight_server`.
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let testdata = datafusion::arrow::util::test_util::parquet_test_data();
+    let testdata = datafusion::crate::test::parquet_test_data();
 
     // Create Flight client
     let mut client = FlightServiceClient::connect("http://localhost:50051").await?;

datafusion-examples/examples/flight_server.rs

@@ -87,7 +87,7 @@ impl FlightService for FlightServiceImpl {
                 // create local execution context
                 let mut ctx = ExecutionContext::new();
 
-                let testdata = datafusion::arrow::util::test_util::parquet_test_data();
+                let testdata = datafusion::crate::test::parquet_test_data();
 
                 // register parquet file with the execution context
                 ctx.register_parquet(

datafusion-examples/examples/parquet_sql.rs

@@ -27,7 +27,7 @@ async fn main() -> Result<()> {
     // create local execution context
     let mut ctx = ExecutionContext::new();
 
-    let testdata = datafusion::arrow::util::test_util::parquet_test_data();
+    let testdata = datafusion::crate::test::parquet_test_data();
 
     // register parquet file with the execution context
     ctx.register_parquet(

datafusion/Cargo.toml

@@ -38,16 +38,14 @@ path = "src/lib.rs"
 
 [features]
 default = ["crypto_expressions", "regex_expressions", "unicode_expressions"]
-simd = ["arrow/simd"]
 crypto_expressions = ["md-5", "sha2"]
 regex_expressions = ["regex", "lazy_static"]
 unicode_expressions = ["unicode-segmentation"]
 
 [dependencies]
 ahash = "0.7"
 hashbrown = "0.11"
-arrow = { git = "https://github.com/apache/arrow-rs", rev = "4449ee96fe3fd4a0b275da8dd25ce2792699bc98", features = ["prettyprint"] }
-parquet = { git = "https://github.com/apache/arrow-rs", rev = "4449ee96fe3fd4a0b275da8dd25ce2792699bc98", features = ["arrow"] }
+arrow2 = { git = "https://github.com/jorgecarleitao/arrow2", rev = "4085967149a71f14589befb4e029220acb4eafe5" }
 sqlparser = "0.9.0"
 paste = "^1.0"
 num_cpus = "1.13.0"

datafusion/benches/aggregate_query_sql.rs

@@ -26,7 +26,7 @@ use tokio::runtime::Runtime;
 extern crate arrow;
 extern crate datafusion;
 
-use arrow::{
+use arrow2::{
     array::Float32Array,
     array::Float64Array,
     array::StringArray,

datafusion/benches/filter_query_sql.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use arrow::{
+use arrow2::{
     array::{Float32Array, Float64Array},
     datatypes::{DataType, Field, Schema},
     record_batch::RecordBatch,

datafusion/benches/math_query_sql.rs

@@ -26,7 +26,7 @@ use tokio::runtime::Runtime;
 extern crate arrow;
 extern crate datafusion;
 
-use arrow::{
+use arrow2::{
     array::{Float32Array, Float64Array},
     datatypes::{DataType, Field, Schema},
     record_batch::RecordBatch,

datafusion/benches/sort_limit_query_sql.rs

@@ -21,10 +21,7 @@ use criterion::Criterion;
 
 use std::sync::{Arc, Mutex};
 
-extern crate arrow;
-extern crate datafusion;
-
-use arrow::datatypes::{DataType, Field, Schema};
+use arrow2::datatypes::{DataType, Field, Schema};
 
 use datafusion::datasource::{CsvFile, CsvReadOptions, MemTable};
 use datafusion::execution::context::ExecutionContext;
@@ -57,7 +54,7 @@ fn create_context() -> Arc<Mutex<ExecutionContext>> {
         Field::new("c13", DataType::Utf8, false),
     ]));
 
-    let testdata = arrow::util::test_util::arrow_test_data();
+    let testdata = crate::test::arrow_test_data();
 
     // create CSV data source
     let csv = CsvFile::try_new(