Rewrite

Cargo.toml

@@ -1,47 +1,41 @@
 [package]
-edition = "2021"
 name = "datafusion-tui"
 description = "Terminal based, extensible, interactive data analysis tool using SQL "
-version = "0.1.0"
 homepage = "https://github.com/datafusion-contrib/datafusion-tui"
 repository = "https://github.com/datafusion-contrib/datafusion-tui"
 readme = "README.md"
 authors = ["Matthew Turner <[email protected]>"]
 license = "Apache-2.0"
 keywords = ["arrow", "query", "sql", "datafusion"]
+version = "0.1.0"
+edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-
-arrow = "13"
-aws-sdk-s3 = "0.9.0"
-aws-types = "0.9.0"
-ballista = { version = "0.7", optional = true }
-clap = { version = "3", features = ["derive", "cargo"] }
-crossterm = { version = "0.23", features = ["serde"] }
-datafusion = "8"
-datafusion-catalogprovider-glue = { version = "0.1.7", optional = true }
-datafusion-objectstore-s3 = { version = "0.2.1", optional = true }
-dirs = "4.0"
-futures = "0.3"
-http = "0.2.6"
-log = "0.4"
-mimalloc = { version = "0.1", default-features = false }
-serde = "1.0.136"
-serde_json = "1.0.79"
-ratatui = { version = "0.28", default-features = false, features = [
-    'crossterm',
-    'serde',
-] }
-tui-logger = {version="0.12", features=["crossterm"]}
-tokio = { version = "1", features = ["full"] }
-unicode-width = "0.1.9"
-
-[features]
-s3 = ["datafusion-objectstore-s3"]
-glue = ["datafusion-catalogprovider-glue"]
+async-trait = "0.1.80"
+clap = { version = "4.5.1", features = ["derive"] }
+color-eyre = "0.6.3"
+crossterm = { version = "0.28.1", features = ["event-stream"] }
+datafusion = "40.0.0"
+directories = "5.0.1"
+futures = "0.3.30"
+itertools = "0.13.0"
+lazy_static = "1.4.0"
+log = "0.4.22"
+ratatui = "0.28.0"
+serde = { version = "1.0.197", features = ["derive"] }
+strum = "0.26.2"
+tokio = { version = "1.36.0", features = ["rt-multi-thread", "macros"] }
+tokio-stream = "0.1.15"
+tokio-util = "0.7.10"
+toml = "0.8.12"
+tui-logger = {version = "0.12", features = ["tracing-support"]}
+tui-textarea = "0.6.1"
 
 [[bin]]
 name = "dft"
 path = "src/main.rs"
+
+[lints.clippy]
+clone_on_ref_ptr = "deny"

README.md

@@ -29,8 +29,6 @@ Some of the current and planned features are:
   - There are ongoing conversations in DataFusion about adopting a new `ObjectStore` interface that would come with bindings to S3, ADLS, and GCP.  I am  monitoring this and plan on updating to use that interface when it is available.
 - `TableProvider` data sources
   - Delta Table => TODO
-  - Google Big Table => (currently in the bigtable branch which isnt up to date with latest DataFusion )
-  - ApiTable => Will allow treating API endpoints as tables by handling pagination and authentication.  Currently being prototyped in [#85](https://github.com/datafusion-contrib/datafusion-tui/pull/85)
 - Preloading DDL from `~/.datafusion/.datafusionrc` for local database available on startup
 
 ## User Guide
@@ -61,15 +59,6 @@ CREATE VIEW OR REPLACE users_listings AS SELECT * FROM users LEFT JOIN listings
 
 This would make the tables `users`, `taxis`, `listings`, and the view  `users_listings` available at startup.  Any of these DDL statements could also be run interactively from the SQL editor as well to create the tables.
 
-### Catalog Providers
-
-You can run `dft` with `--features=glue,s3` to registered databases and tables from your AWS Glue Catalogs.  You can optionally use a configuration file to specify which databases you want to register.  To use a configuration create `~/.datafusion/catalog_providers/glue.json`.  In that file you can specify the databases that you want to register:
-
-```json
-{
-  "databases": ["my_database", "my_other_database"]
-}
-```
 
 ### Key Mappings
 
@@ -109,5 +98,3 @@ The interface is split into several tabs so that relevant information can be vie
   - S3: run / install with `--features=s3`
     - If you want to use your default AWS credentials, then no further action is required. For example your credentials in `~/.aws/credentials` will automatically be picked up.
     - If you want to use a custom S3 provider, such as MinIO, then you must create a `s3.json` configuration file in `~/.datafusion/object_stores/` with the fields `endpoint`, `access_key_id`, and `secret_access_key`.
-- Use Ballista as execution engine
-  -  Run / install with `--features=ballista` and provide `--host` and `--port` of the ballista scheduler when running `dft`

src/app/config.rs

@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::path::PathBuf;
+
+use directories::{ProjectDirs, UserDirs};
+use lazy_static::lazy_static;
+use serde::Deserialize;
+
+lazy_static! {
+    pub static ref PROJECT_NAME: String = env!("CARGO_CRATE_NAME").to_uppercase().to_string();
+    pub static ref DATA_FOLDER: Option<PathBuf> =
+        std::env::var(format!("{}_DATA", PROJECT_NAME.clone()))
+            .ok()
+            .map(PathBuf::from);
+    pub static ref LOG_ENV: String = format!("{}_LOGLEVEL", PROJECT_NAME.clone());
+    pub static ref LOG_FILE: String = format!("{}.log", env!("CARGO_PKG_NAME"));
+}
+
+fn project_directory() -> PathBuf {
+    if let Some(user_dirs) = UserDirs::new() {
+        return user_dirs.home_dir().join(".config").join("dft");
+    };
+
+    let maybe_project_dirs = ProjectDirs::from("", "", env!("CARGO_PKG_NAME"));
+    if let Some(project_dirs) = maybe_project_dirs {
+        project_dirs.data_local_dir().to_path_buf()
+    } else {
+        panic!("No known data directory")
+    }
+}
+
+pub fn get_data_dir() -> PathBuf {
+    if let Some(data_dir) = DATA_FOLDER.clone() {
+        data_dir
+    } else {
+        project_directory()
+    }
+}
+
+#[derive(Debug, Default, Deserialize)]
+pub struct AppConfig {
+    #[serde(default = "default_datafusion_config")]
+    pub datafusion: DataFusionConfig,
+    #[serde(default = "default_display_config")]
+    pub display: DisplayConfig,
+    #[serde(default = "default_interaction_config")]
+    pub interaction: InteractionConfig,
+}
+
+fn default_datafusion_config() -> DataFusionConfig {
+    DataFusionConfig::default()
+}
+
+fn default_display_config() -> DisplayConfig {
+    DisplayConfig::default()
+}
+
+fn default_interaction_config() -> InteractionConfig {
+    InteractionConfig::default()
+}
+
+#[derive(Debug, Deserialize)]
+pub struct DisplayConfig {
+    #[serde(default = "default_tick_rate")]
+    pub tick_rate: f64,
+    #[serde(default = "default_frame_rate")]
+    pub frame_rate: f64,
+}
+
+fn default_tick_rate() -> f64 {
+    60.0
+}
+
+fn default_frame_rate() -> f64 {
+    60.0
+}
+
+impl Default for DisplayConfig {
+    fn default() -> Self {
+        Self {
+            tick_rate: 5.0,
+            frame_rate: 5.0,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct DataFusionConfig {
+    #[serde(default = "default_stream_batch_size")]
+    pub stream_batch_size: usize,
+}
+
+fn default_stream_batch_size() -> usize {
+    1
+}
+
+impl Default for DataFusionConfig {
+    fn default() -> Self {
+        Self {
+            stream_batch_size: 1,
+        }
+    }
+}
+
+#[derive(Debug, Default, Deserialize)]
+pub struct InteractionConfig {
+    #[serde(default = "default_mouse")]
+    pub mouse: bool,
+    #[serde(default = "default_paste")]
+    pub paste: bool,
+}
+
+fn default_mouse() -> bool {
+    false
+}
+
+fn default_paste() -> bool {
+    false
+}