Merge ; commit '0189b38859649424af28b321740041086d026541'

iesahin · Jun 29, 2024 · 4df7a67 · 4df7a67
2 parents 35d7773 + 0189b38
commit 4df7a67
Show file tree

Hide file tree

Showing 34 changed files with 246 additions and 762 deletions.
diff --git a/README.md b/README.md
@@ -74,13 +74,13 @@ $ xvc file track my-models/ --as copy
 Configure a cloud storage to share the files you added.
 
 ```shell
-$ xvc storage new s3 --name my-remote --region us-east-1 --bucket-name my-xvc-remote
+$ xvc storage new s3 --name my-storage --region us-east-1 --bucket-name my-xvc-remote
 ```
 
 You can send the files to this storage.
 
 ```shell
-$ xvc file send --to my-remote
+$ xvc file send --to my-storage
 ```
 
 When you (or someone else) want to access these files later, you can clone the Git repository and get the files from the
@@ -91,7 +91,7 @@ $ git clone https://example.com/my-machine-learning-project
 Cloning into 'my-machine-learning-project'...
 
 $ cd my-machine-learning-project
-$ xvc file bring my-data/ --from my-remote
+$ xvc file bring my-data/ --from my-storage
 
 ```
 

diff --git a/book/src/ref/xvc-file-send.md b/book/src/ref/xvc-file-send.md
@@ -12,7 +12,7 @@ Arguments:
   [TARGETS]...  Targets to send/push/upload to storage
 
 Options:
-  -r, --remote <REMOTE>  Storage name or guid to send the files
+  -r, --storage <REMOTE>  Storage name or guid to send the files
       --force            Force even if the files are already present in the storage
   -h, --help             Print help
 

diff --git a/book/src/ref/xvc-file-share.md b/book/src/ref/xvc-file-share.md
@@ -6,13 +6,13 @@
 $ xvc file share --help
 Share a file from S3 compatible storage for a limited time
 
-Usage: xvc file share [OPTIONS] --remote <REMOTE> <TARGET>
+Usage: xvc file share [OPTIONS] --storage <REMOTE> <TARGET>
 
 Arguments:
   <TARGET>  File to send/push/upload to storage
 
 Options:
-  -r, --remote <REMOTE>      Storage name or guid to send the files
+  -r, --storage <REMOTE>      Storage name or guid to send the files
   -d, --duration <DURATION>  Period to send the files to. You can use s, m, h, d, w suffixes [default: 24h]
   -h, --help                 Print help
 

diff --git a/config/Cargo.toml b/config/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "xvc-config"
-version = "0.6.8-alpha.1"
+version = "0.6.8-alpha.2"
 edition = "2021"
 description = "Xvc configuration management"
 authors = ["Emre Şahin <[email protected]>"]
@@ -20,8 +20,8 @@ debug = true
 
 
 [dependencies]
-xvc-logging = { version = "0.6.8-alpha.1", path = "../logging" }
-xvc-walker = { version = "0.6.8-alpha.1", path = "../walker" }
+xvc-logging = { version = "0.6.8-alpha.2", path = "../logging" }
+xvc-walker = { version = "0.6.8-alpha.2", path = "../walker" }
 
 
 ## Cli and config

diff --git a/config/src/lib.rs b/config/src/lib.rs
@@ -171,10 +171,12 @@ pub struct XvcConfigInitParams {
     /// If `true`, it's read from [USER_CONFIG_DIRS].
     pub include_user_config: bool,
     /// Where should we load the project's (public) configuration?
-    /// If `None`, it's ignored.
+    /// It's loaded in [XvcRootInner::new]
+    /// TODO: Add a option to ignore this
     pub project_config_path: Option<AbsolutePath>,
     /// Where should we load the project's (private) configuration?
-    /// If `None`, it's ignored.
+    /// It's loaded in [XvcRootInner::new]
+    /// TODO: Add a option to ignore this
     pub local_config_path: Option<AbsolutePath>,
     /// Should we include configuration from the environment.
     /// If `true`, look for all variables in the form

diff --git a/core/Cargo.toml b/core/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "xvc-core"
-version = "0.6.8-alpha.1"
+version = "0.6.8-alpha.2"
 edition = "2021"
 description = "Xvc core for common elements for all commands"
 authors = ["Emre Şahin <[email protected]>"]
@@ -19,10 +19,10 @@ crate-type = ["rlib"]
 debug = true
 
 [dependencies]
-xvc-config = { version = "0.6.8-alpha.1", path = "../config" }
-xvc-logging = { version = "0.6.8-alpha.1", path = "../logging" }
-xvc-ecs = { version = "0.6.8-alpha.1", path = "../ecs" }
-xvc-walker = { version = "0.6.8-alpha.1", path = "../walker" }
+xvc-config = { version = "0.6.8-alpha.2", path = "../config" }
+xvc-logging = { version = "0.6.8-alpha.2", path = "../logging" }
+xvc-ecs = { version = "0.6.8-alpha.2", path = "../ecs" }
+xvc-walker = { version = "0.6.8-alpha.2", path = "../walker" }
 
 ## Cli and config
 clap = { version = "^4.4", features = ["derive"] }
@@ -82,6 +82,6 @@ derive_more = "^0.99"
 itertools = "^0.12"
 
 [dev-dependencies]
-xvc-test-helper = { version = "0.6.8-alpha.1", path = "../test_helper/" }
+xvc-test-helper = { version = "0.6.8-alpha.2", path = "../test_helper/" }
 proptest = "^1.4"
 test-case = "^3.3"
diff --git a/core/src/types/xvcroot.rs b/core/src/types/xvcroot.rs
@@ -10,6 +10,7 @@ use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use xvc_ecs::ecs::timestamp;
 use xvc_ecs::{XvcEntity, XvcEntityGenerator};
+use xvc_logging::watch;
 use xvc_walker::AbsolutePath;
 
 use xvc_config::{XvcConfig, XvcConfigInitParams};
@@ -71,15 +72,12 @@ impl Deref for XvcRootInner {
 pub fn load_xvc_root(config_opts: XvcConfigInitParams) -> Result<XvcRoot> {
     let path = config_opts.current_dir.as_ref();
 
-    dbg!(path);
-
     match XvcRootInner::find_root(path) {
         Ok(absolute_path) => {
-            dbg!(&absolute_path);
             Ok(Arc::new(XvcRootInner::new(absolute_path, config_opts)?))
         }
         Err(e) => {
-            dbg!(&e);
+            watch!(&e);
             Err(e)
         }
     }
@@ -154,11 +152,8 @@ impl XvcRootInner {
     /// other locations. `config_opts` can determine which configuration files to read
     pub fn new(absolute_path: AbsolutePath, config_opts: XvcConfigInitParams) -> Result<Self> {
         let xvc_dir = absolute_path.join(XvcRootInner::XVC_DIR);
-        dbg!(&xvc_dir);
         let local_config_path = xvc_dir.join(XvcRootInner::LOCAL_CONFIG_PATH);
-        dbg!(&local_config_path);
         let project_config_path = xvc_dir.join(XvcRootInner::PROJECT_CONFIG_PATH);
-        dbg!(&project_config_path);
         let config_opts = XvcConfigInitParams {
             project_config_path: Some(project_config_path.clone()),
             local_config_path: Some(local_config_path.clone()),
@@ -259,18 +254,15 @@ impl XvcRootInner {
     /// Finds the root of the xvc repository by looking for the .xvc directory
     /// in parents of a given path.
     pub fn find_root(path: &Path) -> Result<AbsolutePath> {
-        dbg!(path);
         let abs_path = PathBuf::from(path)
             .canonicalize()
             .expect("Cannot canonicalize the path. Possible symlink loop.");
-        dbg!(&abs_path);
 
         for parent in abs_path.ancestors() {
-            dbg!(parent);
             let xvc_candidate = parent.join(XVC_DIR);
-            dbg!(&xvc_candidate);
+            watch!(xvc_candidate);
             if parent.join(XVC_DIR).is_dir() {
-                dbg!("XVC DIR: {:?}", parent);
+                watch!(parent);
                 return Ok(parent.into());
             }
         }

diff --git a/ecs/Cargo.toml b/ecs/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "xvc-ecs"
-version = "0.6.8-alpha.1"
+version = "0.6.8-alpha.2"
 edition = "2021"
 description = "Entity-Component System for Xvc"
 authors = ["Emre Şahin <[email protected]>"]
@@ -19,7 +19,7 @@ crate-type = ["rlib"]
 debug = true
 
 [dependencies]
-xvc-logging = { version = "0.6.8-alpha.1", path = "../logging" }
+xvc-logging = { version = "0.6.8-alpha.2", path = "../logging" }
 
 ## Serialization
 serde = { version = "^1.0", features = ["derive"] }

diff --git a/file/Cargo.toml b/file/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "xvc-file"
-version = "0.6.8-alpha.1"
+version = "0.6.8-alpha.2"
 edition = "2021"
 description = "File tracking, versioning, upload and download functions for Xvc"
 authors = ["Emre Şahin <[email protected]>"]
@@ -24,12 +24,12 @@ bench = true
 debug = true
 
 [dependencies]
-xvc-logging = { version = "0.6.8-alpha.1", path = "../logging" }
-xvc-config = { version = "0.6.8-alpha.1", path = "../config" }
-xvc-core = { version = "0.6.8-alpha.1", path = "../core" }
-xvc-ecs = { version = "0.6.8-alpha.1", path = "../ecs" }
-xvc-walker = { version = "0.6.8-alpha.1", path = "../walker" }
-xvc-storage = { version = "0.6.8-alpha.1", path = "../storage" }
+xvc-logging = { version = "0.6.8-alpha.2", path = "../logging" }
+xvc-config = { version = "0.6.8-alpha.2", path = "../config" }
+xvc-core = { version = "0.6.8-alpha.2", path = "../core" }
+xvc-ecs = { version = "0.6.8-alpha.2", path = "../ecs" }
+xvc-walker = { version = "0.6.8-alpha.2", path = "../walker" }
+xvc-storage = { version = "0.6.8-alpha.2", path = "../storage" }
 
 
 ## Cli and config
@@ -94,5 +94,5 @@ parse-size = "^1.0"
 
 
 [dev-dependencies]
-xvc-test-helper = { version = "0.6.8-alpha.1", path = "../test_helper/" }
+xvc-test-helper = { version = "0.6.8-alpha.2", path = "../test_helper/" }
 shellfn = "^0.1"
diff --git a/file/src/list/mod.rs b/file/src/list/mod.rs
@@ -127,7 +127,6 @@ struct ListRow {
 }
 
 impl ListRow {
-    ///
     fn new(path_prefix: &Path, path_match: PathMatch) -> Result<Self> {
         let actual_file_type =
             String::from(if let Some(actual_metadata) = path_match.actual_metadata {

diff --git a/file/src/send/mod.rs b/file/src/send/mod.rs
@@ -7,8 +7,6 @@ use crate::Result;
 
 use clap::Parser;
 
-use humantime;
-
 use xvc_core::{ContentDigest, XvcCachePath, XvcFileType, XvcMetadata, XvcRoot};
 use xvc_ecs::{HStore, XvcStore};
 use xvc_logging::{error, watch, XvcOutputSender};
@@ -25,7 +23,7 @@ use xvc_storage::{storage::get_storage_record, StorageIdentifier, XvcStorageOper
 pub struct SendCLI {
     /// Storage name or guid to send the files
     #[arg(long, short, alias = "to")]
-    remote: StorageIdentifier,
+    storage: StorageIdentifier,
     /// Force even if the files are already present in the storage
     #[arg(long)]
     force: bool,
@@ -36,7 +34,7 @@ pub struct SendCLI {
 
 /// Send a targets in `opts.targets` in `xvc_root`  to `opt.remote`
 pub fn cmd_send(output_snd: &XvcOutputSender, xvc_root: &XvcRoot, opts: SendCLI) -> Result<()> {
-    let remote = get_storage_record(output_snd, xvc_root, &opts.remote)?;
+    let remote = get_storage_record(output_snd, xvc_root, &opts.storage)?;
     watch!(remote);
     let current_dir = xvc_root.config().current_dir()?;
     let targets = load_targets_from_store(xvc_root, current_dir, &opts.targets)?;

diff --git a/file/src/share/mod.rs b/file/src/share/mod.rs
@@ -17,7 +17,7 @@ use xvc_storage::{storage::get_storage_record, StorageIdentifier, XvcStorageOper
 pub struct ShareCLI {
     /// Storage name or guid to send the files
     #[arg(long, short, alias = "from")]
-    remote: StorageIdentifier,
+    storage: StorageIdentifier,
     /// Period to send the files to. You can use s, m, h, d, w suffixes.
     #[arg(long, short, default_value = "24h")]
     duration: String,
@@ -28,8 +28,8 @@ pub struct ShareCLI {
 
 pub fn cmd_share(output_snd: &XvcOutputSender, xvc_root: &XvcRoot, opts: ShareCLI) -> Result<()> {
     // TODO: TIDY UP these implementation to reuse code in other places
-    let remote = get_storage_record(output_snd, xvc_root, &opts.remote)?;
-    watch!(remote);
+    let storage = get_storage_record(output_snd, xvc_root, &opts.storage)?;
+    watch!(storage);
     let current_dir = xvc_root.config().current_dir()?;
     let targets = load_targets_from_store(xvc_root, current_dir, &Some(vec![opts.target]))?;
     watch!(targets);
@@ -66,6 +66,6 @@ pub fn cmd_share(output_snd: &XvcOutputSender, xvc_root: &XvcRoot, opts: ShareCL
 
     watch!(duration);
 
-    remote.share(output_snd, xvc_root, &cache_path, duration)?;
+    storage.share(output_snd, xvc_root, &cache_path, duration)?;
     Ok(())
 }
diff --git a/lib/Cargo.toml b/lib/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "xvc"
-version = "0.6.8-alpha.1"
+version = "0.6.8-alpha.2"
 edition = "2021"
 description = "An MLOps tool to manage data files and pipelines on top of Git"
 authors = ["Emre Şahin <[email protected]>"]
@@ -23,14 +23,14 @@ path = "src/main.rs"
 debug = true
 
 [dependencies]
-xvc-config = { version = "0.6.8-alpha.1", path = "../config" }
-xvc-core = { version = "0.6.8-alpha.1", path = "../core" }
-xvc-logging = { version = "0.6.8-alpha.1", path = "../logging" }
-xvc-ecs = { version = "0.6.8-alpha.1", path = "../ecs" }
-xvc-file = { version = "0.6.8-alpha.1", path = "../file" }
-xvc-pipeline = { version = "0.6.8-alpha.1", path = "../pipeline" }
-xvc-walker = { version = "0.6.8-alpha.1", path = "../walker" }
-xvc-storage = { version = "0.6.8-alpha.1", path = "../storage" }
+xvc-config = { version = "0.6.8-alpha.2", path = "../config" }
+xvc-core = { version = "0.6.8-alpha.2", path = "../core" }
+xvc-logging = { version = "0.6.8-alpha.2", path = "../logging" }
+xvc-ecs = { version = "0.6.8-alpha.2", path = "../ecs" }
+xvc-file = { version = "0.6.8-alpha.2", path = "../file" }
+xvc-pipeline = { version = "0.6.8-alpha.2", path = "../pipeline" }
+xvc-walker = { version = "0.6.8-alpha.2", path = "../walker" }
+xvc-storage = { version = "0.6.8-alpha.2", path = "../storage" }
 
 
 ## Cli and config
@@ -113,4 +113,4 @@ jwalk = "^0.8"
 proptest = "^1.4"
 shellfn = "^0.1"
 test-case = "^3.3"
-xvc-test-helper = { version = "0.6.8-alpha.1", path = "../test_helper/" }
+xvc-test-helper = { version = "0.6.8-alpha.2", path = "../test_helper/" }
diff --git a/logging/Cargo.toml b/logging/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "xvc-logging"
-version = "0.6.8-alpha.1"
+version = "0.6.8-alpha.2"
 edition = "2021"
 description = "Logging crate for Xvc"
 authors = ["Emre Şahin <[email protected]>"]

diff --git a/pipeline/Cargo.toml b/pipeline/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "xvc-pipeline"
-version = "0.6.8-alpha.1"
+version = "0.6.8-alpha.2"
 edition = "2021"
 description = "Xvc data pipeline management"
 authors = ["Emre Şahin <[email protected]>"]
@@ -19,12 +19,12 @@ debug = true
 
 
 [dependencies]
-xvc-config = { version = "0.6.8-alpha.1", path = "../config" }
-xvc-core = { version = "0.6.8-alpha.1", path = "../core" }
-xvc-ecs = { version = "0.6.8-alpha.1", path = "../ecs" }
-xvc-logging = { version = "0.6.8-alpha.1", path = "../logging" }
-xvc-walker = { version = "0.6.8-alpha.1", path = "../walker" }
-xvc-file = { version = "0.6.8-alpha.1", path = "../file" }
+xvc-config = { version = "0.6.8-alpha.2", path = "../config" }
+xvc-core = { version = "0.6.8-alpha.2", path = "../core" }
+xvc-ecs = { version = "0.6.8-alpha.2", path = "../ecs" }
+xvc-logging = { version = "0.6.8-alpha.2", path = "../logging" }
+xvc-walker = { version = "0.6.8-alpha.2", path = "../walker" }
+xvc-file = { version = "0.6.8-alpha.2", path = "../file" }
 
 ## Cli and config
 clap = { version = "^4.4", features = ["derive"] }
@@ -92,5 +92,5 @@ itertools = "^0.12"
 derive_more = "^0.99"
 
 [dev-dependencies]
-xvc-test-helper = { version = "0.6.8-alpha.1", path = "../test_helper/" }
+xvc-test-helper = { version = "0.6.8-alpha.2", path = "../test_helper/" }
 test-case = "^3.3"