Move more types from dora-core to dora-message to avoid dependency

Make `dora-message` a dependency of `dora-core`, instead of the other way around. This way, we can continue to freely bump the version of `dora-core` with the other workspace crates, without introducing errors such as #708.
dora-rs · Nov 13, 2024 · 7c44e7a · 7c44e7a
1 parent 2f5945f
commit 7c44e7a
Show file tree

Hide file tree

Showing 29 changed files with 709 additions and 648 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/binaries/cli/src/attach.rs b/binaries/cli/src/attach.rs
@@ -1,6 +1,6 @@
 use colored::Colorize;
 use communication_layer_request_reply::{TcpConnection, TcpRequestReplyConnection};
-use dora_core::descriptor::{resolve_path, CoreNodeKind, Descriptor};
+use dora_core::descriptor::{resolve_path, CoreNodeKind, Descriptor, DescriptorExt};
 use dora_message::cli_to_coordinator::ControlRequest;
 use dora_message::common::LogMessage;
 use dora_message::coordinator_to_cli::ControlRequestReply;

diff --git a/binaries/cli/src/build.rs b/binaries/cli/src/build.rs
@@ -1,6 +1,6 @@
 use dora_core::{
     config::OperatorId,
-    descriptor::{Descriptor, SINGLE_OPERATOR_DEFAULT_ID},
+    descriptor::{Descriptor, DescriptorExt, NodeExt, SINGLE_OPERATOR_DEFAULT_ID},
 };
 use eyre::{eyre, Context};
 use std::{path::Path, process::Command};

diff --git a/binaries/cli/src/graph/mod.rs b/binaries/cli/src/graph/mod.rs
@@ -1,6 +1,6 @@
 use std::{fs::File, io::Write, path::Path};
 
-use dora_core::descriptor::Descriptor;
+use dora_core::descriptor::{Descriptor, DescriptorExt};
 use eyre::Context;
 
 const MERMAID_TEMPLATE: &str = include_str!("mermaid-template.html");

diff --git a/binaries/cli/src/lib.rs b/binaries/cli/src/lib.rs
@@ -3,7 +3,7 @@ use colored::Colorize;
 use communication_layer_request_reply::{RequestReplyLayer, TcpLayer, TcpRequestReplyConnection};
 use dora_coordinator::Event;
 use dora_core::{
-    descriptor::{source_is_url, Descriptor},
+    descriptor::{source_is_url, Descriptor, DescriptorExt},
     topics::{
         DORA_COORDINATOR_PORT_CONTROL_DEFAULT, DORA_COORDINATOR_PORT_DEFAULT,
         DORA_DAEMON_LOCAL_LISTEN_PORT_DEFAULT,

diff --git a/binaries/coordinator/src/lib.rs b/binaries/coordinator/src/lib.rs
@@ -5,7 +5,6 @@ use crate::{
 pub use control::ControlEvent;
 use dora_core::{
     config::{NodeId, OperatorId},
-    descriptor::{Descriptor, ResolvedNode},
     uhlc::{self, HLC},
 };
 use dora_message::{
@@ -16,6 +15,7 @@ use dora_message::{
     },
     coordinator_to_daemon::{DaemonCoordinatorEvent, RegisterResult, Timestamped},
     daemon_to_coordinator::{DaemonCoordinatorReply, DataflowDaemonResult},
+    descriptor::{Descriptor, ResolvedNode},
 };
 use eyre::{bail, eyre, ContextCompat, Result, WrapErr};
 use futures::{future::join_all, stream::FuturesUnordered, Future, Stream, StreamExt};

diff --git a/binaries/coordinator/src/run/mod.rs b/binaries/coordinator/src/run/mod.rs
@@ -3,13 +3,11 @@ use crate::{
     DaemonConnection,
 };
 
-use dora_core::{
-    descriptor::{Descriptor, ResolvedNode},
-    uhlc::HLC,
-};
+use dora_core::{descriptor::DescriptorExt, uhlc::HLC};
 use dora_message::{
     coordinator_to_daemon::{DaemonCoordinatorEvent, SpawnDataflowNodes, Timestamped},
     daemon_to_coordinator::DaemonCoordinatorReply,
+    descriptor::{Descriptor, ResolvedNode},
 };
 use eyre::{bail, eyre, ContextCompat, WrapErr};
 use std::{

diff --git a/binaries/daemon/src/lib.rs b/binaries/daemon/src/lib.rs
@@ -3,7 +3,10 @@ use coordinator::CoordinatorEvent;
 use crossbeam::queue::ArrayQueue;
 use dora_core::{
     config::{DataId, Input, InputMapping, NodeId, OperatorId},
-    descriptor::{runtime_node_inputs, CoreNodeKind, Descriptor, ResolvedNode},
+    descriptor::{
+        read_as_descriptor, runtime_node_inputs, CoreNodeKind, Descriptor, DescriptorExt,
+        ResolvedNode,
+    },
     topics::LOCALHOST,
     uhlc::{self, HLC},
 };
@@ -162,7 +165,7 @@ impl Daemon {
             .ok_or_else(|| eyre::eyre!("canonicalized dataflow path has no parent"))?
             .to_owned();
 
-        let descriptor = Descriptor::read(dataflow_path).await?;
+        let descriptor = read_as_descriptor(dataflow_path).await?;
         descriptor.check(&working_dir)?;
         let nodes = descriptor.resolve_aliases_and_set_defaults()?;
 

diff --git a/examples/multiple-daemons/run.rs b/examples/multiple-daemons/run.rs
@@ -1,6 +1,6 @@
 use dora_coordinator::{ControlEvent, Event};
 use dora_core::{
-    descriptor::Descriptor,
+    descriptor::{read_as_descriptor, DescriptorExt},
     topics::{DORA_COORDINATOR_PORT_CONTROL_DEFAULT, DORA_COORDINATOR_PORT_DEFAULT},
 };
 use dora_message::{
@@ -115,7 +115,7 @@ async fn start_dataflow(
     dataflow: &Path,
     coordinator_events_tx: &Sender<Event>,
 ) -> eyre::Result<Uuid> {
-    let dataflow_descriptor = Descriptor::read(dataflow)
+    let dataflow_descriptor = read_as_descriptor(dataflow)
         .await
         .wrap_err("failed to read yaml dataflow")?;
     let working_dir = dataflow

diff --git a/libraries/core/Cargo.toml b/libraries/core/Cargo.toml
@@ -10,6 +10,7 @@ repository.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+dora-message = { workspace = true }
 eyre = "0.6.8"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_yaml = "0.9.11"
@@ -22,4 +23,3 @@ tokio = { version = "1.24.1", features = ["fs", "process", "sync"] }
 schemars = "0.8.19"
 serde_json = "1.0.117"
 log = { version = "0.4.21", features = ["serde"] }
-uhlc = "0.5.1"
diff --git a/libraries/core/src/bin/generate_schema.rs b/libraries/core/src/bin/generate_schema.rs
@@ -1,6 +1,6 @@
 use std::{env, path::Path};
 
-use dora_core::descriptor::Descriptor;
+use dora_message::descriptor::Descriptor;
 use schemars::schema_for;
 
 fn main() {