Adding pyarrow-sender and terminal-print in the node-hub

examples/pyarrow-test/.gitignore

@@ -0,0 +1 @@
+*.pt

examples/pyarrow-test/README.md

@@ -0,0 +1,41 @@
+# Python Dataflow Example
+
+This examples shows how to create and connect dora nodes in Python.
+
+## Overview
+
+The [`dataflow.yml`](./dataflow.yml) defines a simple dataflow graph with the following three nodes:
+
+- a webcam node, that connects to your webcam and feed the dataflow with webcam frame as jpeg compressed bytearray.
+- a window plotting node, that will retrieve the webcam image and plot it.
+
+The same dataflow is implemented for a `dynamic-node` in [`dataflow_dynamic.yml`](./dataflow_dynamic.yml). It contains
+the same nodes as the previous dataflow, but the plot node is a dynamic node. See the next section for more
+information on how to start such a dataflow.
+
+## Getting started
+
+After installing Rust, `dora-cli` and `Python >3.11`, you will need to **activate** (or create and **activate**) a
+[Python virtual environment](https://docs.python.org/3/library/venv.html).
+Then, you will need to install the dependencies:
+
+```bash
+cd examples/python-dataflow
+dora build ./dataflow.yml (or dora build ./dataflow_dynamic.yml)
+```
+
+It will install the required dependencies for the Python nodes.
+
+Then you can run the dataflow:
+
+```bash
+dora up
+dora start ./dataflow.yml (or dora start ./dataflow_dynamic.yml)
+```
+
+**Note**: if you're running the dynamic dataflow, you will need to start manually the ultralytics-yolo node:
+
+```bash
+# activate your virtual environment in another terminal
+python opencv-plot --name plot
+```

examples/pyarrow-test/dataflow.yml

@@ -0,0 +1,16 @@
+nodes:
+  - id: pyarrow-sender
+    build: pip install -e ../../node-hub/pyarrow-sender
+    path: dynamic
+    outputs:
+      - data
+    env:
+      DATA: "[1, 2, 3, 4, 5]"
+
+  - id: pyarrow-assert
+    build: pip install -e ../../node-hub/pyarrow-assert
+    path: pyarrow-assert
+    inputs:
+      data: pyarrow-sender/data
+    env:
+      DATA: "pa.array([1, 2, 3, 4, 5])"

examples/terminal-dataflow/.gitignore

@@ -0,0 +1 @@
+*.pt

examples/terminal-dataflow/README.md

@@ -0,0 +1,41 @@
+# Python Dataflow Example
+
+This examples shows how to create and connect dora nodes in Python.
+
+## Overview
+
+The [`dataflow.yml`](./dataflow.yml) defines a simple dataflow graph with the following three nodes:
+
+- a webcam node, that connects to your webcam and feed the dataflow with webcam frame as jpeg compressed bytearray.
+- a window plotting node, that will retrieve the webcam image and plot it.
+
+The same dataflow is implemented for a `dynamic-node` in [`dataflow_dynamic.yml`](./dataflow_dynamic.yml). It contains
+the same nodes as the previous dataflow, but the plot node is a dynamic node. See the next section for more
+information on how to start such a dataflow.
+
+## Getting started
+
+After installing Rust, `dora-cli` and `Python >3.11`, you will need to **activate** (or create and **activate**) a
+[Python virtual environment](https://docs.python.org/3/library/venv.html).
+Then, you will need to install the dependencies:
+
+```bash
+cd examples/python-dataflow
+dora build ./dataflow.yml (or dora build ./dataflow_dynamic.yml)
+```
+
+It will install the required dependencies for the Python nodes.
+
+Then you can run the dataflow:
+
+```bash
+dora up
+dora start ./dataflow.yml (or dora start ./dataflow_dynamic.yml)
+```
+
+**Note**: if you're running the dynamic dataflow, you will need to start manually the ultralytics-yolo node:
+
+```bash
+# activate your virtual environment in another terminal
+python opencv-plot --name plot
+```

examples/terminal-dataflow/dataflow.yml

@@ -0,0 +1,12 @@
+nodes:
+  - id: pyarrow-sender
+    build: pip install -e ../../node-hub/pyarrow-sender
+    path: dynamic
+    outputs:
+      - data
+
+  - id: terminal-print
+    build: pip install -e ../../node-hub/terminal-print
+    path: dynamic
+    inputs:
+      data: pyarrow-sender/data

node-hub/pyarrow-assert/README.md

@@ -0,0 +1,3 @@
+# Dora Node for asserting arrow data.
+
+This node assert that the DATA that is specified within the environemnt variable or from `--data` argument is the same as the data received.

node-hub/pyarrow-assert/pyarrow_assert/__init__.py

@@ -0,0 +1,11 @@
+import os
+
+# Define the path to the README file relative to the package directory
+readme_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "README.md")
+
+# Read the content of the README file
+try:
+    with open(readme_path, "r", encoding="utf-8") as f:
+        __doc__ = f.read()
+except FileNotFoundError:
+    __doc__ = "README file not found."

node-hub/pyarrow-assert/pyarrow_assert/main.py

@@ -0,0 +1,49 @@
+import argparse
+import os
+
+import numpy as np
+import pyarrow as pa
+
+from dora import Node
+
+RUNNER_CI = True if os.getenv("CI") == "true" else False
+
+
+def main():
+
+    # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables.
+    parser = argparse.ArgumentParser(description="Simple arrow sender")
+
+    parser.add_argument(
+        "--name",
+        type=str,
+        required=False,
+        help="The name of the node in the dataflow.",
+        default="arrow-assert",
+    )
+    parser.add_argument(
+        "--data",
+        type=str,
+        required=False,
+        help="Arrow Data as string.",
+        default="",
+    )
+
+    args = parser.parse_args()
+
+    data = os.getenv("DATA", args.data)
+
+    node = Node(
+        args.name
+    )  # provide the name to connect to the dataflow if dynamic node
+
+    assert_data = eval(data)
+
+    for event in node:
+        if event["type"] == "INPUT":
+            value = event["value"]
+            assert value == assert_data, f"Expected {assert_data}, got {value}"
+
+
+if __name__ == "__main__":
+    main()

node-hub/pyarrow-assert/pyproject.toml

@@ -0,0 +1,28 @@
+[tool.poetry]
+name = "pyarrow-assert"
+version = "0.3.5"
+authors = [
+    "Haixuan Xavier Tao <[email protected]>",
+    "Enzo Le Van <[email protected]>",
+]
+description = "Dora Node for plotting text and bbox on image with OpenCV"
+license = "MIT License"
+homepage = "https://github.com/dora-rs/dora.git"
+documentation = "https://github.com/dora-rs/dora/blob/main/node-hub/pyarrow-assert/README.md"
+readme = "README.md"
+packages = [{ include = "pyarrow_assert" }]
+
+[tool.poetry.dependencies]
+dora-rs = "0.3.5"
+numpy = "< 2.0.0"
+pyarrow = ">= 5.0.0"
+
+[tool.poetry.scripts]
+pyarrow-assert = "pyarrow_assert.main:main"
+
+[build-system]
+requires = ["poetry-core>=1.8.0"]
+build-backend = "poetry.core.masonry.api"
+
+[project]
+readme = "README.md"

node-hub/pyarrow-assert/tests/test_opencv_plot.py

@@ -0,0 +1,2 @@
+def test_placeholder():
+    pass

node-hub/pyarrow-sender/README.md

@@ -0,0 +1,3 @@
+# Dora Node for sending arrow data.
+
+This node send DATA that is specified within the environemnt variable or from `--data` argument.

node-hub/pyarrow-sender/pyarrow_sender/__init__.py

@@ -0,0 +1,11 @@
+import os
+
+# Define the path to the README file relative to the package directory
+readme_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "README.md")
+
+# Read the content of the README file
+try:
+    with open(readme_path, "r", encoding="utf-8") as f:
+        __doc__ = f.read()
+except FileNotFoundError:
+    __doc__ = "README file not found."

node-hub/pyarrow-sender/pyarrow_sender/main.py

@@ -0,0 +1,68 @@
+import argparse
+import os
+
+import pyarrow as pa
+
+from dora import Node
+
+RUNNER_CI = True if os.getenv("CI") == "true" else False
+
+
+def main():
+
+    # Handle dynamic nodes, ask for the name of the node in the dataflow, and the same values as the ENV variables.
+    parser = argparse.ArgumentParser(description="Simple arrow sender")
+
+    parser.add_argument(
+        "--name",
+        type=str,
+        required=False,
+        help="The name of the node in the dataflow.",
+        default="pyarrow-sender",
+    )
+    parser.add_argument(
+        "--data",
+        type=str,
+        required=False,
+        help="Arrow Data as string.",
+        default=None,
+    )
+
+    args = parser.parse_args()
+
+    data = os.getenv("DATA", args.data)
+
+    node = Node(
+        args.name
+    )  # provide the name to connect to the dataflow if dynamic node
+
+    if data is None and os.getenv("DORA_NODE_CONFIG") is None:
+        while True:
+            data = input(
+                "Provide the data you want to send:  ",
+            )
+            data = eval(data)
+            if isinstance(data, list):
+                data = pa.array(data)  # initialize pyarrow array
+            elif isinstance(data, str):
+                data = pa.array([data])
+            elif isinstance(data, int):
+                data = pa.array([data])
+            elif isinstance(data, float):
+                data = pa.array([data])
+            else:
+                data = pa.array(data)  # initialize pyarrow array
+            node.send_output("data", data)
+    else:
+        data = eval(data)
+        if isinstance(data, list):
+            data = pa.array(data)  # initialize pyarrow array
+        elif isinstance(data, str):
+            data = pa.array([data])
+        else:
+            data = pa.array(data)  # initialize pyarrow array
+        node.send_output("data", data)
+
+
+if __name__ == "__main__":
+    main()

node-hub/pyarrow-sender/pyproject.toml

@@ -0,0 +1,28 @@
+[tool.poetry]
+name = "pyarrow-sender"
+version = "0.3.5"
+authors = [
+    "Haixuan Xavier Tao <[email protected]>",
+    "Enzo Le Van <[email protected]>",
+]
+description = "Dora pyarrow Sender"
+license = "MIT License"
+homepage = "https://github.com/dora-rs/dora.git"
+documentation = "https://github.com/dora-rs/dora/blob/main/node-hub/pyarrow-sender/README.md"
+readme = "README.md"
+packages = [{ include = "pyarrow_sender" }]
+
+[tool.poetry.dependencies]
+dora-rs = "0.3.5"
+numpy = "< 2.0.0"
+pyarrow = ">= 5.0.0"
+
+[tool.poetry.scripts]
+pyarrow-sender = "pyarrow_sender.main:main"
+
+[build-system]
+requires = ["poetry-core>=1.8.0"]
+build-backend = "poetry.core.masonry.api"
+
+[project]
+readme = "README.md"

node-hub/pyarrow-sender/tests/test_arrow_sender.py

@@ -0,0 +1,2 @@
+def test_placeholder():
+    pass