From cf7c5552d15575eded46a07df72b6056e6ba187b Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 25 Jan 2024 12:17:32 +0100 Subject: [PATCH 1/3] Using python file to declare test array and test them --- Cargo.lock | 20 +- .../extensions/ros2-bridge/python/Cargo.toml | 3 + .../ros2-bridge/python/src/typed/mod.rs | 96 ++++++ .../ros2-bridge/python/test_utils.py | 284 ++++++++++++++++++ 4 files changed, 398 insertions(+), 5 deletions(-) create mode 100644 libraries/extensions/ros2-bridge/python/test_utils.py diff --git a/Cargo.lock b/Cargo.lock index af0242779..ddb014978 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1424,7 +1424,7 @@ dependencies = [ "notify", "serde", "serde_json", - "serde_yaml 0.9.25", + "serde_yaml 0.9.30", "termcolor", "tokio", "tokio-stream", @@ -1461,7 +1461,7 @@ dependencies = [ "once_cell", "serde", "serde-with-expand-env", - "serde_yaml 0.9.25", + "serde_yaml 0.9.30", "tokio", "tracing", "uuid", @@ -1727,6 +1727,7 @@ dependencies = [ "futures", "pyo3", "serde", + "serde_assert", ] [[package]] @@ -4976,6 +4977,15 @@ dependencies = [ "shellexpand 2.1.2", ] +[[package]] +name = "serde_assert" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92b7be0ad5a7b2eefaa5418eb141838270f1ad2d2c6e88acec3795d2425ffa97" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.195" @@ -5035,9 +5045,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.25" +version = "0.9.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" +checksum = "b1bf28c79a99f70ee1f1d83d10c875d2e70618417fda01ad1785e027579d9d38" dependencies = [ "indexmap 2.0.2", "itoa", @@ -6626,7 +6636,7 @@ dependencies = [ "num_cpus", "serde", "serde_json", - "serde_yaml 0.9.25", + "serde_yaml 0.9.30", "validated_struct", "zenoh-cfg-properties", "zenoh-core", diff --git a/libraries/extensions/ros2-bridge/python/Cargo.toml b/libraries/extensions/ros2-bridge/python/Cargo.toml index 5c80cc9b4..fd1b8627f 100644 --- a/libraries/extensions/ros2-bridge/python/Cargo.toml +++ b/libraries/extensions/ros2-bridge/python/Cargo.toml @@ -12,3 +12,6 @@ eyre = "0.6" serde = "1.0.166" arrow = { workspace = true, features = ["pyarrow"] } futures = "0.3.28" + +[dev-dependencies] +serde_assert = "0.7.1" diff --git a/libraries/extensions/ros2-bridge/python/src/typed/mod.rs b/libraries/extensions/ros2-bridge/python/src/typed/mod.rs index 2b91d08b1..c6875b7f1 100644 --- a/libraries/extensions/ros2-bridge/python/src/typed/mod.rs +++ b/libraries/extensions/ros2-bridge/python/src/typed/mod.rs @@ -22,3 +22,99 @@ pub struct TypeInfo<'a> { /// the CDR format of ROS2 does not encode struct or field /// names. const DUMMY_STRUCT_NAME: &str = "struct"; + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use crate::typed::deserialize::StructDeserializer; + use crate::typed::serialize; + use crate::typed::TypeInfo; + use crate::Ros2Context; + + use arrow::array::make_array; + use arrow::pyarrow::FromPyArrow; + use arrow::pyarrow::ToPyArrow; + + use pyo3::types::IntoPyDict; + use pyo3::types::PyDict; + use pyo3::types::PyList; + use pyo3::types::PyModule; + use pyo3::types::PyTuple; + use pyo3::Python; + use serde::de::DeserializeSeed; + use serde::Serialize; + + use serde_assert::Serializer; + use serialize::TypedValue; + + use eyre::{Context, Result}; + use serde_assert::Deserializer; + #[test] + fn test_python_array_code() -> Result<()> { + pyo3::prepare_freethreaded_python(); + let context = Ros2Context::new(None).context("Could not create a context")?; + let messages = context.messages.clone(); + let serializer = Serializer::builder().build(); + + Python::with_gil(|py| -> Result<()> { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); //.join("test_utils.py"); // Adjust this path as needed + + // Add the Python module's directory to sys.path + py.run( + "import sys; sys.path.append(str(path))", + Some([("path", path)].into_py_dict(py)), + None, + )?; + + let my_module = PyModule::import(py, "test_utils")?; + + let arrays: &PyList = my_module.getattr("TEST_ARRAYS")?.extract()?; + for array_wrapper in arrays.iter() { + let arrays: &PyTuple = array_wrapper.extract()?; + let package_name: String = arrays.get_item(0)?.extract()?; + let message_name: String = arrays.get_item(1)?.extract()?; + println!("Checking {}::{}", package_name, message_name); + let in_pyarrow = arrays.get_item(2)?; + + let array = arrow::array::ArrayData::from_pyarrow(in_pyarrow)?; + let type_info = TypeInfo { + package_name: package_name.into(), + message_name: message_name.clone().into(), + messages: messages.clone(), + }; + let typed_value = TypedValue { + value: &make_array(array.clone()), + type_info: &type_info.clone(), + }; + + let typed_deserializer = + StructDeserializer::new(std::borrow::Cow::Owned(type_info)); + let tokens = typed_value.serialize(&serializer)?; + let mut deserializer = Deserializer::builder(tokens).build(); + + let out_value = typed_deserializer + .deserialize(&mut deserializer) + .context("could not deserialize array")?; + + let out_pyarrow = out_value.to_pyarrow(py)?; + + let test_utils = PyModule::import(py, "test_utils")?; + let context = PyDict::new(py); + + context.set_item("test_utils", test_utils)?; + context.set_item("in_pyarrow", in_pyarrow)?; + context.set_item("out_pyarrow", out_pyarrow)?; + + let _ = py + .eval( + "test_utils.is_subset(in_pyarrow, out_pyarrow)", + Some(context), + None, + ) + .context("could not check if it is a subset")?; + } + Ok(()) + }) + } +} diff --git a/libraries/extensions/ros2-bridge/python/test_utils.py b/libraries/extensions/ros2-bridge/python/test_utils.py new file mode 100644 index 000000000..a0dd5e6f7 --- /dev/null +++ b/libraries/extensions/ros2-bridge/python/test_utils.py @@ -0,0 +1,284 @@ +import numpy as np +import pyarrow as pa + + +# Marker Message Example +TEST_ARRAYS = [ + ("std_msgs", "UInt8", pa.array([{"data": np.uint8(2)}])), + ( + "std_msgs", + "String", + pa.array([{"data": "hello"}]), + ), + ( + "std_msgs", + "UInt8MultiArray", + pa.array( + [ + { + "data": np.array([1, 2, 3, 4], np.uint8), + "layout": { + "dim": [ + { + "label": "a", + "size": np.uint32(10), + "stride": np.uint32(20), + } + ], + "data_offset": np.uint32(30), + }, + } + ] + ), + ), + ( + "std_msgs", + "Float32MultiArray", + pa.array( + [ + { + "data": np.array([1, 2, 3, 4], np.float32), + "layout": { + "dim": [ + { + "label": "a", + "size": np.uint32(10), + "stride": np.uint32(20), + } + ], + "data_offset": np.uint32(30), + }, + } + ] + ), + ), + ( + "visualization_msgs", + "Marker", + pa.array( + [ + { + "header": { + "frame_id": "world", # Placeholder value (String type, no numpy equivalent) + }, + "ns": "my_namespace", # Placeholder value (String type, no numpy equivalent) + "id": np.int32(1), # Numpy type + "type": np.int32(0), # Numpy type (ARROW) + "action": np.int32(0), # Numpy type (ADD) + "lifetime": { + "sec": np.int32(1), + "nanosec": np.uint32(2), + }, # Numpy type + "pose": { + "position": { + "x": np.float64(1.0), # Numpy type + "y": np.float64(2.0), # Numpy type + "z": np.float64(3.0), # Numpy type + }, + "orientation": { + "x": np.float64(0.0), # Numpy type + "y": np.float64(0.0), # Numpy type + "z": np.float64(0.0), # Numpy type + "w": np.float64(1.0), # Numpy type + }, + }, + "scale": { + "x": np.float64(1.0), # Numpy type + "y": np.float64(1.0), # Numpy type + "z": np.float64(1.0), # Numpy type + }, + "color": { + "r": np.float32(1.0), # Numpy type + "g": np.float32(0.0), # Numpy type + "b": np.float32(0.0), # Numpy type + "a": np.float32(1.0), # Numpy type (alpha) + }, + "frame_locked": False, # Boolean type, no numpy equivalent + "points": [ # Numpy array for points + { + "x": np.float64(1.0), # Numpy type + "y": np.float64(1.0), # Numpy type + "z": np.float64(1.0), # Numpy type + } + ], + "colors": [ + { + "r": np.float32(1.0), # Numpy type + "g": np.float32(1.0), # Numpy type + "b": np.float32(1.0), # Numpy type + "a": np.float32(1.0), # Numpy type (alpha) + } # Numpy array for colors + ], + "texture_resource": "", + "uv_coordinates": [{}], + "text": "", + "mesh_resource": "", + "mesh_use_embedded_materials": False, # Boolean type, no numpy equivalent + } + ] + ), + ), + ( + "visualization_msgs", + "MarkerArray", + pa.array( + [ + { + "markers": [ + { + "header": { + "frame_id": "world", # Placeholder value (String type, no numpy equivalent) + }, + "ns": "my_namespace", # Placeholder value (String type, no numpy equivalent) + "id": np.int32(1), # Numpy type + "type": np.int32(0), # Numpy type (ARROW) + "action": np.int32(0), # Numpy type (ADD) + "lifetime": { + "sec": np.int32(1), + "nanosec": np.uint32(2), + }, # Numpy type + "pose": { + "position": { + "x": np.float64(1.0), # Numpy type + "y": np.float64(2.0), # Numpy type + "z": np.float64(3.0), # Numpy type + }, + "orientation": { + "x": np.float64(0.0), # Numpy type + "y": np.float64(0.0), # Numpy type + "z": np.float64(0.0), # Numpy type + "w": np.float64(1.0), # Numpy type + }, + }, + "scale": { + "x": np.float64(1.0), # Numpy type + "y": np.float64(1.0), # Numpy type + "z": np.float64(1.0), # Numpy type + }, + "color": { + "r": np.float32(1.0), # Numpy type + "g": np.float32(0.0), # Numpy type + "b": np.float32(0.0), # Numpy type + "a": np.float32(1.0), # Numpy type (alpha) + }, + "frame_locked": False, # Boolean type, no numpy equivalent + "points": [ # Numpy array for points + { + "x": np.float64(1.0), # Numpy type + "y": np.float64(1.0), # Numpy type + "z": np.float64(1.0), # Numpy type + } + ], + "colors": [ + { + "r": np.float32(1.0), # Numpy type + "g": np.float32(1.0), # Numpy type + "b": np.float32(1.0), # Numpy type + "a": np.float32(1.0), # Numpy type (alpha) + } # Numpy array for colors + ], + "texture_resource": "", + "uv_coordinates": [{}], + "text": "", + "mesh_resource": "", + "mesh_use_embedded_materials": False, # Boolean type, no numpy equivalent + } + ] + } + ] + ), + ), + ( + "visualization_msgs", + "ImageMarker", + pa.array( + [ + { + "header": { + "stamp": { + "sec": np.int32(123456), # 32-bit integer + "nanosec": np.uint32(789), # 32-bit unsigned integer + }, + "frame_id": "frame_example", + }, + "ns": "namespace", + "id": np.int32(1), # 32-bit integer + "type": np.int32(0), # 32-bit integer, e.g., CIRCLE = 0 + "action": np.int32(0), # 32-bit integer, e.g., ADD = 0 + "position": { + "x": np.float64(1.0), # 32-bit float + "y": np.float64(2.0), # 32-bit float + "z": np.float64(3.0), # 32-bit float + }, + "scale": np.float32(1.0), # 32-bit float + "outline_color": { + "r": np.float32(255.0), # 32-bit float + "g": np.float32(0.0), # 32-bit float + "b": np.float32(0.0), # 32-bit float + "a": np.float32(1.0), # 32-bit float + }, + "filled": np.uint8(1), # 8-bit unsigned integer + "fill_color": { + "r": np.float32(0.0), # 32-bit float + "g": np.float32(255.0), # 32-bit float + "b": np.float32(0.0), # 32-bit float + "a": np.float32(1.0), # 32-bit float + }, + "lifetime": { + "sec": np.int32(300), # 32-bit integer + "nanosec": np.uint32(0), # 32-bit unsigned integer + }, + "points": [ + { + "x": np.float64(1.0), # 32-bit float + "y": np.float64(2.0), # 32-bit float + "z": np.float64(3.0), # 32-bit float + }, + { + "x": np.float64(4.0), # 32-bit float + "y": np.float64(5.0), # 32-bit float + "z": np.float64(6.0), # 32-bit float + }, + ], + "outline_colors": [ + { + "r": np.float32(255.0), # 32-bit float + "g": np.float32(0.0), # 32-bit float + "b": np.float32(0.0), # 32-bit float + "a": np.float32(1.0), # 32-bit float + }, + { + "r": np.float32(0.0), # 32-bit float + "g": np.float32(255.0), # 32-bit float + "b": np.float32(0.0), # 32-bit float + "a": np.float32(1.0), # 32-bit float + }, + ], + } + ] + ), + ), +] + + +def is_subset(subset, superset): + """ + Check if subset is a subset of superset, to avoid false negatives linked to default values. + """ + if isinstance(subset, pa.Array): + return is_subset(subset.to_pylist(), superset.to_pylist()) + + match subset: + case dict(_): + return all( + key in superset and is_subset(val, superset[key]) + for key, val in subset.items() + ) + case list(_) | set(_): + return all( + any(is_subset(subitem, superitem) for superitem in superset) + for subitem in subset + ) + # assume that subset is a plain value if none of the above match + case _: + return subset == superset From c9e4a696a3b9a1422fed34fbb70a04247ade0b82 Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 25 Jan 2024 12:23:48 +0100 Subject: [PATCH 2/3] Adding numpy and pyarrow dependencies --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 207a5c12f..345edecad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,8 @@ jobs: run: cargo check --all - name: "Build (Without Python node as it is build with maturin)" run: cargo build --all --exclude dora-node-api-python + - name: "Install pyarrow for testing" + run: pip install numpy pyarrow - name: "Test" run: cargo test --all From aa2ffafba18c033b7e48b4be0a6c068365434cba Mon Sep 17 00:00:00 2001 From: haixuanTao Date: Thu, 25 Jan 2024 13:58:32 +0100 Subject: [PATCH 3/3] Test ros2-bridge within the ros2 job to use the right dependencies --- .github/workflows/ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 345edecad..aad0b15a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,10 +64,8 @@ jobs: run: cargo check --all - name: "Build (Without Python node as it is build with maturin)" run: cargo build --all --exclude dora-node-api-python - - name: "Install pyarrow for testing" - run: pip install numpy pyarrow - name: "Test" - run: cargo test --all + run: cargo test --all --exclude dora-ros2-bridge-python # Run examples as separate job because otherwise we will exhaust the disk # space of the GitHub action runners. @@ -166,6 +164,10 @@ jobs: with: required-ros-distributions: humble - run: 'source /opt/ros/humble/setup.bash && echo AMENT_PREFIX_PATH=${AMENT_PREFIX_PATH} >> "$GITHUB_ENV"' + - name: "Install pyarrow for testing" + run: pip install numpy pyarrow + - name: "Test" + run: cargo test -p dora-ros2-bridge-python - name: "Rust ROS2 Bridge example" timeout-minutes: 30 env: