Version 1.1.0 (#92)

* Run test with different python versions (#65)

* Add matrix to workflow

* Change python versions list

* Change python versions list

* Add k4a versions to matrix

* Typofix

* Drop k4a from matrix

* Add dataclasses requirement for python <3.7

* Fix python 3.6 test behavior

* Fix python 3.6 test behavior

* Restore fail-fast option

* fix conversion seconds to ns

* fix conversion seconds to ns

* fix timestamp ns to us

Co-authored-by: Louis-Philippe Asselin <[email protected]>

* fix install using pip --editable --user (#67)

* Codecov support (#64)

* Codecov support

* Add badge

* Order badges

* fix capture.transformed_depth_point_cloud (#73)

* version 1.0.1

* Added transformed_ir with transform_depth_image_to_color_camera_custom functionality (#76)

* Added transform_depth_image_to_color_camera_custom functionality

* keeping things c

* add interpolation option condition as a parameter

* returned the depth image

* unpack return value if not None so avoid error

* Image timestamp support (#88)

* Support for capture images timestamps

* Support for capture images timestamps

* Add more changes

* version 1.1.0

* fix lint

* readme fix wrong example version of SDK

Co-authored-by: Ilya Gruzinov <[email protected]>
Co-authored-by: Samuel Boulanger <[email protected]>

.github/workflows/ci.yml

@@ -70,7 +70,7 @@ jobs:
           pip install -e .
       - name: Run tests
         run: |
-          make test-no-hardware
+          make test-ci
       - name: Coverage
         uses: codecov/codecov-action@v1
         with:

Makefile

@@ -36,4 +36,8 @@ test-hardware:
 	pytest  -m "device" $(TESTS)
 
 test-no-hardware:
-	pytest -m "not device" $(TESTS)
+	pytest -m "not device" $(TESTS)
+
+test-ci:
+	pytest -m "not device and not opengl" $(TESTS)
+

README.md

@@ -32,7 +32,7 @@ pip install pyk4a
 ### Windows
 
 Make sure you replace the paths in the following instructions with your own k4a sdk path.
-It is important to replace `1.2.0` with your installed version of the SDK.
+It is important to replace `1.4.1` with your installed version of the SDK.
 
 ```
 pip install pyk4a --no-use-pep517 --global-option=build_ext --global-option="-IC:\Program Files\Azure Kinect SDK v1.4.1\sdk\include" --global-option="-LC:\Program Files\Azure Kinect SDK v1.4.1\sdk\windows-desktop\amd64\release\lib"

example/viewer_transformation.py

@@ -21,6 +21,8 @@ def main():
             cv2.imshow("Transformed Depth", colorize(capture.transformed_depth, (None, 5000)))
         if capture.transformed_color is not None:
             cv2.imshow("Transformed Color", capture.transformed_color)
+        if capture.transformed_ir is not None:
+            cv2.imshow("Transformed IR", colorize(capture.transformed_ir, (None, 500), colormap=cv2.COLORMAP_JET))
 
         key = cv2.waitKey(10)
         if key != -1:

pyk4a/__init__.py

@@ -13,7 +13,12 @@
 from .errors import K4AException, K4ATimeoutException
 from .playback import PyK4APlayback, SeekOrigin
 from .pyk4a import ColorControlCapabilities, PyK4A
-from .transformation import color_image_to_depth_camera, depth_image_to_color_camera, depth_image_to_point_cloud
+from .transformation import (
+    color_image_to_depth_camera,
+    depth_image_to_color_camera,
+    depth_image_to_color_camera_custom,
+    depth_image_to_point_cloud,
+)
 
 
 __all__ = (
@@ -37,4 +42,5 @@
     "color_image_to_depth_camera",
     "depth_image_to_point_cloud",
     "depth_image_to_color_camera",
+    "depth_image_to_color_camera_custom",
 )

pyk4a/capture.py

@@ -6,7 +6,12 @@
 
 from .calibration import Calibration
 from .config import ImageFormat
-from .transformation import color_image_to_depth_camera, depth_image_to_color_camera, depth_image_to_point_cloud
+from .transformation import (
+    color_image_to_depth_camera,
+    depth_image_to_color_camera,
+    depth_image_to_color_camera_custom,
+    depth_image_to_point_cloud,
+)
 
 
 class PyK4ACapture:
@@ -19,31 +24,60 @@ def __init__(
         self._color_format = color_format
 
         self._color: Optional[np.ndarray] = None
+        self._color_timestamp_usec: int = 0
         self._depth: Optional[np.ndarray] = None
+        self._depth_timestamp_usec: int = 0
         self._ir: Optional[np.ndarray] = None
+        self._ir_timestamp_usec: int = 0
         self._depth_point_cloud: Optional[np.ndarray] = None
         self._transformed_depth: Optional[np.ndarray] = None
         self._transformed_depth_point_cloud: Optional[np.ndarray] = None
         self._transformed_color: Optional[np.ndarray] = None
+        self._transformed_ir: Optional[np.ndarray] = None
 
     @property
     def color(self) -> Optional[np.ndarray]:
         if self._color is None:
-            self._color = k4a_module.capture_get_color_image(self._capture_handle, self.thread_safe)
+            self._color, self._color_timestamp_usec = k4a_module.capture_get_color_image(
+                self._capture_handle, self.thread_safe
+            )
         return self._color
 
+    @property
+    def color_timestamp_usec(self) -> int:
+        """Device timestamp for color image. Not equal host machine timestamp!"""
+        if self._color is None:
+            self.color
+        return self._color_timestamp_usec
+
     @property
     def depth(self) -> Optional[np.ndarray]:
         if self._depth is None:
-            self._depth = k4a_module.capture_get_depth_image(self._capture_handle, self.thread_safe)
+            self._depth, self._depth_timestamp_usec = k4a_module.capture_get_depth_image(
+                self._capture_handle, self.thread_safe
+            )
         return self._depth
 
+    @property
+    def depth_timestamp_usec(self) -> int:
+        """Device timestamp for depth image. Not equal host machine timestamp!. Like as equal IR image timestamp"""
+        if self._depth is None:
+            self.depth
+        return self._depth_timestamp_usec
+
     @property
     def ir(self) -> Optional[np.ndarray]:
+        """Device timestamp for IR image. Not equal host machine timestamp!. Like as equal depth image timestamp"""
         if self._ir is None:
-            self._ir = k4a_module.capture_get_ir_image(self._capture_handle, self.thread_safe)
+            self._ir, self._ir_timestamp_usec = k4a_module.capture_get_ir_image(self._capture_handle, self.thread_safe)
         return self._ir
 
+    @property
+    def ir_timestamp_usec(self) -> int:
+        if self._ir is None:
+            self.ir
+        return self._ir_timestamp_usec
+
     @property
     def transformed_depth(self) -> Optional[np.ndarray]:
         if self._transformed_depth is None and self.depth is not None:
@@ -78,3 +112,13 @@ def transformed_color(self) -> Optional[np.ndarray]:
                 self.color, self.depth, self._calibration, self.thread_safe
             )
         return self._transformed_color
+
+    @property
+    def transformed_ir(self) -> Optional[np.ndarray]:
+        if self._transformed_ir is None and self.ir is not None and self.depth is not None:
+            result = depth_image_to_color_camera_custom(self.depth, self.ir, self._calibration, self.thread_safe)
+            if result is None:
+                return None
+            else:
+                self._transformed_ir, self._transformed_depth = result
+        return self._transformed_ir

pyk4a/pyk4a.cpp

@@ -467,6 +467,7 @@ extern "C" {
                 dims[2] = 1;
                 *img_dst = (PyArrayObject*) PyArray_SimpleNewFromData(3, dims, NPY_UINT8, buffer);
                 break;
+            case K4A_IMAGE_FORMAT_CUSTOM16:
             case K4A_IMAGE_FORMAT_DEPTH16:
             case K4A_IMAGE_FORMAT_IR16:
                 dims[0] = k4a_image_get_height_pixels(*img_src);
@@ -506,6 +507,9 @@ extern "C" {
             case K4A_IMAGE_FORMAT_COLOR_BGRA32:
                 pixel_size = (int)sizeof(uint32_t);
                 break;
+            case K4A_IMAGE_FORMAT_CUSTOM16:
+                pixel_size = (unsigned int)sizeof(int16_t);
+                break;
             default:
                 // Not supported
                 return K4A_RESULT_FAILED;
@@ -596,6 +600,89 @@ extern "C" {
         }
     }
 
+    static PyObject* transformation_depth_image_to_color_camera_custom(PyObject* self, PyObject* args){
+        k4a_transformation_t* transformation_handle;
+        PyObject *capsule;
+        int thread_safe;
+	int interp_nearest;
+        PyThreadState *thread_state;
+        k4a_result_t res;
+        PyArrayObject *d_array;
+        PyArrayObject *c_array;
+        k4a_color_resolution_t color_resolution;
+	k4a_transformation_interpolation_type_t interpolation_type;
+
+
+        PyArg_ParseTuple(args, "OpO!O!Ip", &capsule, &thread_safe, &PyArray_Type, &d_array, &PyArray_Type, &c_array, &color_resolution, &interp_nearest);
+
+	if (interp_nearest){
+             interpolation_type = K4A_TRANSFORMATION_INTERPOLATION_TYPE_NEAREST;
+	}
+	else {
+             interpolation_type = K4A_TRANSFORMATION_INTERPOLATION_TYPE_LINEAR;
+	}
+
+        transformation_handle = (k4a_transformation_t*)PyCapsule_GetPointer(capsule, CAPSULE_TRANSFORMATION_NAME);
+
+        k4a_image_t* depth_image_transformed = (k4a_image_t*) malloc(sizeof(k4a_image_t));
+        k4a_image_t* custom_image_transformed = (k4a_image_t*) malloc(sizeof(k4a_image_t));
+
+        k4a_image_t depth_image;
+        k4a_image_t custom_image;
+        res = numpy_to_k4a_image(d_array, &depth_image, K4A_IMAGE_FORMAT_DEPTH16);
+	if (K4A_RESULT_SUCCEEDED == res) {
+            res = numpy_to_k4a_image(c_array, &custom_image, K4A_IMAGE_FORMAT_CUSTOM16);
+	}
+        thread_state = _gil_release(thread_safe);
+        if (K4A_RESULT_SUCCEEDED == res) {
+            res = k4a_image_create(
+                    k4a_image_get_format(depth_image),
+                    RESOLUTION_TO_DIMS[color_resolution][0],
+                    RESOLUTION_TO_DIMS[color_resolution][1],
+                    RESOLUTION_TO_DIMS[color_resolution][0] * (int)sizeof(uint16_t),
+                    depth_image_transformed);
+        }
+        if (K4A_RESULT_SUCCEEDED == res) {
+            res = k4a_image_create(
+                    k4a_image_get_format(custom_image),
+                    RESOLUTION_TO_DIMS[color_resolution][0],
+                    RESOLUTION_TO_DIMS[color_resolution][1],
+                    RESOLUTION_TO_DIMS[color_resolution][0] * static_cast<int32_t>(sizeof(int16_t)),
+                    custom_image_transformed);
+        }
+
+        if (K4A_RESULT_SUCCEEDED == res) {
+            res = k4a_transformation_depth_image_to_color_camera_custom(
+                    *transformation_handle,
+                    depth_image,
+                    custom_image,
+		    *depth_image_transformed,
+		    *custom_image_transformed,
+		    interpolation_type,
+		    0);
+            k4a_image_release(depth_image);
+            k4a_image_release(custom_image);
+        }
+        _gil_restore(thread_state);
+        PyArrayObject* np_depth_image;
+        PyArrayObject* np_custom_image;
+        if (K4A_RESULT_SUCCEEDED == res) {
+            res = k4a_image_to_numpy(depth_image_transformed, &np_depth_image);
+        }
+        if (K4A_RESULT_SUCCEEDED == res) {
+            res = k4a_image_to_numpy(custom_image_transformed, &np_custom_image);
+        }
+
+        if (K4A_RESULT_SUCCEEDED == res) {
+            return Py_BuildValue("OO", np_custom_image, np_depth_image);
+        }
+        else {
+            free(depth_image_transformed);
+            free(custom_image_transformed);
+            return Py_BuildValue("");
+        }
+    }
+
     static PyObject* transformation_depth_image_to_point_cloud(PyObject* self, PyObject* args) {
         k4a_transformation_t* transformation_handle;
         PyObject *capsule;
@@ -706,6 +793,7 @@ extern "C" {
         k4a_capture_t* capture_handle;
         PyObject *capsule;
         int thread_safe;
+        uint64_t device_timestamp_usec = 0;
         PyThreadState *thread_state;
         k4a_result_t res = K4A_RESULT_FAILED;
 
@@ -715,7 +803,7 @@ extern "C" {
         k4a_image_t* image = (k4a_image_t*) malloc(sizeof(k4a_image_t));
         if (image == NULL) {
             fprintf(stderr, "Cannot allocate memory");
-            return Py_BuildValue("");
+            return Py_BuildValue("NK", Py_None, device_timestamp_usec);
         }
 
         thread_state = _gil_release(thread_safe);
@@ -728,18 +816,20 @@ extern "C" {
         }
 
         if (K4A_RESULT_SUCCEEDED == res) {
-            return PyArray_Return(np_image);
+            device_timestamp_usec = k4a_image_get_device_timestamp_usec(*image);
+            return Py_BuildValue("NK", np_image, device_timestamp_usec);
         }
         else {
             free(image);
-            return Py_BuildValue("");
+            return Py_BuildValue("NK", Py_None, device_timestamp_usec);
         }
     }
 
     static PyObject* capture_get_depth_image(PyObject* self, PyObject* args){
         k4a_capture_t* capture_handle;
         PyObject *capsule;
         int thread_safe;
+        uint64_t device_timestamp_usec = 0;
         PyThreadState *thread_state;
         k4a_result_t res = K4A_RESULT_FAILED;
 
@@ -749,7 +839,7 @@ extern "C" {
         k4a_image_t* image = (k4a_image_t*) malloc(sizeof(k4a_image_t));
         if (image == NULL) {
             fprintf(stderr, "Cannot allocate memory");
-            return Py_BuildValue("");
+            return Py_BuildValue("NK", Py_None, device_timestamp_usec);
         }
 
         thread_state = _gil_release(thread_safe);
@@ -762,18 +852,20 @@ extern "C" {
         }
 
         if (K4A_RESULT_SUCCEEDED == res) {
-            return PyArray_Return(np_image);
+            device_timestamp_usec = k4a_image_get_device_timestamp_usec(*image);
+            return Py_BuildValue("NK", np_image, device_timestamp_usec);
         }
         else {
             free(image);
-            return Py_BuildValue("");
+            return Py_BuildValue("NK", Py_None, device_timestamp_usec);
         }
     }
 
     static PyObject* capture_get_ir_image(PyObject* self, PyObject* args){
         k4a_capture_t* capture_handle;
         PyObject *capsule;
         int thread_safe;
+        uint64_t device_timestamp_usec = 0;
         PyThreadState *thread_state;
         k4a_result_t res = K4A_RESULT_FAILED;
 
@@ -783,7 +875,7 @@ extern "C" {
         k4a_image_t* image = (k4a_image_t*) malloc(sizeof(k4a_image_t));
         if (image == NULL) {
             fprintf(stderr, "Cannot allocate memory");
-            return Py_BuildValue("");
+            return Py_BuildValue("NK", Py_None, device_timestamp_usec);
         }
 
         thread_state = _gil_release(thread_safe);
@@ -796,11 +888,12 @@ extern "C" {
         }
 
         if (K4A_RESULT_SUCCEEDED == res) {
-            return PyArray_Return(np_image);
+            device_timestamp_usec = k4a_image_get_device_timestamp_usec(*image);
+            return Py_BuildValue("NK", np_image, device_timestamp_usec);
         }
         else {
             free(image);
-            return Py_BuildValue("");
+            return Py_BuildValue("NK", Py_None, device_timestamp_usec);
         }
     }
 
@@ -888,6 +981,7 @@ extern "C" {
         return Py_BuildValue("II(fff)", res, valid, target_point3d_mm.xyz.x, target_point3d_mm.xyz.y, target_point3d_mm.xyz.z);
     }
 
+
     static PyObject* playback_open(PyObject* self, PyObject *args) {
         int thread_safe;
         PyThreadState *thread_state;
@@ -1148,6 +1242,7 @@ extern "C" {
         {"calibration_get_from_raw", calibration_get_from_raw, METH_VARARGS, "Create new calibration handle from raw json."},
         {"transformation_create", transformation_create, METH_VARARGS, "Create transformation handle from calibration"},
         {"transformation_depth_image_to_color_camera", transformation_depth_image_to_color_camera, METH_VARARGS, "Transforms the depth map into the geometry of the color camera."},
+        {"transformation_depth_image_to_color_camera_custom", transformation_depth_image_to_color_camera_custom, METH_VARARGS, "Transforms the custom & depth map into the geometry of the color camera."},
         {"transformation_color_image_to_depth_camera", transformation_color_image_to_depth_camera, METH_VARARGS, "Transforms the color image into the geometry of the depth camera."},
         {"transformation_depth_image_to_point_cloud", transformation_depth_image_to_point_cloud, METH_VARARGS, "Transforms the depth map to a point cloud."},
         {"calibration_3d_to_3d", calibration_3d_to_3d, METH_VARARGS, "Transforms the coordinates between 2 3D systems"},

pyk4a/transformation.py

@@ -17,6 +17,18 @@ def depth_image_to_color_camera(depth: np.ndarray, calibration: Calibration, thr
     )
 
 
+def depth_image_to_color_camera_custom(
+    depth: np.ndarray, custom: np.ndarray, calibration: Calibration, thread_safe: bool, interp_nearest: bool = True,
+) -> Optional[np.ndarray]:
+    """
+    Transforms depth image and custom image to color_image space
+    Return empty result if transformation failed
+    """
+    return k4a_module.transformation_depth_image_to_color_camera_custom(
+        calibration.transformation_handle, thread_safe, depth, custom, calibration.color_resolution, interp_nearest,
+    )
+
+
 def depth_image_to_point_cloud(
     depth: np.ndarray, calibration: Calibration, thread_safe: bool, calibration_type_depth=True
 ) -> Optional[np.ndarray]:

pyproject.toml

@@ -6,6 +6,7 @@ line-length = 120
 
 [tool.pytest.ini_options]
 markers = [
-        "device: Tests require connected real device"
+        "device: Tests require connected real device",
+        "opengl: Tests require opengl for GPU accelerated depth engine software"
 ]
 addopts = "--cov=pyk4a --cov-report=xml --verbose"