From ab701e17b80c1769b05431511e63aad2eea3cbd2 Mon Sep 17 00:00:00 2001 From: "denis.smirnov" Date: Tue, 7 Sep 2021 08:12:10 -0500 Subject: [PATCH 1/9] Add methods __bool__, __float__ and __int__ to usm_ndarray --- dpctl/tensor/_usmarray.pyx | 30 ++++++++++++++++++++++++++++ dpctl/tests/test_usm_ndarray_ctor.py | 26 ++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index 4b3afd0964..7ea8144d2f 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -491,6 +491,36 @@ cdef class usm_ndarray: res.flags_ |= (self.flags_ & USM_ARRAY_WRITEABLE) return res + def __bool__(self): + if self.size == 1: + return self.usm_data.copy_to_host().view(self.dtype).__bool__() + + if self.size == 0: + raise ValueError( + "The truth value of an empty array is ambiguous" + ) + + raise ValueError( + "The truth value of an array with more than one element is " + "ambiguous. Use a.any() or a.all()" + ) + + def __float__(self): + if self.size == 1: + return self.usm_data.copy_to_host().view(self.dtype).__float__() + + raise ValueError( + "only size-1 arrays can be converted to Python scalars" + ) + + def __int__(self): + if self.size == 1: + return self.usm_data.copy_to_host().view(self.dtype).__int__() + + raise ValueError( + "only size-1 arrays can be converted to Python scalars" + ) + def to_device(self, target_device): """ Transfer array to target device diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index 6493a28fe8..db13611912 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -114,6 +114,32 @@ def test_properties(): assert isinstance(X.ndim, numbers.Integral) +@pytest.mark.parametrize("func", [bool, float, int]) +@pytest.mark.parametrize("shape", [(1,), (1, 1), (1, 1, 1)]) +def test_copy_scalar_with_func(func, shape): + X = dpt.usm_ndarray(shape) + Y = np.arange(1, X.size + 1, dtype=X.dtype) + X.usm_data.copy_from_host(Y.view("|u1")) + assert func(X) == func(Y) + + +@pytest.mark.parametrize("method", ["__bool__", "__float__", "__int__"]) +@pytest.mark.parametrize("shape", [(1,), (1, 1), (1, 1, 1)]) +def test_copy_scalar_with_method(method, shape): + X = dpt.usm_ndarray(shape) + Y = np.arange(1, X.size + 1, dtype=X.dtype) + X.usm_data.copy_from_host(Y.view("|u1")) + assert getattr(X, method)() == getattr(Y, method)() + + +@pytest.mark.parametrize("func", [bool, float, int]) +@pytest.mark.parametrize("shape", [(2,), (1, 2), (3, 4, 5), (0,)]) +def test_copy_scalar_invalid_shape(func, shape): + X = dpt.usm_ndarray(shape) + with pytest.raises(ValueError): + func(X) + + @pytest.mark.parametrize( "ind", [ From cf9935c813e9b21084d8608ac3a4efb65819f754 Mon Sep 17 00:00:00 2001 From: "denis.smirnov" Date: Wed, 8 Sep 2021 11:25:51 -0500 Subject: [PATCH 2/9] Add __index__ to usm_ndarray --- dpctl/tensor/_usmarray.pyx | 6 ++++ dpctl/tests/test_usm_ndarray_ctor.py | 46 +++++++++++++++++++++++----- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index 7ea8144d2f..d11d02a8ce 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -521,6 +521,12 @@ cdef class usm_ndarray: "only size-1 arrays can be converted to Python scalars" ) + def __index__(self): + if np.issubdtype(self.dtype, np.integer): + return int(self) + + raise IndexError("only integer or boolean arrays are valid indices") + def to_device(self, target_device): """ Transfer array to target device diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index db13611912..cef3d102d1 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -116,19 +116,21 @@ def test_properties(): @pytest.mark.parametrize("func", [bool, float, int]) @pytest.mark.parametrize("shape", [(1,), (1, 1), (1, 1, 1)]) -def test_copy_scalar_with_func(func, shape): - X = dpt.usm_ndarray(shape) - Y = np.arange(1, X.size + 1, dtype=X.dtype) - X.usm_data.copy_from_host(Y.view("|u1")) +@pytest.mark.parametrize("dtype", ["|b1", "|f8", "|i8"]) +def test_copy_scalar_with_func(func, shape, dtype): + X = dpt.usm_ndarray(shape, dtype=dtype) + Y = np.arange(1, X.size + 1, dtype=dtype).reshape(shape) + X.usm_data.copy_from_host(Y.reshape(-1).view("|u1")) assert func(X) == func(Y) @pytest.mark.parametrize("method", ["__bool__", "__float__", "__int__"]) @pytest.mark.parametrize("shape", [(1,), (1, 1), (1, 1, 1)]) -def test_copy_scalar_with_method(method, shape): - X = dpt.usm_ndarray(shape) - Y = np.arange(1, X.size + 1, dtype=X.dtype) - X.usm_data.copy_from_host(Y.view("|u1")) +@pytest.mark.parametrize("dtype", ["|b1", "|f8", "|i8"]) +def test_copy_scalar_with_method(method, shape, dtype): + X = dpt.usm_ndarray(shape, dtype=dtype) + Y = np.arange(1, X.size + 1, dtype=dtype).reshape(shape) + X.usm_data.copy_from_host(Y.reshape(-1).view("|u1")) assert getattr(X, method)() == getattr(Y, method)() @@ -140,6 +142,34 @@ def test_copy_scalar_invalid_shape(func, shape): func(X) +@pytest.mark.parametrize("shape", [(1,), (1, 1), (1, 1, 1)]) +@pytest.mark.parametrize("index_dtype", ["|i8"]) +def test_usm_ndarray_as_index(shape, index_dtype): + X = dpt.usm_ndarray(shape, dtype=index_dtype) + Xnp = np.arange(1, X.size + 1, dtype=index_dtype).reshape(shape) + X.usm_data.copy_from_host(Xnp.reshape(-1).view("|u1")) + Y = np.arange(X.size + 1) + assert Y[X] == Y[1] + + +@pytest.mark.parametrize("shape", [(2,), (1, 2), (3, 4, 5), (0,)]) +@pytest.mark.parametrize("index_dtype", ["|i8"]) +def test_usm_ndarray_as_index_invalid_shape(shape, index_dtype): + X = dpt.usm_ndarray(shape, dtype=index_dtype) + Y = np.arange(X.size + 1) + with pytest.raises(IndexError): + Y[X] + + +@pytest.mark.parametrize("shape", [(1,), (1, 1), (1, 1, 1)]) +@pytest.mark.parametrize("index_dtype", ["|f8"]) +def test_usm_ndarray_as_index_invalid_dtype(shape, index_dtype): + X = dpt.usm_ndarray(shape, dtype=index_dtype) + Y = np.arange(X.size + 1) + with pytest.raises(IndexError): + Y[X] + + @pytest.mark.parametrize( "ind", [ From e849d83bf35551eabaf29b469215a40b37b3b8dc Mon Sep 17 00:00:00 2001 From: "denis.smirnov" Date: Wed, 8 Sep 2021 11:27:20 -0500 Subject: [PATCH 3/9] Fix exception message in usm_ndarray.__index__ --- dpctl/tensor/_usmarray.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index d11d02a8ce..f79306b075 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -525,7 +525,7 @@ cdef class usm_ndarray: if np.issubdtype(self.dtype, np.integer): return int(self) - raise IndexError("only integer or boolean arrays are valid indices") + raise IndexError("only integer arrays are valid indices") def to_device(self, target_device): """ From 5637ff257976f2c479f5dfae444e9f52d0559c78 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 12 Sep 2021 20:21:53 -0500 Subject: [PATCH 4/9] Use dpctl.memory.as_usm_memory in __float__, etc. --- dpctl/tensor/_usmarray.pyx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index f79306b075..2191575c17 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -493,7 +493,8 @@ cdef class usm_ndarray: def __bool__(self): if self.size == 1: - return self.usm_data.copy_to_host().view(self.dtype).__bool__() + mem_view = dpmem.as_usm_memory(self) + return mem_view.copy_to_host().view(self.dtype).__bool__() if self.size == 0: raise ValueError( @@ -507,7 +508,8 @@ cdef class usm_ndarray: def __float__(self): if self.size == 1: - return self.usm_data.copy_to_host().view(self.dtype).__float__() + mem_view = dpmem.as_usm_memory(self) + return mem_view.copy_to_host().view(self.dtype).__float__() raise ValueError( "only size-1 arrays can be converted to Python scalars" @@ -515,7 +517,8 @@ cdef class usm_ndarray: def __int__(self): if self.size == 1: - return self.usm_data.copy_to_host().view(self.dtype).__int__() + mem_view = dpmem.as_usm_memory(self) + return mem_view.copy_to_host().view(self.dtype).__int__() raise ValueError( "only size-1 arrays can be converted to Python scalars" From 71fc06a4f5ee2d38367f167ab2b2d794a469b9cc Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 12 Sep 2021 20:22:37 -0500 Subject: [PATCH 5/9] Fixed dtype validation in _USMBufferData auxiliary class np.number comprises np.integer and np.inexact, so replace issubdtype(dt.type, np.inexact) or issubdtype(dt.type, np.number) with more efficient issubdtype(dt.type, np.number). Also allowed dtype.type to be np.bool_, to accommodate dtype="|b1" Replaced single quotation marks with double quotation marks per linter's flavor. --- dpctl/memory/_sycl_usm_array_interface_utils.pxi | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dpctl/memory/_sycl_usm_array_interface_utils.pxi b/dpctl/memory/_sycl_usm_array_interface_utils.pxi index 446515e5d4..bdd97ad080 100644 --- a/dpctl/memory/_sycl_usm_array_interface_utils.pxi +++ b/dpctl/memory/_sycl_usm_array_interface_utils.pxi @@ -2,7 +2,7 @@ cdef bint _valid_usm_ptr_and_context(DPCTLSyclUSMRef ptr, SyclContext ctx): usm_type = _Memory.get_pointer_type(ptr, ctx) - return usm_type in (b'shared', b'device', b'host') + return usm_type in (b"shared", b"device", b"host") cdef DPCTLSyclQueueRef _queue_ref_copy_from_SyclQueue( @@ -49,7 +49,7 @@ cdef DPCTLSyclQueueRef get_queue_ref_from_ptr_and_syclobj( elif pycapsule.PyCapsule_IsValid(syclobj, "SyclContextRef"): ctx = SyclContext(syclobj) return _queue_ref_copy_from_USMRef_and_SyclContext(ptr, ctx) - elif hasattr(syclobj, '_get_capsule'): + elif hasattr(syclobj, "_get_capsule"): cap = syclobj._get_capsule() if pycapsule.PyCapsule_IsValid(cap, "SyclQueueRef"): q = SyclQueue(cap) @@ -166,8 +166,8 @@ cdef class _USMBufferData: nd = len(ary_shape) try: dt = np.dtype(ary_typestr) - if (dt.hasobject or not (np.issubdtype(dt.type, np.integer) or - np.issubdtype(dt.type, np.inexact))): + if (dt.hasobject or not (np.issubdtype(dt.type, np.number) or + dt.type is np.bool_)): DPCTLQueue_Delete(QRef) raise TypeError("Only integer types, floating and complex " "floating types are supported.") From e32b395aff641b997714369d919b17f6d76b19f8 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 12 Sep 2021 20:25:09 -0500 Subject: [PATCH 6/9] Expanded testing of scalar copy in dtypes --- dpctl/tests/test_usm_ndarray_ctor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index cef3d102d1..9dbcaa8ee1 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -116,7 +116,7 @@ def test_properties(): @pytest.mark.parametrize("func", [bool, float, int]) @pytest.mark.parametrize("shape", [(1,), (1, 1), (1, 1, 1)]) -@pytest.mark.parametrize("dtype", ["|b1", "|f8", "|i8"]) +@pytest.mark.parametrize("dtype", ["|b1", "|u2", "|f4", "|i8"]) def test_copy_scalar_with_func(func, shape, dtype): X = dpt.usm_ndarray(shape, dtype=dtype) Y = np.arange(1, X.size + 1, dtype=dtype).reshape(shape) @@ -126,7 +126,7 @@ def test_copy_scalar_with_func(func, shape, dtype): @pytest.mark.parametrize("method", ["__bool__", "__float__", "__int__"]) @pytest.mark.parametrize("shape", [(1,), (1, 1), (1, 1, 1)]) -@pytest.mark.parametrize("dtype", ["|b1", "|f8", "|i8"]) +@pytest.mark.parametrize("dtype", ["|b1", "|u2", "|f4", "|i8"]) def test_copy_scalar_with_method(method, shape, dtype): X = dpt.usm_ndarray(shape, dtype=dtype) Y = np.arange(1, X.size + 1, dtype=dtype).reshape(shape) From 101bcc29a506b78b632133b277bdc100d91700a8 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 12 Sep 2021 21:06:27 -0500 Subject: [PATCH 7/9] Change to support use of usm_ndarray scalars in slicing Added tests to that effect. --- dpctl/tensor/_slicing.pxi | 11 ++++++++--- dpctl/tests/test_usm_ndarray_ctor.py | 8 ++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/dpctl/tensor/_slicing.pxi b/dpctl/tensor/_slicing.pxi index 321eff0b0e..2de58e32bf 100755 --- a/dpctl/tensor/_slicing.pxi +++ b/dpctl/tensor/_slicing.pxi @@ -44,6 +44,9 @@ cdef object _basic_slice_meta(object ind, tuple shape, Raises IndexError for invalid index `ind`, and NotImplementedError if `ind` is an array. """ + is_integral = lambda x: ( + isinstance(x, numbers.Integral) or callable(getattr(x, "__index__", None)) + ) if ind is Ellipsis: return (shape, strides, offset) elif ind is None: @@ -58,7 +61,8 @@ cdef object _basic_slice_meta(object ind, tuple shape, new_strides, offset + sl_start * strides[0] ) - elif isinstance(ind, numbers.Integral): + elif is_integral(ind): + ind = ind.__index__() if 0 <= ind < shape[0]: return (shape[1:], strides[1:], offset + ind * strides[0]) elif -shape[0] <= ind < 0: @@ -82,7 +86,7 @@ cdef object _basic_slice_meta(object ind, tuple shape, ellipses_count = ellipses_count + 1 elif isinstance(i, slice): axes_referenced = axes_referenced + 1 - elif isinstance(i, numbers.Integral): + elif is_integral(i): explicit_index = explicit_index + 1 axes_referenced = axes_referenced + 1 elif isinstance(i, list): @@ -124,7 +128,8 @@ cdef object _basic_slice_meta(object ind, tuple shape, new_strides.append(str_i) new_offset = new_offset + sl_start * strides[k] k = k_new - elif isinstance(ind_i, numbers.Integral): + elif is_integral(ind_i): + ind_i = ind_i.__index__() if 0 <= ind_i < shape[k]: k_new = k + 1 new_offset = new_offset + ind_i * strides[k] diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index 9dbcaa8ee1..f6ba368386 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -307,6 +307,14 @@ def test_slicing_basic(): Xusm[:, -128] with pytest.raises(TypeError): Xusm[{1, 2, 3, 4, 5, 6, 7}] + X = dpt.usm_ndarray(10, "u1") + X.usm_data.copy_from_host(b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09") + int( + X[X[2]] + ) # check that objects with __index__ method can be used as indices + Xh = dpm.as_usm_memory(X[X[2] : X[5]]).copy_to_host() + Xnp = np.arange(0, 10, dtype="u1") + assert np.array_equal(Xh, Xnp[Xnp[2] : Xnp[5]]) def test_ctor_invalid_shape(): From 61d94d46c42874683273d5c90dc259e219154328 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 12 Sep 2021 21:11:33 -0500 Subject: [PATCH 8/9] Implemented __complex__ magic method --- dpctl/tensor/_usmarray.pyx | 9 +++++++++ dpctl/tests/test_usm_ndarray_ctor.py | 12 +++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/dpctl/tensor/_usmarray.pyx b/dpctl/tensor/_usmarray.pyx index 2191575c17..148847ba2e 100644 --- a/dpctl/tensor/_usmarray.pyx +++ b/dpctl/tensor/_usmarray.pyx @@ -515,6 +515,15 @@ cdef class usm_ndarray: "only size-1 arrays can be converted to Python scalars" ) + def __complex__(self): + if self.size == 1: + mem_view = dpmem.as_usm_memory(self) + return mem_view.copy_to_host().view(self.dtype).__complex__() + + raise ValueError( + "only size-1 arrays can be converted to Python scalars" + ) + def __int__(self): if self.size == 1: mem_view = dpmem.as_usm_memory(self) diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index f6ba368386..a994b15661 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -114,8 +114,8 @@ def test_properties(): assert isinstance(X.ndim, numbers.Integral) -@pytest.mark.parametrize("func", [bool, float, int]) -@pytest.mark.parametrize("shape", [(1,), (1, 1), (1, 1, 1)]) +@pytest.mark.parametrize("func", [bool, float, int, complex]) +@pytest.mark.parametrize("shape", [tuple(), (1,), (1, 1), (1, 1, 1)]) @pytest.mark.parametrize("dtype", ["|b1", "|u2", "|f4", "|i8"]) def test_copy_scalar_with_func(func, shape, dtype): X = dpt.usm_ndarray(shape, dtype=dtype) @@ -124,8 +124,10 @@ def test_copy_scalar_with_func(func, shape, dtype): assert func(X) == func(Y) -@pytest.mark.parametrize("method", ["__bool__", "__float__", "__int__"]) -@pytest.mark.parametrize("shape", [(1,), (1, 1), (1, 1, 1)]) +@pytest.mark.parametrize( + "method", ["__bool__", "__float__", "__int__", "__complex__"] +) +@pytest.mark.parametrize("shape", [tuple(), (1,), (1, 1), (1, 1, 1)]) @pytest.mark.parametrize("dtype", ["|b1", "|u2", "|f4", "|i8"]) def test_copy_scalar_with_method(method, shape, dtype): X = dpt.usm_ndarray(shape, dtype=dtype) @@ -134,7 +136,7 @@ def test_copy_scalar_with_method(method, shape, dtype): assert getattr(X, method)() == getattr(Y, method)() -@pytest.mark.parametrize("func", [bool, float, int]) +@pytest.mark.parametrize("func", [bool, float, int, complex]) @pytest.mark.parametrize("shape", [(2,), (1, 2), (3, 4, 5), (0,)]) def test_copy_scalar_invalid_shape(func, shape): X = dpt.usm_ndarray(shape) From 1f34ee8b505305d767d818dd7eddfb907c74ae8e Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 12 Sep 2021 21:56:46 -0500 Subject: [PATCH 9/9] Added a test to cover Device class (data-API style) --- dpctl/tests/test_usm_ndarray_ctor.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dpctl/tests/test_usm_ndarray_ctor.py b/dpctl/tests/test_usm_ndarray_ctor.py index a994b15661..7bb71da149 100644 --- a/dpctl/tests/test_usm_ndarray_ctor.py +++ b/dpctl/tests/test_usm_ndarray_ctor.py @@ -357,3 +357,19 @@ def test_usm_ndarray_props(): except dpctl.SyclQueueCreationError: pytest.skip("Sycl device CPU was not detected") Xusm.to_device("cpu") + + +def test_datapi_device(): + X = dpt.usm_ndarray(1) + dev_t = type(X.device) + with pytest.raises(TypeError): + dev_t() + dev_t.create_device(X.device) + dev_t.create_device(X.sycl_queue) + dev_t.create_device(X.sycl_device) + dev_t.create_device(X.sycl_device.filter_string) + dev_t.create_device(None) + X.device.sycl_context + X.device.sycl_queue + X.device.sycl_device + repr(X.device)