Merge pull request #9 from static-frame/8/null-bytes

README.rst

@@ -36,6 +36,12 @@ ArrayMap requires the following:
 What is New in ArrayMap
 -------------------------
 
+0.1.5
+........
+
+Improved handling for Unicode elements that contain non-terminal NULL strings.
+
+
 0.1.4
 ........
 

arraymap.c

@@ -124,12 +124,14 @@ typedef enum ViewKind{
 
 // NOTE: would like to use strchr(str, '\0') instead of this routine, but some buffers might not have a null terminator and stread by full to the the dt_size.
 static inline Py_UCS4*
-ucs4_get_end_p(Py_UCS4* p, Py_ssize_t dt_size) {
-    Py_UCS4* p_end = p + dt_size;
-    while (p < p_end && *p != '\0') {
-        p++;
+ucs4_get_end_p(Py_UCS4* p_start, Py_ssize_t dt_size) {
+    Py_UCS4* p;
+    for (p = p_start + dt_size - 1; p >= p_start; p--) {
+        if (*p != '\0') {
+            return p + 1; // return 1 more than the first non-null from the right
+        }
     }
-    return p;
+    return p; // p is equal to p_start
 }
 
 

setup.py

@@ -4,7 +4,7 @@
 import os
 
 
-AM_VERSION = "0.1.4"
+AM_VERSION = "0.1.5"
 
 
 with open("README.rst") as file:

test/test_property.py

@@ -52,7 +52,10 @@ def strategy(contiguous: bool):
             shape=1, unique=True, fill=st.nothing(), dtype=scalar_dtypes()
         ).map(partial(proc, contiguous=contiguous))
 
-    return st.one_of(strategy(contiguous=True), strategy(contiguous=False))
+    return st.one_of(
+        strategy(contiguous=True),
+        strategy(contiguous=False),
+    )
 
 
 @given(keys=hypothesis.infer)

test/test_unit.py

@@ -92,11 +92,6 @@ def test_fam_constructor_array_int_d():
         assert k in fam
 
 
-# def test_fam_constructor_array_a3():
-#     a1 = np.array(("a", "bb", "ccc"))
-#     with pytest.raises(TypeError):
-#         fam = FrozenAutoMap(a1)
-
 # ------------------------------------------------------------------------------
 
 
@@ -142,6 +137,38 @@ def test_fam_constructor_array_unicode_c():
     fam = FrozenAutoMap(a1)
 
 
+# NOTE
+# >>> u = "\x000\x00"
+# >>> len(u)
+# 3
+# >>> a1 = np.array(['', ''], dtype='U4')
+# >>> a1[0] = u
+# >>> a1
+# array(['\x000', ''], dtype='<U4')
+# >>> len(a1[0])
+# 2
+
+
+def test_fam_constructor_array_unicode_d1():
+    a1 = np.array(["", "\x000"], dtype="U2")
+    a1.flags.writeable = False
+    fam = FrozenAutoMap(a1)
+    assert len(fam) == 2
+    assert list(fam) == ["", "\x000"]
+    assert "" in fam
+    assert "\x000" in fam
+
+
+def test_fam_constructor_array_unicode_d2():
+    a1 = np.array(["", "\x000\x00"], dtype="U3")
+    a1.flags.writeable = False
+    fam = FrozenAutoMap(a1)
+    assert len(fam) == 2
+    assert list(fam) == ["", "\x000"]  # we lost the last null
+    assert "" in fam
+    assert "\x000" in fam
+
+
 def test_fam_copy_array_unicode_a():
     a1 = np.array(("a", "ccc", "bb"))
     a1.flags.writeable = False