improve handling of None/nan

CHANGELOG.md

@@ -1,5 +1,13 @@
 ## Changelog
 
+### [2.14.0] -
+#### Changed
+- handle `float("nan")` similar to None for query / choice, since this is common for
+  non-existent data in tools like numpy
+
+#### Fixed
+- fix handling on `None`/`float("nan")` in `process.distance`
+
 ### [2.13.3] - 2022-12-03
 #### Fixed
 - improve handling of functions wrapped using `functools.wraps`

src/rapidfuzz/_utils.py

@@ -5,6 +5,7 @@
 
 from enum import Flag
 from typing import Any, Callable
+from math import isnan
 
 
 class ScorerFlag(Flag):
@@ -29,19 +30,33 @@ def _get_scorer_flags_similarity(**_kwargs: Any) -> dict[str, Any]:
     }
 
 
+def is_none(s: Any) -> bool:
+    if s is None:
+        return True
+
+    if isinstance(s, float) and isnan(s):
+        return True
+
+    return False
+
+
 def _get_scorer_flags_normalized_distance(**_kwargs: Any) -> dict[str, Any]:
     return {"optimal_score": 0, "worst_score": 1, "flags": ScorerFlag.RESULT_F64}
 
 
 def _get_scorer_flags_normalized_similarity(**_kwargs: Any) -> dict[str, Any]:
     return {"optimal_score": 1, "worst_score": 0, "flags": ScorerFlag.RESULT_F64}
 
-def _create_scorer(func: Any, cached_scorer_call: dict[str, Callable[..., dict[str, Any]]]):
+
+def _create_scorer(
+    func: Any, cached_scorer_call: dict[str, Callable[..., dict[str, Any]]]
+):
     func._RF_ScorerPy = cached_scorer_call
     # used to detect the function hasn't been wrapped afterwards
     func._RF_OriginalScorer = func
     return func
 
+
 def fallback_import(
     module: str,
     name: str,
@@ -93,6 +108,7 @@ def fallback_import(
 
     return cpp_func
 
+
 default_distance_attribute: dict[str, Callable[..., dict[str, Any]]] = {
     "get_scorer_flags": _get_scorer_flags_distance
 }

src/rapidfuzz/distance/DamerauLevenshtein_py.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 from typing import Callable, Hashable, Sequence
+from rapidfuzz._utils import is_none
 
 
 def _damerau_levenshtein_distance_zhao(
@@ -174,6 +175,9 @@ def normalized_distance(
     norm_dist : float
         normalized distance between s1 and s2 as a float between 0 and 1.0
     """
+    if is_none(s1) or is_none(s2):
+        return 1.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)
@@ -215,6 +219,9 @@ def normalized_similarity(
     norm_sim : float
         normalized similarity between s1 and s2 as a float between 0 and 1.0
     """
+    if is_none(s1) or is_none(s2):
+        return 0.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)

src/rapidfuzz/distance/Hamming_py.py

@@ -6,6 +6,7 @@
 from typing import Callable, Hashable, Sequence
 
 from rapidfuzz.distance._initialize import Editops, Opcodes
+from rapidfuzz._utils import is_none
 
 
 def distance(
@@ -144,6 +145,9 @@ def normalized_distance(
     ValueError
         If s1 and s2 have a different length
     """
+    if is_none(s1) or is_none(s2):
+        return 1.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)
@@ -191,6 +195,9 @@ def normalized_similarity(
     ValueError
         If s1 and s2 have a different length
     """
+    if is_none(s1) or is_none(s2):
+        return 0.0
+
     norm_dist = normalized_distance(s1, s2, processor=processor)
     norm_sim = 1 - norm_dist
 

src/rapidfuzz/distance/Indel_py.py

@@ -8,6 +8,7 @@
 from rapidfuzz.distance._initialize import Editops, Opcodes
 from rapidfuzz.distance.LCSseq_py import _block_similarity as lcs_seq_block_similarity
 from rapidfuzz.distance.LCSseq_py import similarity as lcs_seq_similarity
+from rapidfuzz._utils import is_none
 
 
 def distance(
@@ -152,6 +153,9 @@ def normalized_distance(
     norm_dist : float
         normalized distance between s1 and s2 as a float between 0 and 1.0
     """
+    if is_none(s1) or is_none(s2):
+        return 1.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)
@@ -224,6 +228,9 @@ def normalized_similarity(
     >>> Indel.normalized_similarity(["lewenstein"], ["levenshtein"], processor=lambda s: s[0])
     0.8571428571428572
     """
+    if is_none(s1) or is_none(s2):
+        return 0.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)

src/rapidfuzz/distance/JaroWinkler_py.py

@@ -6,6 +6,7 @@
 from typing import Callable, Hashable, Sequence
 
 from rapidfuzz.distance import Jaro
+from rapidfuzz._utils import is_none
 
 
 def similarity(
@@ -46,6 +47,9 @@ def similarity(
     ValueError
         If prefix_weight is invalid
     """
+    if is_none(s1) or is_none(s2):
+        return 0.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)
@@ -167,6 +171,9 @@ def distance(
     ValueError
         If prefix_weight is invalid
     """
+    if is_none(s1) or is_none(s2):
+        return 1.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)

src/rapidfuzz/distance/Jaro_py.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 from typing import Callable, Hashable, Sequence
+from rapidfuzz._utils import is_none
 
 
 def _jaro_calculate_similarity(
@@ -95,6 +96,9 @@ def similarity(
     similarity : float
         similarity between s1 and s2 as a float between 0 and 1.0
     """
+    if is_none(s1) or is_none(s2):
+        return 0.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)
@@ -209,6 +213,9 @@ def distance(
     distance : float
         distance between s1 and s2 as a float between 1.0 and 0.0
     """
+    if is_none(s1) or is_none(s2):
+        return 1.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)

src/rapidfuzz/distance/LCSseq_py.py

@@ -6,6 +6,7 @@
 from typing import Callable, Hashable, Sequence
 
 from rapidfuzz.distance._initialize import Editops, Opcodes
+from rapidfuzz._utils import is_none
 
 
 def similarity(
@@ -173,6 +174,9 @@ def normalized_distance(
     norm_dist : float
         normalized distance between s1 and s2 as a float between 0 and 1.0
     """
+    if is_none(s1) or is_none(s2):
+        return 1.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)
@@ -235,6 +239,9 @@ def normalized_similarity(
     >>> LCSseq.normalized_similarity(["lewenstein"], ["levenshtein"], processor=lambda s: s[0])
     0.81818181818181
     """
+    if is_none(s1) or is_none(s2):
+        return 0.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)

src/rapidfuzz/distance/Levenshtein_py.py

@@ -7,6 +7,7 @@
 
 from rapidfuzz.distance import Indel
 from rapidfuzz.distance._initialize import Editops, Opcodes
+from rapidfuzz._utils import is_none
 
 
 def _levenshtein_maximum(
@@ -268,6 +269,9 @@ def normalized_distance(
     ValueError
         If unsupported weights are provided a ValueError is thrown
     """
+    if is_none(s1) or is_none(s2):
+        return 1.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)
@@ -350,6 +354,9 @@ def normalized_similarity(
     >>> Levenshtein.normalized_similarity(["lewenstein"], ["levenshtein"], processor=lambda s: s[0])
     0.81818181818181
     """
+    if is_none(s1) or is_none(s2):
+        return 0.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)

src/rapidfuzz/distance/OSA_py.py

@@ -5,6 +5,8 @@
 
 from typing import Callable, Hashable, Sequence
 
+from rapidfuzz._utils import is_none
+
 
 def _osa_distance_hyrroe2003(s1: Sequence[Hashable], s2: Sequence[Hashable]) -> int:
     if not s1:
@@ -170,6 +172,9 @@ def normalized_distance(
     norm_dist : float
         normalized distance between s1 and s2 as a float between 0 and 1.0
     """
+    if is_none(s1) or is_none(s2):
+        return 1.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)
@@ -211,6 +216,9 @@ def normalized_similarity(
     norm_sim : float
         normalized similarity between s1 and s2 as a float between 0 and 1.0
     """
+    if is_none(s1) or is_none(s2):
+        return 0.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)

src/rapidfuzz/distance/Postfix_py.py

@@ -5,6 +5,8 @@
 
 from typing import Callable, Hashable, Sequence
 
+from rapidfuzz._utils import is_none
+
 
 def distance(
     s1: Sequence[Hashable],
@@ -123,6 +125,9 @@ def normalized_distance(
     norm_dist : float
         normalized distance between s1 and s2 as a float between 0 and 1.0
     """
+    if is_none(s1) or is_none(s2):
+        return 1.0
+
     norm_sim = normalized_similarity(s1, s2, processor=processor)
     norm_dist = 1.0 - norm_sim
 
@@ -160,6 +165,9 @@ def normalized_similarity(
     norm_sim : float
         normalized similarity between s1 and s2 as a float between 0 and 1.0
     """
+    if is_none(s1) or is_none(s2):
+        return 0.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)

src/rapidfuzz/distance/Prefix_py.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 from typing import Callable, Hashable, Sequence
+from rapidfuzz._utils import is_none
 
 
 def distance(
@@ -123,6 +124,9 @@ def normalized_distance(
     norm_dist : float
         normalized distance between s1 and s2 as a float between 0 and 1.0
     """
+    if is_none(s1) or is_none(s2):
+        return 1.0
+
     norm_sim = normalized_similarity(s1, s2, processor=processor)
     norm_dist = 1.0 - norm_sim
 
@@ -160,6 +164,9 @@ def normalized_similarity(
     norm_sim : float
         normalized similarity between s1 and s2 as a float between 0 and 1.0
     """
+    if is_none(s1) or is_none(s2):
+        return 0.0
+
     if processor is not None:
         s1 = processor(s1)
         s2 = processor(s2)