collapse whitespace for all langs

common-voice · Jan 30, 2019 · ba2b52e · ba2b52e
1 parent b0cbeb8
commit ba2b52e
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/src/corporacreator/preprocessors/common.py b/src/corporacreator/preprocessors/common.py
@@ -75,5 +75,7 @@ def common(sentence):
     sentence = _strip_tags(sentence)
     # Remove non-printable characters
     sentence = _strip_string(sentence)
+    # collapse all whitespace and replace with single space
+    sentence = (' ').join(sentence.split())
     # TODO: Clean up data in a language independent manner
     return sentence