Merge pull request #80 from mozilla/whitespace

common.py --- collapse whitespace for all langs
common-voice · Jan 31, 2019 · 0eafd2a · 0eafd2a
2 parents b0cbeb8 + ba2b52e
commit 0eafd2a
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/src/corporacreator/preprocessors/common.py b/src/corporacreator/preprocessors/common.py
@@ -75,5 +75,7 @@ def common(sentence):
     sentence = _strip_tags(sentence)
     # Remove non-printable characters
     sentence = _strip_string(sentence)
+    # collapse all whitespace and replace with single space
+    sentence = (' ').join(sentence.split())
     # TODO: Clean up data in a language independent manner
     return sentence