Merge pull request #8 from mozilla/issue7

Fixed #7 (There is no language independent preprocessor)

src/corporacreator/corpora.py

@@ -5,6 +5,7 @@
 import pandas as pd
 
 from corporacreator import Corpus
+from corporacreator.preprocessors import common
 
 
 _logger = logging.getLogger(__name__)
@@ -29,6 +30,7 @@ def create(self):
         """
         _logger.info("Creating corpora...")
         corpora_data = self._parse_tsv()
+        corpora_data["sentence"] = corpora_data["sentence"].apply(func=common)
         for locale in corpora_data.locale.unique():
             _logger.info("Selecting %s corpus data..." % locale)
             corpus_data = corpora_data.loc[

src/corporacreator/preprocessors/__init__.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+from .common import common
 from .br import br
 from .ca import ca
 from .cv import cv

src/corporacreator/preprocessors/common.py

@@ -0,0 +1,11 @@
+def common(sentence):
+    """Cleans up the passed sentence in a language independent manner, removing or reformatting invalid data.
+
+    Args:
+      sentence (str): Sentence to be cleaned up.
+
+    Returns:
+      (str): Cleaned up sentence.
+    """
+    # TODO: Clean up data in a language independent manner
+    return sentence