Merge pull request #19 from mozilla/issue18

Fixed #18 (Some sentences contain HTML).

src/corporacreator/preprocessors/common.py

@@ -1,3 +1,39 @@
+from html.parser import HTMLParser
+
+class _HTMLStripper(HTMLParser):
+    """Class that strips HTML from strings.
+
+    Examples:
+        >>> stripper = _HTMLStripper()
+        >>> stripper.feed(html)
+        >>> nohtml = stripper.get_data()
+    """
+    def __init__(self):
+        super().__init__()
+        self.reset()
+        self.strict = False
+        self.convert_charrefs= True
+        self.fed = []
+
+    def handle_data(self, d):
+        self.fed.append(d)
+
+    def get_data(self):
+        return ''.join(self.fed)
+
+def _strip_tags(html):
+    """Removes HTML tags from passed text.
+
+    Args:
+      html (str): String containing HTML
+
+    Returns:
+      (str): String with HTML removed
+    """
+    s = _HTMLStripper()
+    s.feed(html)
+    return s.get_data()
+
 def common(sentence):
     """Cleans up the passed sentence in a language independent manner, removing or reformatting invalid data.
 
@@ -7,5 +43,7 @@ def common(sentence):
     Returns:
       (str): Cleaned up sentence.
     """
+    # Remove any HTML tags
+    sentence = _strip_tags(sentence)
     # TODO: Clean up data in a language independent manner
     return sentence