Used ASCII to check the summarization of the OCR text

puterbot/strategies/demo.py

@@ -21,6 +21,7 @@
 
 from gensim.summarization.summarizer import summarize
 from nltk.corpus import wordnet
+import re
 
 
 class DemoReplayStrategy(
@@ -44,22 +45,33 @@ def get_next_input_event(
         ascii_text = self.get_ascii_text(screenshot)
         #logger.info(f"ascii_text=\n{ascii_text}")
 
+        cleaned_ascii = clean_ascii(ascii_text)
+
+        summarized_ascii = summarize(cleaned_ascii, word_count=1, split=False)
+        synonyms_ascii = set(wordnet.synsets(summarized_ascii))
+
         ocr_text = self.get_ocr_text(screenshot)
         #logger.info(f"ocr_text=\n{ocr_text}")
 
         # identify what the window is
         summarized_ocr = summarize(ocr_text, word_count=1, split=False)
+        synonyms_ocr = set(wordnet.synsets(summarized_ocr))
+
+        # words that describe the screenshot display
+        summarized_screenshot = set(synonyms_ascii & synonyms_ocr)
 
         index = self.screenshots.index(screenshot)
         if index != 0:
             last_screenshot = self.screenshots[index - 1]
-            last_summarized_ocr = summarize(last_screenshot, word_count=1, split=False)
+            last_summarized_ascii = summarize(self.get_ascii_text(last_screenshot), word_count=1,
+                                              split=False)
+            last_summarized_ocr = summarize(self.get_ocr_text(last_screenshot), word_count=1,
+                                            split=False)
+            # summarize the last screenshot
+            summarized_screenshot = set(last_summarized_ascii & last_summarized_ocr)
 
-            # check if the last screenshot and current screenshot picture the same image
-            synonyms_screenshot = set(wordnet.synsets(summarized_ocr))
-            synonyms_last_screenshot = set(wordnet.synsets(last_summarized_ocr))
-
-            common_synonyms = list(synonyms_screenshot & synonyms_last_screenshot)
+            # check whether the last screenshot and the current screenshot are the same
+            common_synonyms = list(summarized_screenshot & summarized_screenshot)
 
             # may want to change the number of required common synonyms
             if len(common_synonyms) > 0:
@@ -93,3 +105,17 @@ def get_next_input_event(
         # TODO: parse result into InputEvent(s)
 
         return None
+
+
+def clean_ascii(
+    text,
+):
+    # replace the irrelevant symbols in the ascii with spaces
+    no_symbols = re.sub(r'[^\w\s]+', '', text)
+
+    ascii_words = []
+
+    for word in no_symbols:
+        if wordnet.synsets(word):
+            ascii_words.append(word)
+    return ascii_words