fix: add downloading nltk punkt for tokenization (#140)

deeppavlov · Mar 23, 2018 · 3618ae6 · 3618ae6
1 parent ff8468f
commit 3618ae6
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/deeppavlov/models/preprocessors/squad_preprocessor.py b/deeppavlov/models/preprocessors/squad_preprocessor.py
@@ -39,6 +39,11 @@ def __init__(self, context_limit, question_limit, char_limit, *args, **kwargs):
         self.context_limit = context_limit
         self.question_limit = question_limit
         self.char_limit = char_limit
+        try:
+            import nltk
+            nltk.data.find('tokenizers/punkt')
+        except LookupError:
+            nltk.download('punkt')
 
     def __call__(self, contexts_raw, questions_raw, **kwargs):
         contexts = []