fix: revert tokenizer chagne

jina-ai · Jul 29, 2022 · d29de44 · d29de44
1 parent 9bfacf0
commit d29de44
Showing 1 changed file with 0 additions and 2 deletions.
diff --git a/server/clip_server/model/tokenization.py b/server/clip_server/model/tokenization.py
@@ -8,9 +8,7 @@ def __init__(self, name: str, **kwargs):
         self._name = name
         if name in _MULTILINGUALCLIP_MODELS:
             import transformers
-            import os
 
-            os.environ["TOKENIZERS_PARALLELISM"] = "false"
             self._tokenizer = transformers.AutoTokenizer.from_pretrained(name)
         else:
             from clip_server.model.simple_tokenizer import SimpleTokenizer