Add fast tokenizer for BARTpho

docs/source/en/model_doc/bartpho.mdx

@@ -70,13 +70,14 @@ Tips:
 >>> print(tokenizer.decode(predictions).split())
 ```
 
-- This implementation is only for tokenization: "monolingual_vocab_file" consists of Vietnamese-specialized types
-  extracted from the pre-trained SentencePiece model "vocab_file" that is available from the multilingual XLM-RoBERTa.
-  Other languages, if employing this pre-trained multilingual SentencePiece model "vocab_file" for subword
-  segmentation, can reuse BartphoTokenizer with their own language-specialized "monolingual_vocab_file".
+- This implementation is only for tokenization.
 
 This model was contributed by [dqnguyen](https://huggingface.co/dqnguyen). The original code can be found [here](https://github.com/VinAIResearch/BARTpho).
 
 ## BartphoTokenizer
 
 [[autodoc]] BartphoTokenizer
+
+## BartphoTokenizerFast
+
+[[autodoc]] BartphoTokenizerFast

src/transformers/__init__.py

@@ -473,6 +473,7 @@
     _import_structure["models.albert"].append("AlbertTokenizerFast")
     _import_structure["models.bart"].append("BartTokenizerFast")
     _import_structure["models.barthez"].append("BarthezTokenizerFast")
+    _import_structure["models.bartpho"].append("BartphoTokenizerFast")
     _import_structure["models.bert"].append("BertTokenizerFast")
     _import_structure["models.big_bird"].append("BigBirdTokenizerFast")
     _import_structure["models.blenderbot"].append("BlenderbotTokenizerFast")
@@ -2922,6 +2923,7 @@
         from .models.albert import AlbertTokenizerFast
         from .models.bart import BartTokenizerFast
         from .models.barthez import BarthezTokenizerFast
+        from .models.bartpho import BartphoTokenizerFast
         from .models.bert import BertTokenizerFast
         from .models.big_bird import BigBirdTokenizerFast
         from .models.blenderbot import BlenderbotTokenizerFast

src/transformers/convert_slow_tokenizer.py

@@ -551,6 +551,60 @@ def post_processor(self):
         )
 
 
+class BartphoConverter(SpmConverter):
+    def vocab(self, proto):
+        vocab = [
+            ("<s>", 0.0),
+            ("<pad>", 0.0),
+            ("</s>", 0.0),
+            ("<unk>", 0.0),
+        ]
+        vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]]
+        vocab += [
+            ("ar_AR", 0.0),
+            ("cs_CZ", 0.0),
+            ("de_DE", 0.0),
+            ("en_XX", 0.0),
+            ("es_XX", 0.0),
+            ("et_EE", 0.0),
+            ("fi_FI", 0.0),
+            ("fr_XX", 0.0),
+            ("gu_IN", 0.0),
+            ("hi_IN", 0.0),
+            ("it_IT", 0.0),
+            ("ja_XX", 0.0),
+            ("kk_KZ", 0.0),
+            ("ko_KR", 0.0),
+            ("lt_LT", 0.0),
+            ("lv_LV", 0.0),
+            ("my_MM", 0.0),
+            ("ne_NP", 0.0),
+            ("nl_XX", 0.0),
+            ("ro_RO", 0.0),
+            ("ru_RU", 0.0),
+            ("si_LK", 0.0),
+            ("tr_TR", 0.0),
+            ("vi_VN", 0.0),
+            ("zh_CN", 0.0),
+        ]
+        vocab += [("<mask>", 0.0)]
+        return vocab
+
+    def unk_id(self, proto):
+        unk_id = 3
+        return unk_id
+
+    def post_processor(self):
+        return processors.TemplateProcessing(
+            single="<s> $A </s>",
+            pair="<s> $A </s> </s> $B </s>",
+            special_tokens=[
+                ("<s>", self.original_tokenizer.convert_tokens_to_ids("<s>")),
+                ("</s>", self.original_tokenizer.convert_tokens_to_ids("</s>")),
+            ],
+        )
+
+
 class CamembertConverter(SpmConverter):
     def vocab(self, proto):
         vocab = [
@@ -1004,6 +1058,7 @@ def post_processor(self):
     "AlbertTokenizer": AlbertConverter,
     "BartTokenizer": RobertaConverter,
     "BarthezTokenizer": BarthezConverter,
+    "BartphoTokenizer": BartphoConverter,
     "BertTokenizer": BertConverter,
     "BigBirdTokenizer": BigBirdConverter,
     "BlenderbotTokenizer": BlenderbotConverter,

src/transformers/models/auto/tokenization_auto.py

@@ -197,7 +197,13 @@
             ),
             ("herbert", ("HerbertTokenizer", "HerbertTokenizerFast" if is_tokenizers_available() else None)),
             ("phobert", ("PhobertTokenizer", None)),
-            ("bartpho", ("BartphoTokenizer", None)),
+            (
+                "bartpho",
+                (
+                    "BartphoTokenizer" if is_sentencepiece_available() else None,
+                    "BartphoTokenizerFast" if is_tokenizers_available() else None,
+                ),
+            ),
             (
                 "barthez",
                 (

src/transformers/models/bartpho/__init__.py

@@ -18,7 +18,7 @@
 
 from typing import TYPE_CHECKING
 
-from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available
+from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_tokenizers_available
 
 
 _import_structure = {}
@@ -31,6 +31,14 @@
 else:
     _import_structure["tokenization_bartpho"] = ["BartphoTokenizer"]
 
+try:
+    if not is_tokenizers_available():
+        raise OptionalDependencyNotAvailable()
+except OptionalDependencyNotAvailable:
+    pass
+else:
+    _import_structure["tokenization_bartpho_fast"] = ["BartphoTokenizerFast"]
+
 if TYPE_CHECKING:
     try:
         if not is_sentencepiece_available():
@@ -40,6 +48,14 @@
     else:
         from .tokenization_bartpho import BartphoTokenizer
 
+    try:
+        if not is_tokenizers_available():
+            raise OptionalDependencyNotAvailable()
+    except OptionalDependencyNotAvailable:
+        pass
+    else:
+        from .tokenization_bartpho_fast import BartphoTokenizerFast
+
 else:
     import sys