Add LumiOpen/Viking-7B tokenizer support

ggerganov · May 17, 2024 · ab842e3 · ab842e3
1 parent 3afb494
commit ab842e3
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 3 deletions.
diff --git a/convert-hf-to-gguf-update.py b/convert-hf-to-gguf-update.py
@@ -80,6 +80,7 @@ class TOKENIZER_TYPE(IntEnum):
     {"name": "jina-v2-en",     "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-en", }, # WPM!
     {"name": "jina-v2-es",     "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-es", },
     {"name": "jina-v2-de",     "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-de", },
+    {"name": "viking-7b",      "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LumiOpen/Viking-7B" },
 ]
 
 

diff --git a/convert-hf-to-gguf.py b/convert-hf-to-gguf.py
@@ -470,6 +470,9 @@ def get_vocab_base_pre(self, tokenizer) -> str:
         if chkhsh == "27949a2493fc4a9f53f5b9b029c82689cfbe5d3a1929bb25e043089e28466de6":
             # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-de
             res = "jina-v2-de"
+        if chkhsh == "7fc505bd3104ca1083b150b17d088b59534ede9bde81f0dd2090967d7fe52cee":
+            # ref: https://huggingface.co/LumiOpen/Viking-7B
+            res = "viking-7b"
 
         if res is None:
             logger.warning("\n")

diff --git a/llama.cpp b/llama.cpp
@@ -4426,9 +4426,10 @@ static void llm_load_vocab(
                     tokenizer_pre == "default") {
                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
             } else if (
-                    tokenizer_pre == "llama3"   ||
-                    tokenizer_pre == "llama-v3" ||
-                    tokenizer_pre == "llama-bpe") {
+                    tokenizer_pre == "llama3"    ||
+                    tokenizer_pre == "llama-v3"  ||
+                    tokenizer_pre == "llama-bpe" ||
+                    tokenizer_pre == "viking-7b") {
                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_LLAMA3;
             } else if (
                     tokenizer_pre == "deepseek-llm") {