convert-hf-to-gguf: add Viking-7b (using llama-bpe)

Author: akx

convert-hf-to-gguf-update.py

@@ -80,6 +80,7 @@ class TOKENIZER_TYPE(IntEnum):
     {"name": "jina-v2-en",     "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-en", }, # WPM!
     {"name": "jina-v2-es",     "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-es", },
     {"name": "jina-v2-de",     "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-de", },
+    {"name": "viking-7b",      "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LumiOpen/Viking-7B", "type": "llama-bpe" },
 ]
 
 
@@ -124,6 +125,7 @@ def download_model(model):
 src_ifs = ""
 for model in models:
     name = model["name"]
+    model_type = model.get("type") or name
     tokt = model["tokt"]
 
     if tokt == TOKENIZER_TYPE.SPM:
@@ -164,7 +166,7 @@ def download_model(model):
 
     src_ifs += f"        if chkhsh == \"{chkhsh}\":\n"
     src_ifs += f"            # ref: {model['repo']}\n"
-    src_ifs += f"            res = \"{name}\"\n"
+    src_ifs += f"            res = \"{model_type}\"\n"
 
 src_func = f"""
     def get_vocab_base_pre(self, tokenizer) -> str:

convert-hf-to-gguf.py

@@ -470,6 +470,9 @@ def get_vocab_base_pre(self, tokenizer) -> str:
         if chkhsh == "27949a2493fc4a9f53f5b9b029c82689cfbe5d3a1929bb25e043089e28466de6":
             # ref: https://huggingface.co/jinaai/jina-embeddings-v2-base-de
             res = "jina-v2-de"
+        if chkhsh == "7fc505bd3104ca1083b150b17d088b59534ede9bde81f0dd2090967d7fe52cee":
+            # ref: https://huggingface.co/LumiOpen/Viking-7B
+            res = "llama-bpe"
 
         if res is None:
             logger.warning("\n")