UTF8 test

Author: Mug

tests/test_llama.py

@@ -93,4 +93,27 @@ def test_llama_pickle():
 
     text = b"Hello World"
 
-    assert llama.detokenize(llama.tokenize(text)) == text
+    assert llama.detokenize(llama.tokenize(text)) == text
+
+def test_llama_utf8(monkeypatch):
+    llama = llama_cpp.Llama(model_path=MODEL, vocab_only=True)
+
+    ## Set up mock function
+    def mock_eval(*args, **kwargs):
+        return 0
+
+    monkeypatch.setattr("llama_cpp.llama_cpp.llama_eval", mock_eval)
+
+    output_text = "😀"
+    output_tokens = [243, 162, 155, 131]
+
+    def mock_tokenize(*args, **kwargs):
+        return output_tokens[:-1]
+
+    llama.tokenize = mock_tokenize
+
+    ## Test basic completion until eos
+    n = 0  # reset
+    completion = llama.create_completion("ignored", max_tokens=1)
+    assert completion["choices"][0]["text"] == output_text
+