ENH: support chat_template_kwargs for llama.cpp (#3988)

Author: qinxuye

xinference/model/llm/llama_cpp/core.py

@@ -23,7 +23,7 @@
 
 from ....constants import XINFERENCE_MAX_TOKENS
 from ....types import ChatCompletion, ChatCompletionChunk, Completion, CompletionChunk
-from ..core import LLM
+from ..core import LLM, chat_context_var
 from ..llm_family import LLMFamilyV2, LLMSpecV1
 from ..utils import ChatModelMixin
 
@@ -297,6 +297,15 @@ def chat(
         if not generate_config.get("max_tokens") and XINFERENCE_MAX_TOKENS:
             generate_config["max_tokens"] = XINFERENCE_MAX_TOKENS
         stream = generate_config.get("stream", False)
+
+        chat_template_kwargs = (
+            self._get_chat_template_kwargs_from_generate_config(
+                generate_config, self.reasoning_parser
+            )
+            or {}
+        )
+        chat_context_var.set(chat_template_kwargs)
+
         tools = generate_config.pop("tools", []) if generate_config else None
         q: queue.Queue = queue.Queue()
 
@@ -314,6 +323,9 @@ def _handle_chat_completion():
                     "model": self.model_uid,
                 }
             )
+            if chat_template_kwargs:
+                data["chat_template_kwargs"] = chat_template_kwargs
+
             try:
 
                 def _callback(res):

xinference/model/llm/llm_family.json

@@ -7687,7 +7687,7 @@
       "packages": [
         "transformers>=4.51.3",
         "mlx-lm>=0.23.1 ; sys_platform=='darwin'",
-        "numpy==1.26.4"
+        "#system_numpy#"
       ]
     }
   },
@@ -15521,7 +15521,7 @@
     "virtualenv": {
       "packages": [
         "git+https://github.com/huggingface/[email protected]",
-        "numpy==1.26.4",
+        "#system_numpy#",
         "qwen_omni_utils",
         "soundfile"
       ]
@@ -17302,7 +17302,7 @@
       "packages": [
         "transformers>=4.51.0",
         "mlx-lm>=0.24.0 ; sys_platform=='darwin'",
-        "numpy==1.26.4"
+        "#system_numpy#"
       ]
     }
   },

xinference/ui/gradio/chat_interface.py

@@ -135,6 +135,8 @@ def generate_wrapper(
                     generate_config=generate_config,  # type: ignore
                 ):
                     assert isinstance(chunk, dict)
+                    if not chunk["choices"]:
+                        continue
                     delta = chunk["choices"][0]["delta"]
 
                     if (