From f51a158469e2394793101f6cb62efc0218a41975 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 26 Jan 2025 12:16:47 +0000 Subject: [PATCH] [Bugfix] Fix Granite 3.0 MoE model loading Signed-off-by: DarkLight1337 --- vllm/model_executor/models/granitemoe.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vllm/model_executor/models/granitemoe.py b/vllm/model_executor/models/granitemoe.py index 51296ef0cc08..b518a0a6cbde 100644 --- a/vllm/model_executor/models/granitemoe.py +++ b/vllm/model_executor/models/granitemoe.py @@ -348,6 +348,7 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): self.config = config self.lora_config = lora_config + self.quant_config = quant_config # Required by MixtralForCausalLM self.model = GraniteMoeModel(vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model"))