Address PR feedback

Author: tdakhran

convert_hf_to_gguf.py

@@ -8837,7 +8837,7 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
 
 
 @ModelBase.register("Lfm2MoeForCausalLM")
-class LFM2MOEModel(TextModel):
+class LFM2MoeModel(TextModel):
     model_arch = gguf.MODEL_ARCH.LFM2MOE
 
     def set_gguf_parameters(self):
@@ -8865,18 +8865,20 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
         if 'conv.conv' in name:
             data_torch = data_torch.squeeze(1)
 
+        if name.endswith(".expert_bias"):
+            name = name.replace(".expert_bias", ".expert_bias.bias")
+
         # merge expert weights
         if 'experts' in name:
             n_experts = self.hparams["num_experts"]
             assert bid is not None
 
-            if bid not in self._experts_cache:
-                self._experts_cache[bid] = {}
-            self._experts_cache[bid][name] = data_torch
+            expert_cache = self._experts_cache.setdefault(bid, {})
+            expert_cache[name] = data_torch
             expert_weights = ["w1", "w2", "w3"]
 
             # not enough expert weights to merge
-            if len(self._experts_cache[bid]) < n_experts * len(expert_weights):
+            if len(expert_cache) < n_experts * len(expert_weights):
                 return []
 
             tensors: list[tuple[str, Tensor]] = []
@@ -8885,8 +8887,8 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
 
                 for xid in range(n_experts):
                     ename = f"model.layers.{bid}.feed_forward.experts.{xid}.{w_name}.weight"
-                    datas.append(self._experts_cache[bid][ename])
-                    del self._experts_cache[bid][ename]
+                    datas.append(expert_cache[ename])
+                    del expert_cache[ename]
 
                 data_torch = torch.stack(datas, dim=0)
                 merged_name = f"layers.{bid}.feed_forward.experts.{w_name}.weight"

src/llama-model.cpp

@@ -5853,7 +5853,7 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
                             layer.ffn_gate_exps   = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, hparams.n_ff_exp, n_expert}, 0);
                             layer.ffn_down_exps   = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {hparams.n_ff_exp,   n_embd, n_expert}, 0);
                             layer.ffn_up_exps     = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i),   {n_embd, hparams.n_ff_exp, n_expert}, 0);
-                            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, i),         {n_expert}, 0);
+                            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);
                         } else {  // dense
                             layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd,   n_ff}, 0);
                             layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {  n_ff, n_embd}, 0);