fix ckpt conversion issue

tools/checkpoint/loader_mixtral_hf.py

@@ -262,6 +262,7 @@ def check_for_arg(arg_name, default=None):
     md.position_embedding_type = margs.position_embedding_type
     md.linear_bias = margs.add_bias_linear
     md.norm_has_bias = False
+    md.qkv_bias = False
     md.swiglu = margs.swiglu
     md.previous_tensor_parallel_size = margs.tensor_model_parallel_size
     md.previous_pipeline_parallel_size = margs.pipeline_model_parallel_size

tools/checkpoint/saver_base.py

@@ -483,9 +483,14 @@ def pad_weight(orig_word_embed, true_vocab_size):
                             "mlp_norm_weight" : post_norm_weight
                         }
                         if self.margs.num_experts:
+                            num_local_experts = self.margs.num_experts // self.args.target_expert_parallel_size
                             params_dict.update({
-                                "mlp_fc1_weight" : mlp_l0_weight[ep_rank][tp_rank],
-                                "mlp_fc2_weight" : mlp_l1_weight[ep_rank][tp_rank]
+                                f"mlp_fc1_weight.{i}": mlp_l0_weight[ep_rank][tp_rank][i]
+                                for i in range(num_local_experts)
+                            })
+                            params_dict.update({
+                                f"mlp_fc2_weight.{i}": mlp_l1_weight[ep_rank][tp_rank][i]
+                                for i in range(num_local_experts)
                             })
                         else:
                             params_dict.update({