From 696797a16f06b028edc913cec9b833ba6cc6475b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 08:34:03 +0000 Subject: [PATCH 1/2] kimi-k3 : fixes for loading and running the full-size model Graph node budget, LLM_TYPE for the 93-layer config, four hparams promoted to required, and an E8M0 NaN check during the MXFP4 repack. Details in the PR description. Assisted-by: Claude Code --- conversion/base.py | 5 +++++ src/models/kimi-k3.cpp | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/conversion/base.py b/conversion/base.py index 72ba6863ad00..f938afa62e5f 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -104,6 +104,11 @@ def repack_mxfp4_blocks(packed: Tensor, scale: Tensor) -> np.ndarray: if tuple(s.shape) != (rows, n_blocks): raise ValueError(f"MXFP4 scale shape {tuple(s.shape)} does not match {(rows, n_blocks)}") + # 0xff is the NaN E8M0 exponent; caught later without the tensor name + n_bad = int((s == 0xFF).sum()) + if n_bad: + raise ValueError(f"invalid E8M0 scale byte 0xff in {n_bad} MXFP4 block(s)") + src = p.reshape(rows, n_blocks, 16) lo = src & 0x0F # elements 0, 2, 4, ... hi = (src >> 4) & 0x0F # elements 1, 3, 5, ... diff --git a/src/models/kimi-k3.cpp b/src/models/kimi-k3.cpp index d1479791514d..79c8aad4555b 100644 --- a/src/models/kimi-k3.cpp +++ b/src/models/kimi-k3.cpp @@ -35,12 +35,15 @@ void llama_model_kimi_k3::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func); - ml.get_key(LLM_KV_EXPERT_LATENT_LENGTH, hparams.n_expert_latent); - + // required: a silent default here loads cleanly and produces garbage + ml.get_key(LLM_KV_EXPERT_LATENT_LENGTH, hparams.n_expert_latent); ml.get_key(LLM_KV_ATTN_RES_BLOCK_SIZE, hparams.attn_res_block_size); ml.get_key(LLM_KV_ACTIVATION_SITU_BETA, hparams.situ_beta); ml.get_key(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, hparams.situ_linear_beta); + GGML_ASSERT(hparams.attn_res_block_size > 0 && "Kimi-K3 requires attn_res.block_size"); + GGML_ASSERT(hparams.n_expert_latent > 0 && "Kimi-K3 requires expert_latent_length"); + switch (hparams.n_layer()) { case 93: type = LLM_TYPE_2_8T_A50B; break; // Kimi-K3 default: type = LLM_TYPE_UNKNOWN; From daef2b3e1b5b1ac7b575f13b13a9450cb2d02862 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 29 Jul 2026 08:34:03 +0000 Subject: [PATCH 2/2] kimi-k3 : add the MoonViT-3d vision tower (image path) New kimik3 projector type, its graph builder and the mmproj converter. Also adds an optional clip.%s.attention.head_dim so build_vit stops deriving d_head from n_embd, which is wrong whenever a tower's qkv width differs from n_embd. Assisted-by: Claude Code --- conversion/kimivl.py | 76 ++++++++++++++++++++++++++++++++++ gguf-py/gguf/constants.py | 1 + tools/mtmd/CMakeLists.txt | 1 + tools/mtmd/clip-impl.h | 2 + tools/mtmd/clip.cpp | 31 ++++++++++++++ tools/mtmd/models/kimik3.cpp | 80 ++++++++++++++++++++++++++++++++++++ tools/mtmd/models/models.h | 7 ++++ tools/mtmd/mtmd.cpp | 1 + 8 files changed, 199 insertions(+) create mode 100644 tools/mtmd/models/kimik3.cpp diff --git a/conversion/kimivl.py b/conversion/kimivl.py index 5ff3c39ca9c1..a5e278f1c090 100644 --- a/conversion/kimivl.py +++ b/conversion/kimivl.py @@ -168,3 +168,79 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter name = name.replace("mm_projector.linear_", "mm_projector.proj.linear_", 1) yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("KimiK3ForConditionalGeneration") +class KimiK3VisionModel(MmprojModel): + """Kimi-K3 MoonViT-3d vision tower (image path). + + Structurally the Kimi-K2.5 tower with RMSNorm, no biases, a non-square fused QKV + (qkv_hidden_size 1536 vs vt_hidden_size 1024) and a post-norm patchmergerv2 projector. + Video is out of scope: for t == 1 the temporal pool and temporal position term vanish. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + assert self.hparams_vision is not None, "Kimi-K3 requires vision_config in config.json" + self.merge_kernel_size = tuple(self.hparams_vision.get("merge_kernel_size", [2, 2])) + self.patch_size = self.hparams_vision.get("patch_size", 14) + pos_emb_h = self.hparams_vision.get("init_pos_emb_height", 64) + self.hparams_vision["image_size"] = pos_emb_h * self.patch_size + + def set_gguf_parameters(self): + super().set_gguf_parameters() + assert self.hparams_vision is not None + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.KIMIK3) + + # qkv width != n_embd, so the runtime cannot derive d_head + n_head = self.hparams_vision["vt_num_attention_heads"] + qkv_hidden = self.hparams_vision.get("qkv_hidden_size") or self.hparams_vision["vt_hidden_size"] + assert qkv_hidden % n_head == 0, f"qkv_hidden_size {qkv_hidden} not divisible by {n_head} heads" + self.gguf_writer.add_vision_head_dim(qkv_hidden // n_head) + + self.gguf_writer.add_vision_use_gelu(True) # activation_func is gelu_pytorch_tanh + self.gguf_writer.add_vision_attention_layernorm_eps( + self.hparams_vision.get("projector_ln_eps", 1e-5)) + self.gguf_writer.add_vision_projector_scale_factor(self.merge_kernel_size[0]) + + in_patch_limit = self.preprocessor_config.get("media_proc_cfg", {}).get( + "in_patch_limit", self.preprocessor_config.get("in_patch_limit", 16384)) + pixels_per_patch = self.patch_size ** 2 + self.gguf_writer.add_vision_min_pixels(8 * pixels_per_patch) + self.gguf_writer.add_vision_max_pixels(in_patch_limit * pixels_per_patch) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, _ = item + if not name.startswith(("vision_tower.", "mm_projector.")): + return None + return super().filter_tensors(item) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + assert self.hparams_vision is not None + n_head = self.hparams_vision["vt_num_attention_heads"] + + if "wqkv" in name and "weight" in name: + # de-interleave Q/K so the runtime can use build_rope_2d(interleave_freq=false) + out_dim = data_torch.shape[0] + qkv_dim = out_dim // 3 + head_dim = qkv_dim // n_head + wq, wk, wv = (data_torch[:qkv_dim], data_torch[qkv_dim:2 * qkv_dim], data_torch[2 * qkv_dim:]) + + def deinterleave(w: Tensor) -> Tensor: + return (w.reshape(n_head, head_dim // 4, 2, 2, w.shape[-1]) + .permute(0, 2, 1, 3, 4) + .reshape(w.shape[0], w.shape[-1])) + + data_torch = torch.cat([deinterleave(wq), deinterleave(wk), wv], dim=0) + + if "pos_emb.weight" in name: + # kept 3D: the runtime reads grid extents from ne[1]/ne[2] + pass + + if "mm_projector.proj.0." in name: + name = name.replace(".proj.0.", ".proj.linear_1.") + elif "mm_projector.proj.2." in name: + name = name.replace(".proj.2.", ".proj.linear_2.") + + yield from super().modify_tensors(data_torch, name, bid) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index a1c0843be115..925d6af1382f 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -4991,6 +4991,7 @@ class VisionProjectorType: KIMIVL = "kimivl" PADDLEOCR = "paddleocr" KIMIK25 = "kimik25" + KIMIK3 = "kimik3" LIGHTONOCR = "lightonocr" COGVLM = "cogvlm" JANUS_PRO = "janus_pro" diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 461c4f429cfb..490235b1b548 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -39,6 +39,7 @@ add_library(mtmd models/hunyuanvl.cpp models/internvl.cpp models/inkling.cpp + models/kimik3.cpp models/kimivl.cpp models/kimik25.cpp models/nemotron-v2-vl.cpp diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 47ef161ab8a7..7844a6a81f96 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -406,6 +406,7 @@ enum projector_type { PROJECTOR_TYPE_YOUTUVL, PROJECTOR_TYPE_YASA2, PROJECTOR_TYPE_KIMIK25, + PROJECTOR_TYPE_KIMIK3, PROJECTOR_TYPE_NEMOTRON_V2_VL, PROJECTOR_TYPE_HUNYUANVL, PROJECTOR_TYPE_PARAKEET, @@ -464,6 +465,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_YOUTUVL, "youtuvl"}, { PROJECTOR_TYPE_YASA2, "yasa2"}, { PROJECTOR_TYPE_KIMIK25, "kimik25"}, + { PROJECTOR_TYPE_KIMIK3, "kimik3"}, { PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"}, { PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"}, { PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"}, diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index fd9958981baf..c0c0e2104b1f 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -977,6 +977,10 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_KIMIK3: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_COGVLM: { builder = std::make_unique(ctx, img); @@ -1462,6 +1466,23 @@ struct clip_model_loader { hparams.rope_theta = 10000.0f; get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + int min_pixels = 0, max_pixels = 0; + get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels, false); + get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels, false); + if (min_pixels > 0 && max_pixels > 0) { + hparams.image_min_pixels = min_pixels; + hparams.image_max_pixels = max_pixels; + hparams.warmup_image_size = static_cast(std::sqrt(max_pixels)); + } else { + hparams.set_limit_image_tokens(2, 4096); + } + } break; + case PROJECTOR_TYPE_KIMIK3: + { + hparams.image_resize_algo = RESIZE_ALGO_BILINEAR; + hparams.rope_theta = 10000.0f; + get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); + int min_pixels = 0, max_pixels = 0; get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels, false); get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels, false); @@ -2487,6 +2508,13 @@ struct clip_model_loader { model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); } break; + case PROJECTOR_TYPE_KIMIK3: + { + // patchmergerv2, bias-free, norm after the projection + model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_post_norm_w = get_tensor(string_format(TN_MM_POST_NORM, "weight")); + } break; case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_KIMIK25: @@ -3692,6 +3720,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_LFM2: case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_KIMIK3: { // dynamic size int out_patch_size = params.patch_size * ctx->model.hparams.n_merge; @@ -4397,6 +4426,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 case PROJECTOR_TYPE_PIXTRAL: case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_KIMIK3: case PROJECTOR_TYPE_LIGHTONOCR: { // set the 2D positions @@ -5088,6 +5118,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_KIMIVL: case PROJECTOR_TYPE_PADDLEOCR: case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_KIMIK3: case PROJECTOR_TYPE_YASA2: return ctx->model.mm_2_w->ne[1]; case PROJECTOR_TYPE_HUNYUANVL: diff --git a/tools/mtmd/models/kimik3.cpp b/tools/mtmd/models/kimik3.cpp new file mode 100644 index 000000000000..fe06c7fdbb38 --- /dev/null +++ b/tools/mtmd/models/kimik3.cpp @@ -0,0 +1,80 @@ +#include "models.h" + +#include +#include + +// Kimi-K3 MoonViT-3d, image path. +// Follows clip_graph_kimik25, but with RMSNorm, no biases, qkv width != n_embd, and a post-norm patchmergerv2 projector. +// Images only: at t == 1 the temporal pool and the temporal position term vanish. + +ggml_tensor * clip_graph_kimik3::resize_position_embeddings_3d(uint32_t interpolation_mode) { + ggml_tensor * pos_embd = model.position_embeddings; + const int height = img.ny() / patch_size; + const int width = img.nx() / patch_size; + + GGML_ASSERT(pos_embd); + + const int64_t stored_c = pos_embd->ne[0]; + const int64_t orig_w = pos_embd->ne[1]; + const int64_t orig_h = pos_embd->ne[2]; + + GGML_ASSERT(stored_c == n_embd); + + if (height == (int) orig_h && width == (int) orig_w) { + return ggml_cont_2d(ctx0, pos_embd, n_embd, width * height); + } + + pos_embd = ggml_permute(ctx0, pos_embd, 2, 1, 0, 3); + pos_embd = ggml_interpolate(ctx0, pos_embd, height, width, n_embd, 1, interpolation_mode); + pos_embd = ggml_permute(ctx0, pos_embd, 2, 1, 0, 3); + pos_embd = ggml_cont_2d(ctx0, pos_embd, n_embd, width * height); + return pos_embd; +} + +ggml_cgraph * clip_graph_kimik3::build() { + ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches); + ggml_set_name(pos_h, "pos_h"); + ggml_set_input(pos_h); + + ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches); + ggml_set_name(pos_w, "pos_w"); + ggml_set_input(pos_w); + + ggml_tensor * learned_pos_embd = resize_position_embeddings_3d(GGML_SCALE_MODE_BILINEAR); + + // Q/K are de-interleaved during conversion. + auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + return build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false); + }; + + ggml_tensor * inp = build_inp(); + inp = ggml_add(ctx0, inp, learned_pos_embd); + + ggml_tensor * cur = build_vit( + inp, n_patches, + NORM_TYPE_RMS, + hparams.ffn_op, + nullptr, + add_pos); + cb(cur, "vit_out", -1); + + { + const int scale_factor = model.hparams.n_merge; + cur = build_patch_merge_permute(cur, scale_factor); + + cur = build_ffn(cur, + model.mm_1_w, nullptr, + nullptr, nullptr, + model.mm_2_w, nullptr, + FFN_GELU, + -1); + cb(cur, "proj_mlp_out", -1); + + cur = build_norm(cur, model.mm_post_norm_w, nullptr, NORM_TYPE_RMS, hparams.eps, -1); + cb(cur, "proj_out", -1); + } + + ggml_build_forward_expand(gf, cur); + + return gf; +} diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 6735c9cdebf9..0a969dc56681 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -238,6 +238,13 @@ struct clip_graph_parakeet : clip_graph { ggml_cgraph * build() override; }; +struct clip_graph_kimik3 : clip_graph { + clip_graph_kimik3(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; + + ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode); +}; + struct clip_graph_exaone4_5 : clip_graph { clip_graph_exaone4_5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index b16c10e2dc2f..34ae8c071b4b 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -567,6 +567,7 @@ struct mtmd_context { image_preproc = std::make_unique(ctx_v); } break; case PROJECTOR_TYPE_KIMIK25: + case PROJECTOR_TYPE_KIMIK3: { // GLM-5.2-V reuses the Kimi-K2.5 vision encoder and projector, but marks // images with its own tokens, so decide based on the text model vocab