feat: share multimodal hash helpers

lmdeploy/pytorch/engine/engine.py

@@ -23,7 +23,6 @@
 from ..adapter.adapter import AdapterManager
 from ..config import CacheConfig, ModelConfig
 from ..messages import MessageStatus, SchedulerSequence, UpdateTokenMode
-from ..multimodal.data_type import ensure_multimodal_content_hashes
 from ..paging import Scheduler
 from ..strategies import build_strategy_factory
 from .base import EngineBase
@@ -416,8 +415,6 @@ def _on_add_message(self, reqs: list[Request], **kwargs):
 
             input_ids = result.input_ids
             input_multimodals = result.input_multimodals
-            if self.cache_config.enable_prefix_caching:
-                input_multimodals = ensure_multimodal_content_hashes(input_multimodals)
 
             req_data['token_ids'] = input_ids
             req_data['input_multimodals'] = input_multimodals

lmdeploy/pytorch/messages.py

@@ -10,9 +10,10 @@
 
 from lmdeploy.messages import EngineEvent, EventType, GenerationConfig, LogitsProcessor
 from lmdeploy.pytorch.disagg.conn.protocol import MigrationRequest
-from lmdeploy.pytorch.multimodal.data_type import MultiModalInputs, make_multimodal_content_hash
+from lmdeploy.pytorch.multimodal.data_type import MultiModalInputs
 from lmdeploy.utils import get_logger
 from lmdeploy.vl.constants import Modality
+from lmdeploy.vl.hasher import make_multimodal_content_hash
 
 from .block import LogicalTokenBlocks
 
@@ -1035,8 +1036,8 @@ def _update_prefix_cache_metas(self, multimodals: MultiModalInputs):
                     modality = modality.value
                 content_hash = modal_data.content_hash
                 if content_hash is None:
-                    # Most request paths precompute the hash after model
-                    # preprocessing.  Keep this fallback for unit tests and
+                    # Most request paths carry a hash from the shared VL path.
+                    # Keep this fallback for unit tests and
                     # defensive correctness if a processor omits it.
                     content_hash = make_multimodal_content_hash(modal_data.data, modal_data.meta,
                                                                 modal_data.mrope_pos_ids)

lmdeploy/pytorch/models/chatglm2.py

@@ -883,7 +883,8 @@ def preprocess_input(self,
             mm_data = MultiModalData(data=pixel_values,
                                      start=offset,
                                      end=offset + num_pad,
-                                     meta=dict(image_token_id=image_token_id))
+                                     meta=dict(image_token_id=image_token_id),
+                                     content_hash=input_mm.get('content_hash'))
             input_imgs.append(mm_data)
 
         result = PreprocessInputResult(

lmdeploy/pytorch/models/cogvlm.py

@@ -909,7 +909,8 @@ def preprocess_input(self, input_ids: list[int], input_multimodals=None, **kwarg
             mm_data = MultiModalData(data=pixel_values,
                                      start=offset,
                                      end=offset + num_pad,
-                                     meta=dict(image_token_id=image_token_id))
+                                     meta=dict(image_token_id=image_token_id),
+                                     content_hash=input_mm.get('content_hash'))
             input_imgs.append(mm_data)
 
         result = PreprocessInputResult(

lmdeploy/pytorch/models/deepseek_vl2.py

@@ -447,7 +447,8 @@ def preprocess_input(self,
                                      meta=dict(
                                          image_token_id=image_token_id,
                                          images_spatial_crop=images_spatial_crop,
-                                     ))
+                                     ),
+                                     content_hash=input_mm.get('content_hash'))
 
             input_imgs.append(mm_data)
 

lmdeploy/pytorch/models/gemma3_vl.py

@@ -112,7 +112,8 @@ def preprocess_input(self,
             mm_data = MultiModalData(data=pixel_values,
                                      start=offset,
                                      end=offset + num_pad,
-                                     meta=dict(image_token_id=image_token_id))
+                                     meta=dict(image_token_id=image_token_id),
+                                     content_hash=input_mm.get('content_hash'))
             input_imgs.append(mm_data)
 
         result = PreprocessInputResult(

lmdeploy/pytorch/models/glm4_1v.py

@@ -745,7 +745,8 @@ def preprocess_input(self,
                                      start=offset[0],
                                      end=offset[1],
                                      mrope_pos_ids=mrope_pos_ids,
-                                     meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id))
+                                     meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id),
+                                     content_hash=input_mm.get('content_hash'))
             input_imgs.append(mm_data)
 
         result = PreprocessInputResult(

lmdeploy/pytorch/models/interns1_pro.py

@@ -374,7 +374,8 @@ def _make_image_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData:
                                  data=pixel_values,
                                  start=offset[0],
                                  end=offset[1],
-                                 meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id))
+                                 meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id),
+                                 content_hash=input_mm.get('content_hash'))
         return mm_data
 
     def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData:
@@ -391,7 +392,8 @@ def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData:
                                  meta=dict(
                                      grid_thw=video_grid_thw,
                                      video_token_id=video_token_id,
-                                 ))
+                                 ),
+                                 content_hash=input_mm.get('content_hash'))
         return mm_data
 
     def _make_time_series_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData:
@@ -406,7 +408,8 @@ def _make_time_series_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData:
                                  data=ts_values,
                                  start=offset[0],
                                  end=offset[1],
-                                 meta=dict(ts_lens=ts_lens, ts_sr=ts_sr, ts_token_id=ts_token_id))
+                                 meta=dict(ts_lens=ts_lens, ts_sr=ts_sr, ts_token_id=ts_token_id),
+                                 content_hash=input_mm.get('content_hash'))
         return mm_data
 
     def preprocess_input(self,

lmdeploy/pytorch/models/internvl.py

@@ -946,7 +946,8 @@ def preprocess_input(self,
             mm_data = MultiModalData(data=pixel_values,
                                      start=offset,
                                      end=offset + num_pad,
-                                     meta=dict(image_token_id=image_token_id))
+                                     meta=dict(image_token_id=image_token_id),
+                                     content_hash=input_mm.get('content_hash'))
             input_imgs.append(mm_data)
 
         result = PreprocessInputResult(

lmdeploy/pytorch/models/internvl3_hf.py

@@ -740,7 +740,8 @@ def preprocess_input(self,
             mm_data = MultiModalData(data=pixel_values,
                                      start=offset,
                                      end=offset + num_pad,
-                                     meta=dict(image_token_id=image_token_id))
+                                     meta=dict(image_token_id=image_token_id),
+                                     content_hash=input_mm.get('content_hash'))
             input_imgs.append(mm_data)
 
         result = PreprocessInputResult(

lmdeploy/pytorch/models/llama4.py

@@ -1041,7 +1041,8 @@ def preprocess_input(self,
             mm_data = MultiModalData(data=pixel_values,
                                      start=offset,
                                      end=offset + num_pad,
-                                     meta=dict(image_token_id=image_token_id))
+                                     meta=dict(image_token_id=image_token_id),
+                                     content_hash=input_mm.get('content_hash'))
             input_imgs.append(mm_data)
 
         result = PreprocessInputResult(

lmdeploy/pytorch/models/llava.py

@@ -559,7 +559,8 @@ def preprocess_input(self,
             mm_data = MultiModalData(data=pixel_values,
                                      start=offset,
                                      end=offset + num_pad,
-                                     meta=dict(image_token_id=image_token_id))
+                                     meta=dict(image_token_id=image_token_id),
+                                     content_hash=input_mm.get('content_hash'))
             input_imgs.append(mm_data)
 
         result = PreprocessInputResult(
@@ -838,7 +839,8 @@ def preprocess_input(self,
             mm_data = MultiModalData(data=pixel_values,
                                      start=offset,
                                      end=offset + num_pad,
-                                     meta=dict(image_sizes=image_sizes, image_token_id=image_token_id))
+                                     meta=dict(image_sizes=image_sizes, image_token_id=image_token_id),
+                                     content_hash=input_mm.get('content_hash'))
             input_imgs.append(mm_data)
 
         result = PreprocessInputResult(

lmdeploy/pytorch/models/phi3_v.py

@@ -383,7 +383,8 @@ def preprocess_input(self,
             mm_data = MultiModalData(data=pixel_values,
                                      start=offset,
                                      end=offset + num_pad,
-                                     meta=dict(image_sizes=image_sizes, image_token_id=image_token_id))
+                                     meta=dict(image_sizes=image_sizes, image_token_id=image_token_id),
+                                     content_hash=input_mm.get('content_hash'))
             input_imgs.append(mm_data)
 
         result = PreprocessInputResult(

lmdeploy/pytorch/models/qwen2_vl.py

@@ -814,7 +814,8 @@ def preprocess_input(self,
                                      start=start,
                                      end=start + num_pad,
                                      mrope_pos_ids=mrope_pos_ids,
-                                     meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id))
+                                     meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id),
+                                     content_hash=input_mm.get('content_hash'))
             input_imgs.append(mm_data)
 
         result = PreprocessInputResult(

lmdeploy/pytorch/models/qwen3_omni_moe_thinker.py

@@ -846,7 +846,8 @@ def _make_image_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData:
                                  start=offset[0],
                                  end=offset[1],
                                  mrope_pos_ids=mrope_pos_ids,
-                                 meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id))
+                                 meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id),
+                                 content_hash=input_mm.get('content_hash'))
         return mm_data
 
     def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData:
@@ -867,7 +868,8 @@ def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData:
                                      grid_thw=video_grid_thw,
                                      video_token_id=video_token_id,
                                      second_per_grid=input_mm.get('second_per_grid'),
-                                 ))
+                                 ),
+                                 content_hash=input_mm.get('content_hash'))
         return mm_data
 
     def _make_audio_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData:
@@ -896,7 +898,8 @@ def _make_audio_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData:
                                  meta=dict(
                                      audio_token_id=audio_token_id,
                                      audio_feature_lengths=audio_feature_lengths,
-                                 ))
+                                 ),
+                                 content_hash=input_mm.get('content_hash'))
         return mm_data
 
     def preprocess_input(self,

lmdeploy/pytorch/models/qwen3_vl.py

@@ -682,7 +682,8 @@ def _make_image_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData:
                                  start=offset[0],
                                  end=offset[1],
                                  mrope_pos_ids=mrope_pos_ids,
-                                 meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id))
+                                 meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id),
+                                 content_hash=input_mm.get('content_hash'))
         return mm_data
 
     def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData:
@@ -702,7 +703,8 @@ def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData:
                                  meta=dict(
                                      grid_thw=video_grid_thw,
                                      video_token_id=video_token_id,
-                                 ))
+                                 ),
+                                 content_hash=input_mm.get('content_hash'))
         return mm_data
 
     def _make_time_series_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData:
@@ -717,7 +719,8 @@ def _make_time_series_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData:
                                  data=ts_values,
                                  start=offset[0],
                                  end=offset[1],
-                                 meta=dict(ts_lens=ts_lens, ts_sr=ts_sr, ts_token_id=ts_token_id))
+                                 meta=dict(ts_lens=ts_lens, ts_sr=ts_sr, ts_token_id=ts_token_id),
+                                 content_hash=input_mm.get('content_hash'))
         return mm_data
 
     def preprocess_input(self,

lmdeploy/pytorch/multimodal/data_type.py

@@ -1,58 +1,15 @@
 # Copyright (c) OpenMMLab. All rights reserved.
-import enum
-import hashlib
 from dataclasses import dataclass, fields
 from typing import Any
 
 import numpy as np
-import torch
 from torch import Tensor
 
 from lmdeploy.vl.constants import Modality
 
 NestedTensor = Tensor | list[Tensor]
 
 
-def _hash_multimodal_value(hasher: 'hashlib._Hash', value: Any):
-    """Update a hash with a deterministic multimodal value representation."""
-    if isinstance(value, Tensor):
-        tensor = value.detach().cpu().contiguous()
-        hasher.update(f'tensor:{tensor.dtype}:{tuple(tensor.shape)}:'.encode())
-        hasher.update(tensor.view(torch.uint8).numpy().tobytes())
-    elif isinstance(value, np.ndarray):
-        array = np.ascontiguousarray(value)
-        hasher.update(f'ndarray:{array.dtype}:{array.shape}:'.encode())
-        hasher.update(array.tobytes())
-    elif isinstance(value, dict):
-        hasher.update(b'dict:{')
-        for key in sorted(value, key=lambda x: repr(x)):
-            _hash_multimodal_value(hasher, key)
-            hasher.update(b':')
-            _hash_multimodal_value(hasher, value[key])
-            hasher.update(b',')
-        hasher.update(b'}')
-    elif isinstance(value, (list, tuple)):
-        hasher.update(f'{type(value).__name__}:['.encode())
-        for item in value:
-            _hash_multimodal_value(hasher, item)
-            hasher.update(b',')
-        hasher.update(b']')
-    elif isinstance(value, enum.Enum):
-        _hash_multimodal_value(hasher, value.value)
-    else:
-        hasher.update(f'{type(value).__name__}:{repr(value)}'.encode())
-
-
-def make_multimodal_content_hash(data: Any, meta: dict[str, Any] | None,
-                                 mrope_pos_ids: np.ndarray | None = None) -> str:
-    """Create a stable content hash for prefix-cache multimodal matching."""
-    hasher = hashlib.sha256()
-    _hash_multimodal_value(hasher, data)
-    _hash_multimodal_value(hasher, meta)
-    _hash_multimodal_value(hasher, mrope_pos_ids)
-    return hasher.hexdigest()
-
-
 @dataclass
 class MultiModalData:
     data: NestedTensor
@@ -101,16 +58,3 @@ def to_device(self, device: str, non_blocking: bool = False):
 
 
 MultiModalInputs = dict[str, list[MultiModalData]]
-
-
-def ensure_multimodal_content_hashes(input_mms: MultiModalInputs | None):
-    """Populate missing multimodal content hashes in-place."""
-    if input_mms is None:
-        return input_mms
-
-    for modal_datas in input_mms.values():
-        for modal_data in modal_datas:
-            if modal_data.content_hash is None:
-                modal_data.content_hash = make_multimodal_content_hash(modal_data.data, modal_data.meta,
-                                                                       modal_data.mrope_pos_ids)
-    return input_mms

lmdeploy/vl/engine.py

@@ -10,6 +10,7 @@
 from lmdeploy.messages import PytorchEngineConfig, TurbomindEngineConfig, VisionConfig
 from lmdeploy.utils import is_bf16_supported
 from lmdeploy.vl.model.builder import load_vl_model
+from lmdeploy.vl.model.preprocess_utils import attach_multimodal_content_hashes
 
 
 def _get_hf_config_mm_feature_dtype(hf_config) -> torch.dtype | None:
@@ -119,6 +120,7 @@ async def preprocess(self,
                 self.executor, self.model.preprocess, messages)
         future.add_done_callback(_raise_exception_on_finish)
         outputs = await future
+        attach_multimodal_content_hashes(outputs)
         return outputs
 
     async def async_infer(self, messages: list[dict]) -> list[dict]:

lmdeploy/vl/hasher.py

@@ -0,0 +1,50 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+import enum
+import hashlib
+from collections.abc import Mapping
+from typing import Any
+
+import numpy as np
+import torch
+from torch import Tensor
+
+
+def _hash_multimodal_value(hasher: 'hashlib._Hash', value: Any):
+    """Update a hash with a deterministic multimodal value representation."""
+    if isinstance(value, Tensor):
+        tensor = value.detach().cpu().contiguous()
+        hasher.update(f'tensor:{tensor.dtype}:{tuple(tensor.shape)}:'.encode())
+        hasher.update(tensor.view(torch.uint8).numpy().tobytes())
+    elif isinstance(value, np.ndarray):
+        array = np.ascontiguousarray(value)
+        hasher.update(f'ndarray:{array.dtype}:{array.shape}:'.encode())
+        hasher.update(array.tobytes())
+    elif isinstance(value, Mapping):
+        hasher.update(b'dict:{')
+        for key in sorted(value, key=lambda x: repr(x)):
+            _hash_multimodal_value(hasher, key)
+            hasher.update(b':')
+            _hash_multimodal_value(hasher, value[key])
+            hasher.update(b',')
+        hasher.update(b'}')
+    elif isinstance(value, (list, tuple)):
+        hasher.update(f'{type(value).__name__}:['.encode())
+        for item in value:
+            _hash_multimodal_value(hasher, item)
+            hasher.update(b',')
+        hasher.update(b']')
+    elif isinstance(value, enum.Enum):
+        _hash_multimodal_value(hasher, value.value)
+    else:
+        hasher.update(f'{type(value).__name__}:{repr(value)}'.encode())
+
+
+def make_multimodal_content_hash(data: Any,
+                                 meta: Mapping[str, Any] | None = None,
+                                 mrope_pos_ids: np.ndarray | None = None) -> str:
+    """Create a stable content hash for multimodal content identity."""
+    hasher = hashlib.sha256()
+    _hash_multimodal_value(hasher, data)
+    _hash_multimodal_value(hasher, meta)
+    _hash_multimodal_value(hasher, mrope_pos_ids)
+    return hasher.hexdigest()