diff --git a/lmdeploy/pytorch/engine/engine.py b/lmdeploy/pytorch/engine/engine.py index 3b3fd2e117..756eab80b9 100644 --- a/lmdeploy/pytorch/engine/engine.py +++ b/lmdeploy/pytorch/engine/engine.py @@ -23,7 +23,6 @@ from ..adapter.adapter import AdapterManager from ..config import CacheConfig, ModelConfig from ..messages import MessageStatus, SchedulerSequence, UpdateTokenMode -from ..multimodal.data_type import ensure_multimodal_content_hashes from ..paging import Scheduler from ..strategies import build_strategy_factory from .base import EngineBase @@ -416,8 +415,6 @@ def _on_add_message(self, reqs: list[Request], **kwargs): input_ids = result.input_ids input_multimodals = result.input_multimodals - if self.cache_config.enable_prefix_caching: - input_multimodals = ensure_multimodal_content_hashes(input_multimodals) req_data['token_ids'] = input_ids req_data['input_multimodals'] = input_multimodals diff --git a/lmdeploy/pytorch/messages.py b/lmdeploy/pytorch/messages.py index b1c5a59e68..76493e880a 100644 --- a/lmdeploy/pytorch/messages.py +++ b/lmdeploy/pytorch/messages.py @@ -10,9 +10,10 @@ from lmdeploy.messages import EngineEvent, EventType, GenerationConfig, LogitsProcessor from lmdeploy.pytorch.disagg.conn.protocol import MigrationRequest -from lmdeploy.pytorch.multimodal.data_type import MultiModalInputs, make_multimodal_content_hash +from lmdeploy.pytorch.multimodal.data_type import MultiModalInputs from lmdeploy.utils import get_logger from lmdeploy.vl.constants import Modality +from lmdeploy.vl.hasher import make_multimodal_content_hash from .block import LogicalTokenBlocks @@ -1035,8 +1036,8 @@ def _update_prefix_cache_metas(self, multimodals: MultiModalInputs): modality = modality.value content_hash = modal_data.content_hash if content_hash is None: - # Most request paths precompute the hash after model - # preprocessing. Keep this fallback for unit tests and + # Most request paths carry a hash from the shared VL path. + # Keep this fallback for unit tests and # defensive correctness if a processor omits it. content_hash = make_multimodal_content_hash(modal_data.data, modal_data.meta, modal_data.mrope_pos_ids) diff --git a/lmdeploy/pytorch/models/chatglm2.py b/lmdeploy/pytorch/models/chatglm2.py index 2988f3c9ed..b506b9f276 100644 --- a/lmdeploy/pytorch/models/chatglm2.py +++ b/lmdeploy/pytorch/models/chatglm2.py @@ -883,7 +883,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_token_id=image_token_id)) + meta=dict(image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/cogvlm.py b/lmdeploy/pytorch/models/cogvlm.py index 98d2f02583..fa6c874dd5 100644 --- a/lmdeploy/pytorch/models/cogvlm.py +++ b/lmdeploy/pytorch/models/cogvlm.py @@ -909,7 +909,8 @@ def preprocess_input(self, input_ids: list[int], input_multimodals=None, **kwarg mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_token_id=image_token_id)) + meta=dict(image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/deepseek_vl2.py b/lmdeploy/pytorch/models/deepseek_vl2.py index b556ffc2b8..b881849c0e 100644 --- a/lmdeploy/pytorch/models/deepseek_vl2.py +++ b/lmdeploy/pytorch/models/deepseek_vl2.py @@ -447,7 +447,8 @@ def preprocess_input(self, meta=dict( image_token_id=image_token_id, images_spatial_crop=images_spatial_crop, - )) + ), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) diff --git a/lmdeploy/pytorch/models/gemma3_vl.py b/lmdeploy/pytorch/models/gemma3_vl.py index 9f444e81f5..a0eb3c9241 100644 --- a/lmdeploy/pytorch/models/gemma3_vl.py +++ b/lmdeploy/pytorch/models/gemma3_vl.py @@ -112,7 +112,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_token_id=image_token_id)) + meta=dict(image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/glm4_1v.py b/lmdeploy/pytorch/models/glm4_1v.py index ac802fe6a2..698181a5bc 100644 --- a/lmdeploy/pytorch/models/glm4_1v.py +++ b/lmdeploy/pytorch/models/glm4_1v.py @@ -745,7 +745,8 @@ def preprocess_input(self, start=offset[0], end=offset[1], mrope_pos_ids=mrope_pos_ids, - meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id)) + meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/interns1_pro.py b/lmdeploy/pytorch/models/interns1_pro.py index 9a03f30640..f863db270e 100644 --- a/lmdeploy/pytorch/models/interns1_pro.py +++ b/lmdeploy/pytorch/models/interns1_pro.py @@ -374,7 +374,8 @@ def _make_image_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: data=pixel_values, start=offset[0], end=offset[1], - meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id)) + meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) return mm_data def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: @@ -391,7 +392,8 @@ def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: meta=dict( grid_thw=video_grid_thw, video_token_id=video_token_id, - )) + ), + content_hash=input_mm.get('content_hash')) return mm_data def _make_time_series_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: @@ -406,7 +408,8 @@ def _make_time_series_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: data=ts_values, start=offset[0], end=offset[1], - meta=dict(ts_lens=ts_lens, ts_sr=ts_sr, ts_token_id=ts_token_id)) + meta=dict(ts_lens=ts_lens, ts_sr=ts_sr, ts_token_id=ts_token_id), + content_hash=input_mm.get('content_hash')) return mm_data def preprocess_input(self, diff --git a/lmdeploy/pytorch/models/internvl.py b/lmdeploy/pytorch/models/internvl.py index 2a2f53dae3..9243e62639 100644 --- a/lmdeploy/pytorch/models/internvl.py +++ b/lmdeploy/pytorch/models/internvl.py @@ -946,7 +946,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_token_id=image_token_id)) + meta=dict(image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/internvl3_hf.py b/lmdeploy/pytorch/models/internvl3_hf.py index 09f159c22a..6c0f3ad00e 100644 --- a/lmdeploy/pytorch/models/internvl3_hf.py +++ b/lmdeploy/pytorch/models/internvl3_hf.py @@ -740,7 +740,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_token_id=image_token_id)) + meta=dict(image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/llama4.py b/lmdeploy/pytorch/models/llama4.py index 2552467218..7c507be9ad 100644 --- a/lmdeploy/pytorch/models/llama4.py +++ b/lmdeploy/pytorch/models/llama4.py @@ -1041,7 +1041,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_token_id=image_token_id)) + meta=dict(image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/llava.py b/lmdeploy/pytorch/models/llava.py index bf0618ec9a..71150df4cd 100644 --- a/lmdeploy/pytorch/models/llava.py +++ b/lmdeploy/pytorch/models/llava.py @@ -559,7 +559,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_token_id=image_token_id)) + meta=dict(image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( @@ -838,7 +839,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_sizes=image_sizes, image_token_id=image_token_id)) + meta=dict(image_sizes=image_sizes, image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/phi3_v.py b/lmdeploy/pytorch/models/phi3_v.py index 589e04e20d..3d4cb618a2 100644 --- a/lmdeploy/pytorch/models/phi3_v.py +++ b/lmdeploy/pytorch/models/phi3_v.py @@ -383,7 +383,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_sizes=image_sizes, image_token_id=image_token_id)) + meta=dict(image_sizes=image_sizes, image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/qwen2_vl.py b/lmdeploy/pytorch/models/qwen2_vl.py index ff2bdb2e7d..7fede4d37c 100644 --- a/lmdeploy/pytorch/models/qwen2_vl.py +++ b/lmdeploy/pytorch/models/qwen2_vl.py @@ -814,7 +814,8 @@ def preprocess_input(self, start=start, end=start + num_pad, mrope_pos_ids=mrope_pos_ids, - meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id)) + meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/qwen3_omni_moe_thinker.py b/lmdeploy/pytorch/models/qwen3_omni_moe_thinker.py index 733c323dbf..7a4c6cf7f3 100644 --- a/lmdeploy/pytorch/models/qwen3_omni_moe_thinker.py +++ b/lmdeploy/pytorch/models/qwen3_omni_moe_thinker.py @@ -846,7 +846,8 @@ def _make_image_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: start=offset[0], end=offset[1], mrope_pos_ids=mrope_pos_ids, - meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id)) + meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) return mm_data def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: @@ -867,7 +868,8 @@ def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: grid_thw=video_grid_thw, video_token_id=video_token_id, second_per_grid=input_mm.get('second_per_grid'), - )) + ), + content_hash=input_mm.get('content_hash')) return mm_data def _make_audio_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: @@ -896,7 +898,8 @@ def _make_audio_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: meta=dict( audio_token_id=audio_token_id, audio_feature_lengths=audio_feature_lengths, - )) + ), + content_hash=input_mm.get('content_hash')) return mm_data def preprocess_input(self, diff --git a/lmdeploy/pytorch/models/qwen3_vl.py b/lmdeploy/pytorch/models/qwen3_vl.py index db3f69af55..d9053a8088 100644 --- a/lmdeploy/pytorch/models/qwen3_vl.py +++ b/lmdeploy/pytorch/models/qwen3_vl.py @@ -682,7 +682,8 @@ def _make_image_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: start=offset[0], end=offset[1], mrope_pos_ids=mrope_pos_ids, - meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id)) + meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) return mm_data def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: @@ -702,7 +703,8 @@ def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: meta=dict( grid_thw=video_grid_thw, video_token_id=video_token_id, - )) + ), + content_hash=input_mm.get('content_hash')) return mm_data def _make_time_series_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: @@ -717,7 +719,8 @@ def _make_time_series_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: data=ts_values, start=offset[0], end=offset[1], - meta=dict(ts_lens=ts_lens, ts_sr=ts_sr, ts_token_id=ts_token_id)) + meta=dict(ts_lens=ts_lens, ts_sr=ts_sr, ts_token_id=ts_token_id), + content_hash=input_mm.get('content_hash')) return mm_data def preprocess_input(self, diff --git a/lmdeploy/pytorch/multimodal/data_type.py b/lmdeploy/pytorch/multimodal/data_type.py index f778c2aeb5..070f35867e 100644 --- a/lmdeploy/pytorch/multimodal/data_type.py +++ b/lmdeploy/pytorch/multimodal/data_type.py @@ -1,11 +1,8 @@ # Copyright (c) OpenMMLab. All rights reserved. -import enum -import hashlib from dataclasses import dataclass, fields from typing import Any import numpy as np -import torch from torch import Tensor from lmdeploy.vl.constants import Modality @@ -13,46 +10,6 @@ NestedTensor = Tensor | list[Tensor] -def _hash_multimodal_value(hasher: 'hashlib._Hash', value: Any): - """Update a hash with a deterministic multimodal value representation.""" - if isinstance(value, Tensor): - tensor = value.detach().cpu().contiguous() - hasher.update(f'tensor:{tensor.dtype}:{tuple(tensor.shape)}:'.encode()) - hasher.update(tensor.view(torch.uint8).numpy().tobytes()) - elif isinstance(value, np.ndarray): - array = np.ascontiguousarray(value) - hasher.update(f'ndarray:{array.dtype}:{array.shape}:'.encode()) - hasher.update(array.tobytes()) - elif isinstance(value, dict): - hasher.update(b'dict:{') - for key in sorted(value, key=lambda x: repr(x)): - _hash_multimodal_value(hasher, key) - hasher.update(b':') - _hash_multimodal_value(hasher, value[key]) - hasher.update(b',') - hasher.update(b'}') - elif isinstance(value, (list, tuple)): - hasher.update(f'{type(value).__name__}:['.encode()) - for item in value: - _hash_multimodal_value(hasher, item) - hasher.update(b',') - hasher.update(b']') - elif isinstance(value, enum.Enum): - _hash_multimodal_value(hasher, value.value) - else: - hasher.update(f'{type(value).__name__}:{repr(value)}'.encode()) - - -def make_multimodal_content_hash(data: Any, meta: dict[str, Any] | None, - mrope_pos_ids: np.ndarray | None = None) -> str: - """Create a stable content hash for prefix-cache multimodal matching.""" - hasher = hashlib.sha256() - _hash_multimodal_value(hasher, data) - _hash_multimodal_value(hasher, meta) - _hash_multimodal_value(hasher, mrope_pos_ids) - return hasher.hexdigest() - - @dataclass class MultiModalData: data: NestedTensor @@ -101,16 +58,3 @@ def to_device(self, device: str, non_blocking: bool = False): MultiModalInputs = dict[str, list[MultiModalData]] - - -def ensure_multimodal_content_hashes(input_mms: MultiModalInputs | None): - """Populate missing multimodal content hashes in-place.""" - if input_mms is None: - return input_mms - - for modal_datas in input_mms.values(): - for modal_data in modal_datas: - if modal_data.content_hash is None: - modal_data.content_hash = make_multimodal_content_hash(modal_data.data, modal_data.meta, - modal_data.mrope_pos_ids) - return input_mms diff --git a/lmdeploy/vl/engine.py b/lmdeploy/vl/engine.py index f2a5f62ccf..5ce37fda69 100644 --- a/lmdeploy/vl/engine.py +++ b/lmdeploy/vl/engine.py @@ -10,6 +10,7 @@ from lmdeploy.messages import PytorchEngineConfig, TurbomindEngineConfig, VisionConfig from lmdeploy.utils import is_bf16_supported from lmdeploy.vl.model.builder import load_vl_model +from lmdeploy.vl.model.preprocess_utils import attach_multimodal_content_hashes def _get_hf_config_mm_feature_dtype(hf_config) -> torch.dtype | None: @@ -119,6 +120,7 @@ async def preprocess(self, self.executor, self.model.preprocess, messages) future.add_done_callback(_raise_exception_on_finish) outputs = await future + attach_multimodal_content_hashes(outputs) return outputs async def async_infer(self, messages: list[dict]) -> list[dict]: diff --git a/lmdeploy/vl/hasher.py b/lmdeploy/vl/hasher.py new file mode 100644 index 0000000000..500449bb08 --- /dev/null +++ b/lmdeploy/vl/hasher.py @@ -0,0 +1,50 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import enum +import hashlib +from collections.abc import Mapping +from typing import Any + +import numpy as np +import torch +from torch import Tensor + + +def _hash_multimodal_value(hasher: 'hashlib._Hash', value: Any): + """Update a hash with a deterministic multimodal value representation.""" + if isinstance(value, Tensor): + tensor = value.detach().cpu().contiguous() + hasher.update(f'tensor:{tensor.dtype}:{tuple(tensor.shape)}:'.encode()) + hasher.update(tensor.view(torch.uint8).numpy().tobytes()) + elif isinstance(value, np.ndarray): + array = np.ascontiguousarray(value) + hasher.update(f'ndarray:{array.dtype}:{array.shape}:'.encode()) + hasher.update(array.tobytes()) + elif isinstance(value, Mapping): + hasher.update(b'dict:{') + for key in sorted(value, key=lambda x: repr(x)): + _hash_multimodal_value(hasher, key) + hasher.update(b':') + _hash_multimodal_value(hasher, value[key]) + hasher.update(b',') + hasher.update(b'}') + elif isinstance(value, (list, tuple)): + hasher.update(f'{type(value).__name__}:['.encode()) + for item in value: + _hash_multimodal_value(hasher, item) + hasher.update(b',') + hasher.update(b']') + elif isinstance(value, enum.Enum): + _hash_multimodal_value(hasher, value.value) + else: + hasher.update(f'{type(value).__name__}:{repr(value)}'.encode()) + + +def make_multimodal_content_hash(data: Any, + meta: Mapping[str, Any] | None = None, + mrope_pos_ids: np.ndarray | None = None) -> str: + """Create a stable content hash for multimodal content identity.""" + hasher = hashlib.sha256() + _hash_multimodal_value(hasher, data) + _hash_multimodal_value(hasher, meta) + _hash_multimodal_value(hasher, mrope_pos_ids) + return hasher.hexdigest() diff --git a/lmdeploy/vl/model/preprocess_utils.py b/lmdeploy/vl/model/preprocess_utils.py index f267722ff7..8e6346625f 100644 --- a/lmdeploy/vl/model/preprocess_utils.py +++ b/lmdeploy/vl/model/preprocess_utils.py @@ -6,6 +6,7 @@ from lmdeploy.utils import get_logger from lmdeploy.vl.constants import Modality +from lmdeploy.vl.hasher import make_multimodal_content_hash if TYPE_CHECKING: from lmdeploy.vl.model.base import MultimodalSpecialTokens @@ -203,6 +204,26 @@ def _expand_bundled_audio_items(item: dict, token_id: int) -> list[dict]: return expanded_mm_items +def attach_multimodal_content_hashes(result): + """Attach content hashes to new-style and legacy preprocess outputs.""" + if isinstance(result, dict): + mm_items = result.get('multimodal') or [] + else: + mm_items = result + if result and 'role' in result[0]: + mm_items = [] + for message in result: + if message.get('role') == 'preprocess': + mm_items = message.get('content') or [] + break + + for item in mm_items: + content_view = {key: value for key, value in item.items() if key not in ('content_hash', 'offset')} + item['content_hash'] = make_multimodal_content_hash(content_view) + + return result + + # adapted from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/managers/mm_utils.py def get_expanded_mm_items(collected_mm_items, mm_tokens: 'MultimodalSpecialTokens'): """When multiple mm items of the same modality present, HF processors diff --git a/tests/pytorch/engine/test_multimodal_content_hash.py b/tests/pytorch/engine/test_multimodal_content_hash.py new file mode 100644 index 0000000000..c29182419c --- /dev/null +++ b/tests/pytorch/engine/test_multimodal_content_hash.py @@ -0,0 +1,39 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch + +from lmdeploy.pytorch.models.qwen3_vl import Qwen3VLInputProcessor +from lmdeploy.vl.constants import Modality + + +def test_qwen3vl_input_processor_preserves_image_content_hash(): + processor = Qwen3VLInputProcessor(config=object(), dtype=torch.float32) + result = processor.preprocess_input( + input_ids=[1, 2, 3], + input_multimodals=[{ + 'modality': Modality.IMAGE, + 'pixel_values': torch.ones(1, 1), + 'image_grid_thw': torch.tensor([1, 2, 2]), + 'offset': (1, 2), + 'image_token_id': 99, + 'content_hash': 'image-a', + }], + ) + + assert result.input_multimodals['mm_data'][0].content_hash == 'image-a' + + +def test_qwen3vl_input_processor_preserves_video_content_hash(): + processor = Qwen3VLInputProcessor(config=object(), dtype=torch.float32) + result = processor.preprocess_input( + input_ids=[1, 2, 3], + input_multimodals=[{ + 'modality': Modality.VIDEO, + 'pixel_values_videos': torch.ones(1, 1), + 'video_grid_thw': torch.tensor([1, 2, 2]), + 'offset': (1, 2), + 'video_token_id': 99, + 'content_hash': 'video-a', + }], + ) + + assert result.input_multimodals['mm_data'][0].content_hash == 'video-a' diff --git a/tests/test_lmdeploy/test_vl/test_hasher.py b/tests/test_lmdeploy/test_vl/test_hasher.py new file mode 100644 index 0000000000..f0045ab3c2 --- /dev/null +++ b/tests/test_lmdeploy/test_vl/test_hasher.py @@ -0,0 +1,33 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numpy as np +import torch + +from lmdeploy.vl.constants import Modality +from lmdeploy.vl.hasher import make_multimodal_content_hash + + +def test_multimodal_content_hash_is_stable_for_nested_values(): + data = { + 'tensor': torch.arange(4, dtype=torch.float32).reshape(2, 2), + 'array': np.arange(3, dtype=np.int64), + 'modality': Modality.IMAGE, + } + meta = {'image_token_id': 99, 'shape': [2, 2]} + mrope_pos_ids = np.arange(6, dtype=np.int64).reshape(2, 3) + + hash1 = make_multimodal_content_hash(data, meta, mrope_pos_ids) + hash2 = make_multimodal_content_hash(dict(reversed(data.items())), dict(reversed(meta.items())), + mrope_pos_ids.copy()) + + assert hash1 == hash2 + + +def test_multimodal_content_hash_changes_with_payload_meta_or_mrope(): + data = torch.arange(4, dtype=torch.float32).reshape(2, 2) + meta = {'image_token_id': 99} + mrope_pos_ids = np.arange(6, dtype=np.int64).reshape(2, 3) + base = make_multimodal_content_hash(data, meta, mrope_pos_ids) + + assert base != make_multimodal_content_hash(data + 1, meta, mrope_pos_ids) + assert base != make_multimodal_content_hash(data, {'image_token_id': 100}, mrope_pos_ids) + assert base != make_multimodal_content_hash(data, meta, mrope_pos_ids + 1) diff --git a/tests/test_lmdeploy/test_vl/test_preprocess_utils.py b/tests/test_lmdeploy/test_vl/test_preprocess_utils.py index 22084f0e88..58b0e1ea43 100644 --- a/tests/test_lmdeploy/test_vl/test_preprocess_utils.py +++ b/tests/test_lmdeploy/test_vl/test_preprocess_utils.py @@ -3,7 +3,8 @@ import torch from lmdeploy.vl.constants import Modality -from lmdeploy.vl.model.preprocess_utils import get_expanded_mm_items +from lmdeploy.vl.hasher import make_multimodal_content_hash +from lmdeploy.vl.model.preprocess_utils import attach_multimodal_content_hashes, get_expanded_mm_items class _Tokens: @@ -97,3 +98,86 @@ def test_expand_audio_items_use_compact_tensor_storage(): for entry in expanded: _assert_compact_storage(entry['input_features']) _assert_compact_storage(entry['feature_attention_mask']) + + +def test_attach_multimodal_content_hashes_to_single_image(): + items = { + Modality.IMAGE: { + 'feature': torch.arange(2, dtype=torch.float32).reshape(2, 1), + 'image_grid_thw': torch.tensor([[1, 2, 1]]), + 'offset': [(0, 2)], + } + } + expanded = get_expanded_mm_items(items, _Tokens()) + + attach_multimodal_content_hashes(expanded) + + assert len(expanded[0]['content_hash']) == 64 + content_view = {key: value for key, value in expanded[0].items() if key not in ('content_hash', 'offset')} + assert expanded[0]['content_hash'] == make_multimodal_content_hash(content_view) + + +def test_attach_multimodal_content_hashes_ignores_offset(): + item1 = { + 'modality': Modality.IMAGE, + 'pixel_values': torch.arange(2, dtype=torch.float32).reshape(2, 1), + 'image_grid_thw': torch.tensor([1, 2, 1]), + 'offset': (0, 2), + 'image_token_id': 42, + } + item2 = dict(item1, offset=(10, 12)) + + attach_multimodal_content_hashes([item1, item2]) + + assert item1['content_hash'] == item2['content_hash'] + + +def test_attach_multimodal_content_hashes_changes_with_processed_content(): + item1 = { + 'modality': Modality.IMAGE, + 'pixel_values': torch.arange(2, dtype=torch.float32).reshape(2, 1), + 'image_grid_thw': torch.tensor([1, 2, 1]), + 'offset': (0, 2), + 'image_token_id': 42, + } + item2 = dict(item1, pixel_values=item1['pixel_values'] + 1) + + attach_multimodal_content_hashes([item1, item2]) + + assert item1['content_hash'] != item2['content_hash'] + + +def test_attach_multimodal_content_hashes_to_frame_split_videos(): + items = { + Modality.VIDEO: { + 'feature': torch.arange(8, dtype=torch.float32).reshape(8, 1), + 'video_grid_thw': torch.tensor([[2, 2, 1], [2, 2, 1]]), + 'offset': [(0, 1), (1, 2), (2, 3), (3, 4)], + } + } + expanded = get_expanded_mm_items(items, _Tokens()) + + attach_multimodal_content_hashes(expanded) + + assert len({entry['content_hash'] for entry in expanded}) == len(expanded) + + +def test_attach_multimodal_content_hashes_preserves_prompt_order_for_mixed_modalities(): + items = { + Modality.AUDIO: { + 'feature': torch.arange(1 * 2 * 3, dtype=torch.float32).reshape(1, 2, 3), + 'feature_attention_mask': torch.ones(1, 3, dtype=torch.long), + 'offset': [(5, 8)], + }, + Modality.IMAGE: { + 'feature': torch.arange(2, dtype=torch.float32).reshape(2, 1), + 'image_grid_thw': torch.tensor([[1, 2, 1]]), + 'offset': [(0, 2)], + }, + } + expanded = get_expanded_mm_items(items, _Tokens()) + + attach_multimodal_content_hashes(expanded) + + assert [entry['modality'] for entry in expanded] == [Modality.IMAGE, Modality.AUDIO] + assert all(len(entry['content_hash']) == 64 for entry in expanded)