From d2e3866b062f0c36bb983fc55248d8586024a72f Mon Sep 17 00:00:00 2001 From: zxy Date: Wed, 24 Jun 2026 20:53:17 +0800 Subject: [PATCH 1/6] feat: share multimodal hash helpers --- lmdeploy/pytorch/engine/engine.py | 4 +- lmdeploy/pytorch/messages.py | 7 +- lmdeploy/pytorch/multimodal/data_type.py | 59 +---------- lmdeploy/turbomind/turbomind.py | 4 + lmdeploy/vl/hasher.py | 95 +++++++++++++++++ tests/pytorch/paging/test_block_trie.py | 4 +- tests/test_lmdeploy/test_vl/test_hasher.py | 113 +++++++++++++++++++++ 7 files changed, 223 insertions(+), 63 deletions(-) create mode 100644 lmdeploy/vl/hasher.py create mode 100644 tests/test_lmdeploy/test_vl/test_hasher.py diff --git a/lmdeploy/pytorch/engine/engine.py b/lmdeploy/pytorch/engine/engine.py index 3b3fd2e117..387467d391 100644 --- a/lmdeploy/pytorch/engine/engine.py +++ b/lmdeploy/pytorch/engine/engine.py @@ -19,11 +19,11 @@ DistServeInitRequest, ) from lmdeploy.utils import get_logger, get_model +from lmdeploy.vl import hasher as mm_hasher from ..adapter.adapter import AdapterManager from ..config import CacheConfig, ModelConfig from ..messages import MessageStatus, SchedulerSequence, UpdateTokenMode -from ..multimodal.data_type import ensure_multimodal_content_hashes from ..paging import Scheduler from ..strategies import build_strategy_factory from .base import EngineBase @@ -417,7 +417,7 @@ def _on_add_message(self, reqs: list[Request], **kwargs): input_ids = result.input_ids input_multimodals = result.input_multimodals if self.cache_config.enable_prefix_caching: - input_multimodals = ensure_multimodal_content_hashes(input_multimodals) + input_multimodals = mm_hasher.ensure_multimodal_content_hashes(input_multimodals) req_data['token_ids'] = input_ids req_data['input_multimodals'] = input_multimodals diff --git a/lmdeploy/pytorch/messages.py b/lmdeploy/pytorch/messages.py index b1c5a59e68..f4d41eca2e 100644 --- a/lmdeploy/pytorch/messages.py +++ b/lmdeploy/pytorch/messages.py @@ -10,8 +10,9 @@ from lmdeploy.messages import EngineEvent, EventType, GenerationConfig, LogitsProcessor from lmdeploy.pytorch.disagg.conn.protocol import MigrationRequest -from lmdeploy.pytorch.multimodal.data_type import MultiModalInputs, make_multimodal_content_hash +from lmdeploy.pytorch.multimodal.data_type import MultiModalInputs from lmdeploy.utils import get_logger +from lmdeploy.vl import hasher as mm_hasher from lmdeploy.vl.constants import Modality from .block import LogicalTokenBlocks @@ -1038,8 +1039,8 @@ def _update_prefix_cache_metas(self, multimodals: MultiModalInputs): # Most request paths precompute the hash after model # preprocessing. Keep this fallback for unit tests and # defensive correctness if a processor omits it. - content_hash = make_multimodal_content_hash(modal_data.data, modal_data.meta, - modal_data.mrope_pos_ids) + content_hash = mm_hasher.make_multimodal_content_hash(modal_data.data, modal_data.meta, + modal_data.mrope_pos_ids) self.prefix_cache.metas.append( PrefixCacheMeta(start=modal_data.start, end=modal_data.end, diff --git a/lmdeploy/pytorch/multimodal/data_type.py b/lmdeploy/pytorch/multimodal/data_type.py index f778c2aeb5..62144fc768 100644 --- a/lmdeploy/pytorch/multimodal/data_type.py +++ b/lmdeploy/pytorch/multimodal/data_type.py @@ -1,56 +1,16 @@ # Copyright (c) OpenMMLab. All rights reserved. -import enum -import hashlib from dataclasses import dataclass, fields from typing import Any import numpy as np -import torch from torch import Tensor +from lmdeploy.vl import hasher as mm_hasher from lmdeploy.vl.constants import Modality NestedTensor = Tensor | list[Tensor] - - -def _hash_multimodal_value(hasher: 'hashlib._Hash', value: Any): - """Update a hash with a deterministic multimodal value representation.""" - if isinstance(value, Tensor): - tensor = value.detach().cpu().contiguous() - hasher.update(f'tensor:{tensor.dtype}:{tuple(tensor.shape)}:'.encode()) - hasher.update(tensor.view(torch.uint8).numpy().tobytes()) - elif isinstance(value, np.ndarray): - array = np.ascontiguousarray(value) - hasher.update(f'ndarray:{array.dtype}:{array.shape}:'.encode()) - hasher.update(array.tobytes()) - elif isinstance(value, dict): - hasher.update(b'dict:{') - for key in sorted(value, key=lambda x: repr(x)): - _hash_multimodal_value(hasher, key) - hasher.update(b':') - _hash_multimodal_value(hasher, value[key]) - hasher.update(b',') - hasher.update(b'}') - elif isinstance(value, (list, tuple)): - hasher.update(f'{type(value).__name__}:['.encode()) - for item in value: - _hash_multimodal_value(hasher, item) - hasher.update(b',') - hasher.update(b']') - elif isinstance(value, enum.Enum): - _hash_multimodal_value(hasher, value.value) - else: - hasher.update(f'{type(value).__name__}:{repr(value)}'.encode()) - - -def make_multimodal_content_hash(data: Any, meta: dict[str, Any] | None, - mrope_pos_ids: np.ndarray | None = None) -> str: - """Create a stable content hash for prefix-cache multimodal matching.""" - hasher = hashlib.sha256() - _hash_multimodal_value(hasher, data) - _hash_multimodal_value(hasher, meta) - _hash_multimodal_value(hasher, mrope_pos_ids) - return hasher.hexdigest() +make_multimodal_content_hash = mm_hasher.make_multimodal_content_hash +ensure_multimodal_content_hashes = mm_hasher.ensure_multimodal_content_hashes @dataclass @@ -101,16 +61,3 @@ def to_device(self, device: str, non_blocking: bool = False): MultiModalInputs = dict[str, list[MultiModalData]] - - -def ensure_multimodal_content_hashes(input_mms: MultiModalInputs | None): - """Populate missing multimodal content hashes in-place.""" - if input_mms is None: - return input_mms - - for modal_datas in input_mms.values(): - for modal_data in modal_datas: - if modal_data.content_hash is None: - modal_data.content_hash = make_multimodal_content_hash(modal_data.data, modal_data.meta, - modal_data.mrope_pos_ids) - return input_mms diff --git a/lmdeploy/turbomind/turbomind.py b/lmdeploy/turbomind/turbomind.py index 5c5a1820ed..836f10a664 100644 --- a/lmdeploy/turbomind/turbomind.py +++ b/lmdeploy/turbomind/turbomind.py @@ -21,6 +21,7 @@ from lmdeploy.serve.openai.protocol import UpdateParamsRequest from lmdeploy.tokenizer import Tokenizer from lmdeploy.utils import get_logger, get_max_batch_size, get_model +from lmdeploy.vl import hasher as mm_hasher from .supported_models import is_supported @@ -283,6 +284,9 @@ def mm_input_converter(self, multimodal: list[dict[str, Any]] | None): logger.warning('Running in language-model-only mode; multimodal inputs will be ignored.') return None + if self.engine_config.enable_prefix_caching: + mm_hasher.ensure_multimodal_item_content_hashes(multimodal) + parser = getattr(self.source_model, 'to_turbomind_multimodal', None) if parser is None: raise ValueError(f'{type(self.source_model).__name__} does not support TurboMind multimodal inputs.') diff --git a/lmdeploy/vl/hasher.py b/lmdeploy/vl/hasher.py new file mode 100644 index 0000000000..4c849b02ac --- /dev/null +++ b/lmdeploy/vl/hasher.py @@ -0,0 +1,95 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import enum +import hashlib +from collections.abc import Mapping +from typing import Any + +import numpy as np +import torch +from torch import Tensor + +_POSITION_KEYS = { + 'content_hash', + 'offset', + 'start', + 'end', + 'token_begin', + 'token_end', +} + + +def _hash_multimodal_value(hasher: 'hashlib._Hash', value: Any): + """Update a hash with a deterministic multimodal value representation.""" + if isinstance(value, Tensor): + tensor = value.detach().cpu().contiguous() + hasher.update(f'tensor:{tensor.dtype}:{tuple(tensor.shape)}:'.encode()) + hasher.update(tensor.view(torch.uint8).numpy().tobytes()) + elif isinstance(value, np.ndarray): + array = np.ascontiguousarray(value) + hasher.update(f'ndarray:{array.dtype}:{array.shape}:'.encode()) + hasher.update(array.tobytes()) + elif isinstance(value, Mapping): + hasher.update(b'dict:{') + for key in sorted(value, key=lambda x: repr(x)): + _hash_multimodal_value(hasher, key) + hasher.update(b':') + _hash_multimodal_value(hasher, value[key]) + hasher.update(b',') + hasher.update(b'}') + elif isinstance(value, (list, tuple)): + hasher.update(f'{type(value).__name__}:['.encode()) + for item in value: + _hash_multimodal_value(hasher, item) + hasher.update(b',') + hasher.update(b']') + elif isinstance(value, enum.Enum): + _hash_multimodal_value(hasher, value.value) + else: + hasher.update(f'{type(value).__name__}:{repr(value)}'.encode()) + + +def make_multimodal_content_hash(data: Any, + meta: Mapping[str, Any] | None = None, + mrope_pos_ids: np.ndarray | None = None) -> str: + """Create a stable content hash for prefix-cache multimodal matching.""" + hasher = hashlib.sha256() + _hash_multimodal_value(hasher, data) + _hash_multimodal_value(hasher, meta) + _hash_multimodal_value(hasher, mrope_pos_ids) + return hasher.hexdigest() + + +def ensure_multimodal_content_hashes(input_mms): + """Populate missing ``content_hash`` values on PyTorch multimodal data.""" + if input_mms is None: + return input_mms + + for modal_datas in input_mms.values(): + for modal_data in modal_datas: + if modal_data.content_hash is None: + modal_data.content_hash = make_multimodal_content_hash(modal_data.data, modal_data.meta, + modal_data.mrope_pos_ids) + return input_mms + + +def make_multimodal_item_content_hash(item: Mapping[str, Any]) -> str: + """Create a stable content hash for dict-style multimodal items. + + Prompt positions stay outside the content hash so backends can combine the + same content identity with block-relative offsets when building cache keys. + """ + content_view = {key: value for key, value in item.items() if key not in _POSITION_KEYS} + hasher = hashlib.sha256() + _hash_multimodal_value(hasher, content_view) + return hasher.hexdigest() + + +def ensure_multimodal_item_content_hashes(items: list[dict[str, Any]] | None): + """Populate missing ``content_hash`` values on dict-style multimodal data.""" + if not items: + return items + + for item in items: + if item.get('content_hash') is None: + item['content_hash'] = make_multimodal_item_content_hash(item) + return items diff --git a/tests/pytorch/paging/test_block_trie.py b/tests/pytorch/paging/test_block_trie.py index 3ccb9f777b..1817d83923 100644 --- a/tests/pytorch/paging/test_block_trie.py +++ b/tests/pytorch/paging/test_block_trie.py @@ -2,11 +2,11 @@ import pytest import torch -from lmdeploy.pytorch import messages as messages_module from lmdeploy.pytorch.config import CacheConfig, SchedulerConfig from lmdeploy.pytorch.messages import SamplingParam, SequenceMeta, UpdateTokenMode from lmdeploy.pytorch.multimodal.data_type import MultiModalData from lmdeploy.pytorch.paging import Scheduler +from lmdeploy.vl import hasher as mm_hasher from lmdeploy.vl.constants import Modality @@ -597,7 +597,7 @@ def test_multimodal_prefix_cache_meta_skips_hash_when_prefix_cache_disabled(self def _fail_hash(*args, **kwargs): raise AssertionError('disabled prefix cache should not hash multimodal payloads') - monkeypatch.setattr(messages_module, 'make_multimodal_content_hash', _fail_hash) + monkeypatch.setattr(mm_hasher, 'make_multimodal_content_hash', _fail_hash) sess = scheduler.add_session(0) seq = sess.add_sequence([99] * sess.seq_meta.block_size, diff --git a/tests/test_lmdeploy/test_vl/test_hasher.py b/tests/test_lmdeploy/test_vl/test_hasher.py new file mode 100644 index 0000000000..3934fc1c7a --- /dev/null +++ b/tests/test_lmdeploy/test_vl/test_hasher.py @@ -0,0 +1,113 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from dataclasses import dataclass + +import numpy as np +import torch + +from lmdeploy.vl import hasher as mm_hasher +from lmdeploy.vl.constants import Modality + + +@dataclass +class DummyModalData: + data: object + meta: dict | None = None + mrope_pos_ids: np.ndarray | None = None + content_hash: str | None = None + + +def test_multimodal_content_hash_is_stable_for_nested_values(): + data = { + 'tensor': torch.arange(4, dtype=torch.float32).reshape(2, 2), + 'array': np.arange(3, dtype=np.int64), + 'modality': Modality.IMAGE, + } + meta = {'image_token_id': 99, 'shape': [2, 2]} + mrope_pos_ids = np.arange(6, dtype=np.int64).reshape(2, 3) + + hash1 = mm_hasher.make_multimodal_content_hash(data, meta, mrope_pos_ids) + hash2 = mm_hasher.make_multimodal_content_hash(dict(reversed(data.items())), dict(reversed(meta.items())), + mrope_pos_ids.copy()) + + assert hash1 == hash2 + + +def test_multimodal_content_hash_changes_with_payload_meta_or_mrope(): + data = torch.arange(4, dtype=torch.float32).reshape(2, 2) + meta = {'image_token_id': 99} + mrope_pos_ids = np.arange(6, dtype=np.int64).reshape(2, 3) + base = mm_hasher.make_multimodal_content_hash(data, meta, mrope_pos_ids) + + assert base != mm_hasher.make_multimodal_content_hash(data + 1, meta, mrope_pos_ids) + assert base != mm_hasher.make_multimodal_content_hash(data, {'image_token_id': 100}, mrope_pos_ids) + assert base != mm_hasher.make_multimodal_content_hash(data, meta, mrope_pos_ids + 1) + + +def test_ensure_multimodal_content_hashes_preserves_existing_hash(): + modal_data = DummyModalData(data=torch.ones(2, 2), meta={'image_token_id': 99}, content_hash='preset') + input_mms = {'image': [modal_data]} + + assert mm_hasher.ensure_multimodal_content_hashes(input_mms) is input_mms + assert modal_data.content_hash == 'preset' + + +def test_ensure_multimodal_content_hashes_populates_missing_hash(): + modal_data = DummyModalData(data=torch.ones(2, 2), meta={'image_token_id': 99}) + mm_hasher.ensure_multimodal_content_hashes({'image': [modal_data]}) + + assert isinstance(modal_data.content_hash, str) + assert len(modal_data.content_hash) == 64 + + +def test_multimodal_item_hash_ignores_position_keys(): + item = { + 'modality': Modality.IMAGE, + 'pixel_values': torch.arange(4, dtype=torch.float32).reshape(2, 2), + 'image_grid_thw': torch.tensor([1, 2, 2]), + 'image_token_id': 99, + 'offset': torch.tensor([4, 8]), + 'start': 4, + 'end': 8, + 'token_begin': 4, + 'token_end': 8, + } + moved_item = dict(item, offset=torch.tensor([12, 16]), start=12, end=16, token_begin=12, token_end=16) + + assert mm_hasher.make_multimodal_item_content_hash(item) == mm_hasher.make_multimodal_item_content_hash(moved_item) + + +def test_multimodal_item_hash_includes_content_keys(): + item = { + 'modality': Modality.IMAGE, + 'pixel_values': torch.arange(4, dtype=torch.float32).reshape(2, 2), + 'image_grid_thw': torch.tensor([1, 2, 2]), + 'image_token_id': 99, + 'offset': torch.tensor([4, 8]), + } + changed = dict(item, pixel_values=item['pixel_values'] + 1) + + assert mm_hasher.make_multimodal_item_content_hash(item) != mm_hasher.make_multimodal_item_content_hash(changed) + + +def test_ensure_multimodal_item_content_hashes_preserves_existing_hash(): + item = { + 'modality': Modality.IMAGE, + 'pixel_values': torch.ones(2, 2), + 'content_hash': 'preset', + } + items = [item] + + assert mm_hasher.ensure_multimodal_item_content_hashes(items) is items + assert item['content_hash'] == 'preset' + + +def test_ensure_multimodal_item_content_hashes_populates_missing_hash(): + item = { + 'modality': Modality.IMAGE, + 'pixel_values': torch.ones(2, 2), + } + items = [item] + + assert mm_hasher.ensure_multimodal_item_content_hashes(items) is items + assert isinstance(item['content_hash'], str) + assert len(item['content_hash']) == 64 From 85eec130da2dcbd7363e9375eabb0d40c4498570 Mon Sep 17 00:00:00 2001 From: zxy Date: Thu, 25 Jun 2026 15:18:33 +0800 Subject: [PATCH 2/6] fix: skip turbomind multimodal hash hook --- lmdeploy/turbomind/turbomind.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lmdeploy/turbomind/turbomind.py b/lmdeploy/turbomind/turbomind.py index 836f10a664..5c5a1820ed 100644 --- a/lmdeploy/turbomind/turbomind.py +++ b/lmdeploy/turbomind/turbomind.py @@ -21,7 +21,6 @@ from lmdeploy.serve.openai.protocol import UpdateParamsRequest from lmdeploy.tokenizer import Tokenizer from lmdeploy.utils import get_logger, get_max_batch_size, get_model -from lmdeploy.vl import hasher as mm_hasher from .supported_models import is_supported @@ -284,9 +283,6 @@ def mm_input_converter(self, multimodal: list[dict[str, Any]] | None): logger.warning('Running in language-model-only mode; multimodal inputs will be ignored.') return None - if self.engine_config.enable_prefix_caching: - mm_hasher.ensure_multimodal_item_content_hashes(multimodal) - parser = getattr(self.source_model, 'to_turbomind_multimodal', None) if parser is None: raise ValueError(f'{type(self.source_model).__name__} does not support TurboMind multimodal inputs.') From 5bb5df6969c9b8476194d56299e4a1d63eb6caec Mon Sep 17 00:00:00 2001 From: zxy Date: Thu, 25 Jun 2026 17:29:22 +0800 Subject: [PATCH 3/6] style: format multimodal hasher docstrings --- lmdeploy/vl/hasher.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lmdeploy/vl/hasher.py b/lmdeploy/vl/hasher.py index 4c849b02ac..04fa126455 100644 --- a/lmdeploy/vl/hasher.py +++ b/lmdeploy/vl/hasher.py @@ -75,8 +75,8 @@ def ensure_multimodal_content_hashes(input_mms): def make_multimodal_item_content_hash(item: Mapping[str, Any]) -> str: """Create a stable content hash for dict-style multimodal items. - Prompt positions stay outside the content hash so backends can combine the - same content identity with block-relative offsets when building cache keys. + Prompt positions stay outside the content hash so backends can combine the same content identity with block-relative + offsets when building cache keys. """ content_view = {key: value for key, value in item.items() if key not in _POSITION_KEYS} hasher = hashlib.sha256() @@ -85,7 +85,8 @@ def make_multimodal_item_content_hash(item: Mapping[str, Any]) -> str: def ensure_multimodal_item_content_hashes(items: list[dict[str, Any]] | None): - """Populate missing ``content_hash`` values on dict-style multimodal data.""" + """Populate missing ``content_hash`` values on dict-style multimodal + data.""" if not items: return items From fe5b216ec55f4a744f827c693b83f1e54c4b9842 Mon Sep 17 00:00:00 2001 From: zxy Date: Tue, 30 Jun 2026 12:04:30 +0800 Subject: [PATCH 4/6] refactor: use pytorch multimodal hash aliases --- lmdeploy/pytorch/engine/engine.py | 4 ++-- lmdeploy/pytorch/messages.py | 7 +++---- tests/pytorch/paging/test_block_trie.py | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lmdeploy/pytorch/engine/engine.py b/lmdeploy/pytorch/engine/engine.py index 387467d391..3b3fd2e117 100644 --- a/lmdeploy/pytorch/engine/engine.py +++ b/lmdeploy/pytorch/engine/engine.py @@ -19,11 +19,11 @@ DistServeInitRequest, ) from lmdeploy.utils import get_logger, get_model -from lmdeploy.vl import hasher as mm_hasher from ..adapter.adapter import AdapterManager from ..config import CacheConfig, ModelConfig from ..messages import MessageStatus, SchedulerSequence, UpdateTokenMode +from ..multimodal.data_type import ensure_multimodal_content_hashes from ..paging import Scheduler from ..strategies import build_strategy_factory from .base import EngineBase @@ -417,7 +417,7 @@ def _on_add_message(self, reqs: list[Request], **kwargs): input_ids = result.input_ids input_multimodals = result.input_multimodals if self.cache_config.enable_prefix_caching: - input_multimodals = mm_hasher.ensure_multimodal_content_hashes(input_multimodals) + input_multimodals = ensure_multimodal_content_hashes(input_multimodals) req_data['token_ids'] = input_ids req_data['input_multimodals'] = input_multimodals diff --git a/lmdeploy/pytorch/messages.py b/lmdeploy/pytorch/messages.py index f4d41eca2e..b1c5a59e68 100644 --- a/lmdeploy/pytorch/messages.py +++ b/lmdeploy/pytorch/messages.py @@ -10,9 +10,8 @@ from lmdeploy.messages import EngineEvent, EventType, GenerationConfig, LogitsProcessor from lmdeploy.pytorch.disagg.conn.protocol import MigrationRequest -from lmdeploy.pytorch.multimodal.data_type import MultiModalInputs +from lmdeploy.pytorch.multimodal.data_type import MultiModalInputs, make_multimodal_content_hash from lmdeploy.utils import get_logger -from lmdeploy.vl import hasher as mm_hasher from lmdeploy.vl.constants import Modality from .block import LogicalTokenBlocks @@ -1039,8 +1038,8 @@ def _update_prefix_cache_metas(self, multimodals: MultiModalInputs): # Most request paths precompute the hash after model # preprocessing. Keep this fallback for unit tests and # defensive correctness if a processor omits it. - content_hash = mm_hasher.make_multimodal_content_hash(modal_data.data, modal_data.meta, - modal_data.mrope_pos_ids) + content_hash = make_multimodal_content_hash(modal_data.data, modal_data.meta, + modal_data.mrope_pos_ids) self.prefix_cache.metas.append( PrefixCacheMeta(start=modal_data.start, end=modal_data.end, diff --git a/tests/pytorch/paging/test_block_trie.py b/tests/pytorch/paging/test_block_trie.py index 1817d83923..3ccb9f777b 100644 --- a/tests/pytorch/paging/test_block_trie.py +++ b/tests/pytorch/paging/test_block_trie.py @@ -2,11 +2,11 @@ import pytest import torch +from lmdeploy.pytorch import messages as messages_module from lmdeploy.pytorch.config import CacheConfig, SchedulerConfig from lmdeploy.pytorch.messages import SamplingParam, SequenceMeta, UpdateTokenMode from lmdeploy.pytorch.multimodal.data_type import MultiModalData from lmdeploy.pytorch.paging import Scheduler -from lmdeploy.vl import hasher as mm_hasher from lmdeploy.vl.constants import Modality @@ -597,7 +597,7 @@ def test_multimodal_prefix_cache_meta_skips_hash_when_prefix_cache_disabled(self def _fail_hash(*args, **kwargs): raise AssertionError('disabled prefix cache should not hash multimodal payloads') - monkeypatch.setattr(mm_hasher, 'make_multimodal_content_hash', _fail_hash) + monkeypatch.setattr(messages_module, 'make_multimodal_content_hash', _fail_hash) sess = scheduler.add_session(0) seq = sess.add_sequence([99] * sess.seq_meta.block_size, From 499e70d62e9e09ab33954f00d4644655845d1f00 Mon Sep 17 00:00:00 2001 From: zxy Date: Mon, 6 Jul 2026 17:48:37 +0800 Subject: [PATCH 5/6] refactor: compute multimodal hash in vl preprocessing --- lmdeploy/pytorch/engine/engine.py | 3 - lmdeploy/pytorch/messages.py | 7 +- lmdeploy/pytorch/models/chatglm2.py | 3 +- lmdeploy/pytorch/models/cogvlm.py | 3 +- lmdeploy/pytorch/models/deepseek_vl2.py | 3 +- lmdeploy/pytorch/models/gemma3_vl.py | 3 +- lmdeploy/pytorch/models/glm4_1v.py | 3 +- lmdeploy/pytorch/models/interns1_pro.py | 9 +- lmdeploy/pytorch/models/internvl.py | 3 +- lmdeploy/pytorch/models/internvl3_hf.py | 3 +- lmdeploy/pytorch/models/llama4.py | 3 +- lmdeploy/pytorch/models/llava.py | 6 +- lmdeploy/pytorch/models/phi3_v.py | 3 +- lmdeploy/pytorch/models/qwen2_vl.py | 3 +- .../pytorch/models/qwen3_omni_moe_thinker.py | 9 +- lmdeploy/pytorch/models/qwen3_vl.py | 9 +- lmdeploy/pytorch/multimodal/data_type.py | 3 - lmdeploy/vl/hasher.py | 48 +--------- lmdeploy/vl/model/base.py | 2 + lmdeploy/vl/model/preprocess_utils.py | 10 ++ .../engine/test_multimodal_content_hash.py | 39 ++++++++ tests/test_lmdeploy/test_vl/test_hasher.py | 96 ++----------------- .../test_vl/test_preprocess_utils.py | 86 ++++++++++++++++- 23 files changed, 191 insertions(+), 166 deletions(-) create mode 100644 tests/pytorch/engine/test_multimodal_content_hash.py diff --git a/lmdeploy/pytorch/engine/engine.py b/lmdeploy/pytorch/engine/engine.py index 3b3fd2e117..756eab80b9 100644 --- a/lmdeploy/pytorch/engine/engine.py +++ b/lmdeploy/pytorch/engine/engine.py @@ -23,7 +23,6 @@ from ..adapter.adapter import AdapterManager from ..config import CacheConfig, ModelConfig from ..messages import MessageStatus, SchedulerSequence, UpdateTokenMode -from ..multimodal.data_type import ensure_multimodal_content_hashes from ..paging import Scheduler from ..strategies import build_strategy_factory from .base import EngineBase @@ -416,8 +415,6 @@ def _on_add_message(self, reqs: list[Request], **kwargs): input_ids = result.input_ids input_multimodals = result.input_multimodals - if self.cache_config.enable_prefix_caching: - input_multimodals = ensure_multimodal_content_hashes(input_multimodals) req_data['token_ids'] = input_ids req_data['input_multimodals'] = input_multimodals diff --git a/lmdeploy/pytorch/messages.py b/lmdeploy/pytorch/messages.py index b1c5a59e68..76493e880a 100644 --- a/lmdeploy/pytorch/messages.py +++ b/lmdeploy/pytorch/messages.py @@ -10,9 +10,10 @@ from lmdeploy.messages import EngineEvent, EventType, GenerationConfig, LogitsProcessor from lmdeploy.pytorch.disagg.conn.protocol import MigrationRequest -from lmdeploy.pytorch.multimodal.data_type import MultiModalInputs, make_multimodal_content_hash +from lmdeploy.pytorch.multimodal.data_type import MultiModalInputs from lmdeploy.utils import get_logger from lmdeploy.vl.constants import Modality +from lmdeploy.vl.hasher import make_multimodal_content_hash from .block import LogicalTokenBlocks @@ -1035,8 +1036,8 @@ def _update_prefix_cache_metas(self, multimodals: MultiModalInputs): modality = modality.value content_hash = modal_data.content_hash if content_hash is None: - # Most request paths precompute the hash after model - # preprocessing. Keep this fallback for unit tests and + # Most request paths carry a hash from the shared VL path. + # Keep this fallback for unit tests and # defensive correctness if a processor omits it. content_hash = make_multimodal_content_hash(modal_data.data, modal_data.meta, modal_data.mrope_pos_ids) diff --git a/lmdeploy/pytorch/models/chatglm2.py b/lmdeploy/pytorch/models/chatglm2.py index 2988f3c9ed..b506b9f276 100644 --- a/lmdeploy/pytorch/models/chatglm2.py +++ b/lmdeploy/pytorch/models/chatglm2.py @@ -883,7 +883,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_token_id=image_token_id)) + meta=dict(image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/cogvlm.py b/lmdeploy/pytorch/models/cogvlm.py index 98d2f02583..fa6c874dd5 100644 --- a/lmdeploy/pytorch/models/cogvlm.py +++ b/lmdeploy/pytorch/models/cogvlm.py @@ -909,7 +909,8 @@ def preprocess_input(self, input_ids: list[int], input_multimodals=None, **kwarg mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_token_id=image_token_id)) + meta=dict(image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/deepseek_vl2.py b/lmdeploy/pytorch/models/deepseek_vl2.py index b556ffc2b8..b881849c0e 100644 --- a/lmdeploy/pytorch/models/deepseek_vl2.py +++ b/lmdeploy/pytorch/models/deepseek_vl2.py @@ -447,7 +447,8 @@ def preprocess_input(self, meta=dict( image_token_id=image_token_id, images_spatial_crop=images_spatial_crop, - )) + ), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) diff --git a/lmdeploy/pytorch/models/gemma3_vl.py b/lmdeploy/pytorch/models/gemma3_vl.py index 9f444e81f5..a0eb3c9241 100644 --- a/lmdeploy/pytorch/models/gemma3_vl.py +++ b/lmdeploy/pytorch/models/gemma3_vl.py @@ -112,7 +112,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_token_id=image_token_id)) + meta=dict(image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/glm4_1v.py b/lmdeploy/pytorch/models/glm4_1v.py index ac802fe6a2..698181a5bc 100644 --- a/lmdeploy/pytorch/models/glm4_1v.py +++ b/lmdeploy/pytorch/models/glm4_1v.py @@ -745,7 +745,8 @@ def preprocess_input(self, start=offset[0], end=offset[1], mrope_pos_ids=mrope_pos_ids, - meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id)) + meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/interns1_pro.py b/lmdeploy/pytorch/models/interns1_pro.py index 9a03f30640..f863db270e 100644 --- a/lmdeploy/pytorch/models/interns1_pro.py +++ b/lmdeploy/pytorch/models/interns1_pro.py @@ -374,7 +374,8 @@ def _make_image_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: data=pixel_values, start=offset[0], end=offset[1], - meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id)) + meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) return mm_data def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: @@ -391,7 +392,8 @@ def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: meta=dict( grid_thw=video_grid_thw, video_token_id=video_token_id, - )) + ), + content_hash=input_mm.get('content_hash')) return mm_data def _make_time_series_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: @@ -406,7 +408,8 @@ def _make_time_series_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: data=ts_values, start=offset[0], end=offset[1], - meta=dict(ts_lens=ts_lens, ts_sr=ts_sr, ts_token_id=ts_token_id)) + meta=dict(ts_lens=ts_lens, ts_sr=ts_sr, ts_token_id=ts_token_id), + content_hash=input_mm.get('content_hash')) return mm_data def preprocess_input(self, diff --git a/lmdeploy/pytorch/models/internvl.py b/lmdeploy/pytorch/models/internvl.py index 2a2f53dae3..9243e62639 100644 --- a/lmdeploy/pytorch/models/internvl.py +++ b/lmdeploy/pytorch/models/internvl.py @@ -946,7 +946,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_token_id=image_token_id)) + meta=dict(image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/internvl3_hf.py b/lmdeploy/pytorch/models/internvl3_hf.py index 09f159c22a..6c0f3ad00e 100644 --- a/lmdeploy/pytorch/models/internvl3_hf.py +++ b/lmdeploy/pytorch/models/internvl3_hf.py @@ -740,7 +740,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_token_id=image_token_id)) + meta=dict(image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/llama4.py b/lmdeploy/pytorch/models/llama4.py index 2552467218..7c507be9ad 100644 --- a/lmdeploy/pytorch/models/llama4.py +++ b/lmdeploy/pytorch/models/llama4.py @@ -1041,7 +1041,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_token_id=image_token_id)) + meta=dict(image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/llava.py b/lmdeploy/pytorch/models/llava.py index bf0618ec9a..71150df4cd 100644 --- a/lmdeploy/pytorch/models/llava.py +++ b/lmdeploy/pytorch/models/llava.py @@ -559,7 +559,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_token_id=image_token_id)) + meta=dict(image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( @@ -838,7 +839,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_sizes=image_sizes, image_token_id=image_token_id)) + meta=dict(image_sizes=image_sizes, image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/phi3_v.py b/lmdeploy/pytorch/models/phi3_v.py index 589e04e20d..3d4cb618a2 100644 --- a/lmdeploy/pytorch/models/phi3_v.py +++ b/lmdeploy/pytorch/models/phi3_v.py @@ -383,7 +383,8 @@ def preprocess_input(self, mm_data = MultiModalData(data=pixel_values, start=offset, end=offset + num_pad, - meta=dict(image_sizes=image_sizes, image_token_id=image_token_id)) + meta=dict(image_sizes=image_sizes, image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/qwen2_vl.py b/lmdeploy/pytorch/models/qwen2_vl.py index ff2bdb2e7d..7fede4d37c 100644 --- a/lmdeploy/pytorch/models/qwen2_vl.py +++ b/lmdeploy/pytorch/models/qwen2_vl.py @@ -814,7 +814,8 @@ def preprocess_input(self, start=start, end=start + num_pad, mrope_pos_ids=mrope_pos_ids, - meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id)) + meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) input_imgs.append(mm_data) result = PreprocessInputResult( diff --git a/lmdeploy/pytorch/models/qwen3_omni_moe_thinker.py b/lmdeploy/pytorch/models/qwen3_omni_moe_thinker.py index 733c323dbf..7a4c6cf7f3 100644 --- a/lmdeploy/pytorch/models/qwen3_omni_moe_thinker.py +++ b/lmdeploy/pytorch/models/qwen3_omni_moe_thinker.py @@ -846,7 +846,8 @@ def _make_image_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: start=offset[0], end=offset[1], mrope_pos_ids=mrope_pos_ids, - meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id)) + meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) return mm_data def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: @@ -867,7 +868,8 @@ def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: grid_thw=video_grid_thw, video_token_id=video_token_id, second_per_grid=input_mm.get('second_per_grid'), - )) + ), + content_hash=input_mm.get('content_hash')) return mm_data def _make_audio_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: @@ -896,7 +898,8 @@ def _make_audio_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: meta=dict( audio_token_id=audio_token_id, audio_feature_lengths=audio_feature_lengths, - )) + ), + content_hash=input_mm.get('content_hash')) return mm_data def preprocess_input(self, diff --git a/lmdeploy/pytorch/models/qwen3_vl.py b/lmdeploy/pytorch/models/qwen3_vl.py index db3f69af55..d9053a8088 100644 --- a/lmdeploy/pytorch/models/qwen3_vl.py +++ b/lmdeploy/pytorch/models/qwen3_vl.py @@ -682,7 +682,8 @@ def _make_image_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: start=offset[0], end=offset[1], mrope_pos_ids=mrope_pos_ids, - meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id)) + meta=dict(grid_thw=image_grid_thw, image_token_id=image_token_id), + content_hash=input_mm.get('content_hash')) return mm_data def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: @@ -702,7 +703,8 @@ def _make_video_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: meta=dict( grid_thw=video_grid_thw, video_token_id=video_token_id, - )) + ), + content_hash=input_mm.get('content_hash')) return mm_data def _make_time_series_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: @@ -717,7 +719,8 @@ def _make_time_series_mm_data(self, input_mm: dict[str, Any]) -> MultiModalData: data=ts_values, start=offset[0], end=offset[1], - meta=dict(ts_lens=ts_lens, ts_sr=ts_sr, ts_token_id=ts_token_id)) + meta=dict(ts_lens=ts_lens, ts_sr=ts_sr, ts_token_id=ts_token_id), + content_hash=input_mm.get('content_hash')) return mm_data def preprocess_input(self, diff --git a/lmdeploy/pytorch/multimodal/data_type.py b/lmdeploy/pytorch/multimodal/data_type.py index 62144fc768..070f35867e 100644 --- a/lmdeploy/pytorch/multimodal/data_type.py +++ b/lmdeploy/pytorch/multimodal/data_type.py @@ -5,12 +5,9 @@ import numpy as np from torch import Tensor -from lmdeploy.vl import hasher as mm_hasher from lmdeploy.vl.constants import Modality NestedTensor = Tensor | list[Tensor] -make_multimodal_content_hash = mm_hasher.make_multimodal_content_hash -ensure_multimodal_content_hashes = mm_hasher.ensure_multimodal_content_hashes @dataclass diff --git a/lmdeploy/vl/hasher.py b/lmdeploy/vl/hasher.py index 04fa126455..500449bb08 100644 --- a/lmdeploy/vl/hasher.py +++ b/lmdeploy/vl/hasher.py @@ -8,15 +8,6 @@ import torch from torch import Tensor -_POSITION_KEYS = { - 'content_hash', - 'offset', - 'start', - 'end', - 'token_begin', - 'token_end', -} - def _hash_multimodal_value(hasher: 'hashlib._Hash', value: Any): """Update a hash with a deterministic multimodal value representation.""" @@ -51,46 +42,9 @@ def _hash_multimodal_value(hasher: 'hashlib._Hash', value: Any): def make_multimodal_content_hash(data: Any, meta: Mapping[str, Any] | None = None, mrope_pos_ids: np.ndarray | None = None) -> str: - """Create a stable content hash for prefix-cache multimodal matching.""" + """Create a stable content hash for multimodal content identity.""" hasher = hashlib.sha256() _hash_multimodal_value(hasher, data) _hash_multimodal_value(hasher, meta) _hash_multimodal_value(hasher, mrope_pos_ids) return hasher.hexdigest() - - -def ensure_multimodal_content_hashes(input_mms): - """Populate missing ``content_hash`` values on PyTorch multimodal data.""" - if input_mms is None: - return input_mms - - for modal_datas in input_mms.values(): - for modal_data in modal_datas: - if modal_data.content_hash is None: - modal_data.content_hash = make_multimodal_content_hash(modal_data.data, modal_data.meta, - modal_data.mrope_pos_ids) - return input_mms - - -def make_multimodal_item_content_hash(item: Mapping[str, Any]) -> str: - """Create a stable content hash for dict-style multimodal items. - - Prompt positions stay outside the content hash so backends can combine the same content identity with block-relative - offsets when building cache keys. - """ - content_view = {key: value for key, value in item.items() if key not in _POSITION_KEYS} - hasher = hashlib.sha256() - _hash_multimodal_value(hasher, content_view) - return hasher.hexdigest() - - -def ensure_multimodal_item_content_hashes(items: list[dict[str, Any]] | None): - """Populate missing ``content_hash`` values on dict-style multimodal - data.""" - if not items: - return items - - for item in items: - if item.get('content_hash') is None: - item['content_hash'] = make_multimodal_item_content_hash(item) - return items diff --git a/lmdeploy/vl/model/base.py b/lmdeploy/vl/model/base.py index 47de994b5a..e3b35f86a1 100644 --- a/lmdeploy/vl/model/base.py +++ b/lmdeploy/vl/model/base.py @@ -12,6 +12,7 @@ from lmdeploy.archs import get_model_arch from lmdeploy.vl.constants import Modality from lmdeploy.vl.model.preprocess_utils import ( + attach_multimodal_content_hashes, get_expanded_input_ids, get_expanded_mm_items, get_mm_items_offset, @@ -233,6 +234,7 @@ def preprocess(self, # expand bundled hf processor outputs into per-image/video entry for lmdeploy to consume expanded_mm_items = get_expanded_mm_items(collected_mm_items, self.mm_tokens) + attach_multimodal_content_hashes(expanded_mm_items) result = dict(input_ids=input_ids.tolist(), multimodal=expanded_mm_items) return result diff --git a/lmdeploy/vl/model/preprocess_utils.py b/lmdeploy/vl/model/preprocess_utils.py index f267722ff7..6f2de620ef 100644 --- a/lmdeploy/vl/model/preprocess_utils.py +++ b/lmdeploy/vl/model/preprocess_utils.py @@ -6,6 +6,7 @@ from lmdeploy.utils import get_logger from lmdeploy.vl.constants import Modality +from lmdeploy.vl.hasher import make_multimodal_content_hash if TYPE_CHECKING: from lmdeploy.vl.model.base import MultimodalSpecialTokens @@ -203,6 +204,15 @@ def _expand_bundled_audio_items(item: dict, token_id: int) -> list[dict]: return expanded_mm_items +def attach_multimodal_content_hashes(expanded_mm_items: list[dict[str, Any]]): + """Attach processed-item content hashes to expanded multimodal items.""" + for item in expanded_mm_items: + content_view = {key: value for key, value in item.items() if key not in ('content_hash', 'offset')} + item['content_hash'] = make_multimodal_content_hash(content_view) + + return expanded_mm_items + + # adapted from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/managers/mm_utils.py def get_expanded_mm_items(collected_mm_items, mm_tokens: 'MultimodalSpecialTokens'): """When multiple mm items of the same modality present, HF processors diff --git a/tests/pytorch/engine/test_multimodal_content_hash.py b/tests/pytorch/engine/test_multimodal_content_hash.py new file mode 100644 index 0000000000..c29182419c --- /dev/null +++ b/tests/pytorch/engine/test_multimodal_content_hash.py @@ -0,0 +1,39 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch + +from lmdeploy.pytorch.models.qwen3_vl import Qwen3VLInputProcessor +from lmdeploy.vl.constants import Modality + + +def test_qwen3vl_input_processor_preserves_image_content_hash(): + processor = Qwen3VLInputProcessor(config=object(), dtype=torch.float32) + result = processor.preprocess_input( + input_ids=[1, 2, 3], + input_multimodals=[{ + 'modality': Modality.IMAGE, + 'pixel_values': torch.ones(1, 1), + 'image_grid_thw': torch.tensor([1, 2, 2]), + 'offset': (1, 2), + 'image_token_id': 99, + 'content_hash': 'image-a', + }], + ) + + assert result.input_multimodals['mm_data'][0].content_hash == 'image-a' + + +def test_qwen3vl_input_processor_preserves_video_content_hash(): + processor = Qwen3VLInputProcessor(config=object(), dtype=torch.float32) + result = processor.preprocess_input( + input_ids=[1, 2, 3], + input_multimodals=[{ + 'modality': Modality.VIDEO, + 'pixel_values_videos': torch.ones(1, 1), + 'video_grid_thw': torch.tensor([1, 2, 2]), + 'offset': (1, 2), + 'video_token_id': 99, + 'content_hash': 'video-a', + }], + ) + + assert result.input_multimodals['mm_data'][0].content_hash == 'video-a' diff --git a/tests/test_lmdeploy/test_vl/test_hasher.py b/tests/test_lmdeploy/test_vl/test_hasher.py index 3934fc1c7a..f0045ab3c2 100644 --- a/tests/test_lmdeploy/test_vl/test_hasher.py +++ b/tests/test_lmdeploy/test_vl/test_hasher.py @@ -1,19 +1,9 @@ # Copyright (c) OpenMMLab. All rights reserved. -from dataclasses import dataclass - import numpy as np import torch -from lmdeploy.vl import hasher as mm_hasher from lmdeploy.vl.constants import Modality - - -@dataclass -class DummyModalData: - data: object - meta: dict | None = None - mrope_pos_ids: np.ndarray | None = None - content_hash: str | None = None +from lmdeploy.vl.hasher import make_multimodal_content_hash def test_multimodal_content_hash_is_stable_for_nested_values(): @@ -25,9 +15,9 @@ def test_multimodal_content_hash_is_stable_for_nested_values(): meta = {'image_token_id': 99, 'shape': [2, 2]} mrope_pos_ids = np.arange(6, dtype=np.int64).reshape(2, 3) - hash1 = mm_hasher.make_multimodal_content_hash(data, meta, mrope_pos_ids) - hash2 = mm_hasher.make_multimodal_content_hash(dict(reversed(data.items())), dict(reversed(meta.items())), - mrope_pos_ids.copy()) + hash1 = make_multimodal_content_hash(data, meta, mrope_pos_ids) + hash2 = make_multimodal_content_hash(dict(reversed(data.items())), dict(reversed(meta.items())), + mrope_pos_ids.copy()) assert hash1 == hash2 @@ -36,78 +26,8 @@ def test_multimodal_content_hash_changes_with_payload_meta_or_mrope(): data = torch.arange(4, dtype=torch.float32).reshape(2, 2) meta = {'image_token_id': 99} mrope_pos_ids = np.arange(6, dtype=np.int64).reshape(2, 3) - base = mm_hasher.make_multimodal_content_hash(data, meta, mrope_pos_ids) - - assert base != mm_hasher.make_multimodal_content_hash(data + 1, meta, mrope_pos_ids) - assert base != mm_hasher.make_multimodal_content_hash(data, {'image_token_id': 100}, mrope_pos_ids) - assert base != mm_hasher.make_multimodal_content_hash(data, meta, mrope_pos_ids + 1) - - -def test_ensure_multimodal_content_hashes_preserves_existing_hash(): - modal_data = DummyModalData(data=torch.ones(2, 2), meta={'image_token_id': 99}, content_hash='preset') - input_mms = {'image': [modal_data]} - - assert mm_hasher.ensure_multimodal_content_hashes(input_mms) is input_mms - assert modal_data.content_hash == 'preset' - - -def test_ensure_multimodal_content_hashes_populates_missing_hash(): - modal_data = DummyModalData(data=torch.ones(2, 2), meta={'image_token_id': 99}) - mm_hasher.ensure_multimodal_content_hashes({'image': [modal_data]}) - - assert isinstance(modal_data.content_hash, str) - assert len(modal_data.content_hash) == 64 - - -def test_multimodal_item_hash_ignores_position_keys(): - item = { - 'modality': Modality.IMAGE, - 'pixel_values': torch.arange(4, dtype=torch.float32).reshape(2, 2), - 'image_grid_thw': torch.tensor([1, 2, 2]), - 'image_token_id': 99, - 'offset': torch.tensor([4, 8]), - 'start': 4, - 'end': 8, - 'token_begin': 4, - 'token_end': 8, - } - moved_item = dict(item, offset=torch.tensor([12, 16]), start=12, end=16, token_begin=12, token_end=16) - - assert mm_hasher.make_multimodal_item_content_hash(item) == mm_hasher.make_multimodal_item_content_hash(moved_item) - - -def test_multimodal_item_hash_includes_content_keys(): - item = { - 'modality': Modality.IMAGE, - 'pixel_values': torch.arange(4, dtype=torch.float32).reshape(2, 2), - 'image_grid_thw': torch.tensor([1, 2, 2]), - 'image_token_id': 99, - 'offset': torch.tensor([4, 8]), - } - changed = dict(item, pixel_values=item['pixel_values'] + 1) - - assert mm_hasher.make_multimodal_item_content_hash(item) != mm_hasher.make_multimodal_item_content_hash(changed) - - -def test_ensure_multimodal_item_content_hashes_preserves_existing_hash(): - item = { - 'modality': Modality.IMAGE, - 'pixel_values': torch.ones(2, 2), - 'content_hash': 'preset', - } - items = [item] - - assert mm_hasher.ensure_multimodal_item_content_hashes(items) is items - assert item['content_hash'] == 'preset' - - -def test_ensure_multimodal_item_content_hashes_populates_missing_hash(): - item = { - 'modality': Modality.IMAGE, - 'pixel_values': torch.ones(2, 2), - } - items = [item] + base = make_multimodal_content_hash(data, meta, mrope_pos_ids) - assert mm_hasher.ensure_multimodal_item_content_hashes(items) is items - assert isinstance(item['content_hash'], str) - assert len(item['content_hash']) == 64 + assert base != make_multimodal_content_hash(data + 1, meta, mrope_pos_ids) + assert base != make_multimodal_content_hash(data, {'image_token_id': 100}, mrope_pos_ids) + assert base != make_multimodal_content_hash(data, meta, mrope_pos_ids + 1) diff --git a/tests/test_lmdeploy/test_vl/test_preprocess_utils.py b/tests/test_lmdeploy/test_vl/test_preprocess_utils.py index 22084f0e88..58b0e1ea43 100644 --- a/tests/test_lmdeploy/test_vl/test_preprocess_utils.py +++ b/tests/test_lmdeploy/test_vl/test_preprocess_utils.py @@ -3,7 +3,8 @@ import torch from lmdeploy.vl.constants import Modality -from lmdeploy.vl.model.preprocess_utils import get_expanded_mm_items +from lmdeploy.vl.hasher import make_multimodal_content_hash +from lmdeploy.vl.model.preprocess_utils import attach_multimodal_content_hashes, get_expanded_mm_items class _Tokens: @@ -97,3 +98,86 @@ def test_expand_audio_items_use_compact_tensor_storage(): for entry in expanded: _assert_compact_storage(entry['input_features']) _assert_compact_storage(entry['feature_attention_mask']) + + +def test_attach_multimodal_content_hashes_to_single_image(): + items = { + Modality.IMAGE: { + 'feature': torch.arange(2, dtype=torch.float32).reshape(2, 1), + 'image_grid_thw': torch.tensor([[1, 2, 1]]), + 'offset': [(0, 2)], + } + } + expanded = get_expanded_mm_items(items, _Tokens()) + + attach_multimodal_content_hashes(expanded) + + assert len(expanded[0]['content_hash']) == 64 + content_view = {key: value for key, value in expanded[0].items() if key not in ('content_hash', 'offset')} + assert expanded[0]['content_hash'] == make_multimodal_content_hash(content_view) + + +def test_attach_multimodal_content_hashes_ignores_offset(): + item1 = { + 'modality': Modality.IMAGE, + 'pixel_values': torch.arange(2, dtype=torch.float32).reshape(2, 1), + 'image_grid_thw': torch.tensor([1, 2, 1]), + 'offset': (0, 2), + 'image_token_id': 42, + } + item2 = dict(item1, offset=(10, 12)) + + attach_multimodal_content_hashes([item1, item2]) + + assert item1['content_hash'] == item2['content_hash'] + + +def test_attach_multimodal_content_hashes_changes_with_processed_content(): + item1 = { + 'modality': Modality.IMAGE, + 'pixel_values': torch.arange(2, dtype=torch.float32).reshape(2, 1), + 'image_grid_thw': torch.tensor([1, 2, 1]), + 'offset': (0, 2), + 'image_token_id': 42, + } + item2 = dict(item1, pixel_values=item1['pixel_values'] + 1) + + attach_multimodal_content_hashes([item1, item2]) + + assert item1['content_hash'] != item2['content_hash'] + + +def test_attach_multimodal_content_hashes_to_frame_split_videos(): + items = { + Modality.VIDEO: { + 'feature': torch.arange(8, dtype=torch.float32).reshape(8, 1), + 'video_grid_thw': torch.tensor([[2, 2, 1], [2, 2, 1]]), + 'offset': [(0, 1), (1, 2), (2, 3), (3, 4)], + } + } + expanded = get_expanded_mm_items(items, _Tokens()) + + attach_multimodal_content_hashes(expanded) + + assert len({entry['content_hash'] for entry in expanded}) == len(expanded) + + +def test_attach_multimodal_content_hashes_preserves_prompt_order_for_mixed_modalities(): + items = { + Modality.AUDIO: { + 'feature': torch.arange(1 * 2 * 3, dtype=torch.float32).reshape(1, 2, 3), + 'feature_attention_mask': torch.ones(1, 3, dtype=torch.long), + 'offset': [(5, 8)], + }, + Modality.IMAGE: { + 'feature': torch.arange(2, dtype=torch.float32).reshape(2, 1), + 'image_grid_thw': torch.tensor([[1, 2, 1]]), + 'offset': [(0, 2)], + }, + } + expanded = get_expanded_mm_items(items, _Tokens()) + + attach_multimodal_content_hashes(expanded) + + assert [entry['modality'] for entry in expanded] == [Modality.IMAGE, Modality.AUDIO] + assert all(len(entry['content_hash']) == 64 for entry in expanded) From 359e006f8423c55d849e941f8ccf57896b23bcd9 Mon Sep 17 00:00:00 2001 From: zxy Date: Fri, 10 Jul 2026 15:18:52 +0800 Subject: [PATCH 6/6] refactor: attach multimodal hashes after vl preprocessing --- lmdeploy/vl/engine.py | 2 ++ lmdeploy/vl/model/base.py | 2 -- lmdeploy/vl/model/preprocess_utils.py | 19 +++++++++++++++---- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/lmdeploy/vl/engine.py b/lmdeploy/vl/engine.py index f2a5f62ccf..5ce37fda69 100644 --- a/lmdeploy/vl/engine.py +++ b/lmdeploy/vl/engine.py @@ -10,6 +10,7 @@ from lmdeploy.messages import PytorchEngineConfig, TurbomindEngineConfig, VisionConfig from lmdeploy.utils import is_bf16_supported from lmdeploy.vl.model.builder import load_vl_model +from lmdeploy.vl.model.preprocess_utils import attach_multimodal_content_hashes def _get_hf_config_mm_feature_dtype(hf_config) -> torch.dtype | None: @@ -119,6 +120,7 @@ async def preprocess(self, self.executor, self.model.preprocess, messages) future.add_done_callback(_raise_exception_on_finish) outputs = await future + attach_multimodal_content_hashes(outputs) return outputs async def async_infer(self, messages: list[dict]) -> list[dict]: diff --git a/lmdeploy/vl/model/base.py b/lmdeploy/vl/model/base.py index e3b35f86a1..47de994b5a 100644 --- a/lmdeploy/vl/model/base.py +++ b/lmdeploy/vl/model/base.py @@ -12,7 +12,6 @@ from lmdeploy.archs import get_model_arch from lmdeploy.vl.constants import Modality from lmdeploy.vl.model.preprocess_utils import ( - attach_multimodal_content_hashes, get_expanded_input_ids, get_expanded_mm_items, get_mm_items_offset, @@ -234,7 +233,6 @@ def preprocess(self, # expand bundled hf processor outputs into per-image/video entry for lmdeploy to consume expanded_mm_items = get_expanded_mm_items(collected_mm_items, self.mm_tokens) - attach_multimodal_content_hashes(expanded_mm_items) result = dict(input_ids=input_ids.tolist(), multimodal=expanded_mm_items) return result diff --git a/lmdeploy/vl/model/preprocess_utils.py b/lmdeploy/vl/model/preprocess_utils.py index 6f2de620ef..8e6346625f 100644 --- a/lmdeploy/vl/model/preprocess_utils.py +++ b/lmdeploy/vl/model/preprocess_utils.py @@ -204,13 +204,24 @@ def _expand_bundled_audio_items(item: dict, token_id: int) -> list[dict]: return expanded_mm_items -def attach_multimodal_content_hashes(expanded_mm_items: list[dict[str, Any]]): - """Attach processed-item content hashes to expanded multimodal items.""" - for item in expanded_mm_items: +def attach_multimodal_content_hashes(result): + """Attach content hashes to new-style and legacy preprocess outputs.""" + if isinstance(result, dict): + mm_items = result.get('multimodal') or [] + else: + mm_items = result + if result and 'role' in result[0]: + mm_items = [] + for message in result: + if message.get('role') == 'preprocess': + mm_items = message.get('content') or [] + break + + for item in mm_items: content_view = {key: value for key, value in item.items() if key not in ('content_hash', 'offset')} item['content_hash'] = make_multimodal_content_hash(content_view) - return expanded_mm_items + return result # adapted from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/managers/mm_utils.py