From 928a2b7f4d664abed727db76b5876177a1626057 Mon Sep 17 00:00:00 2001 From: rrrxxx0510 <132631746+rrrxxx0510@users.noreply.github.com> Date: Tue, 8 Sep 2026 00:39:52 +0800 Subject: [PATCH 1/2] Replace removed tokenizer.batch_encode_plus with direct call batch_encode_plus was removed in transformers 5.x, which breaks the HuggingFace backends for any model whose tokenizer is a Qwen2Tokenizer (e.g. Qwen3.5) and likely others: AttributeError: Qwen2Tokenizer has no attribute batch_encode_plus The tokenizer __call__ API accepts the same kwargs and returns the same BatchEncoding, so this is a drop-in replacement. Covers all call sites in huggingface_above_v4_33.py and huggingface.py. Fixes #2573 --- opencompass/models/huggingface.py | 11 ++++++----- opencompass/models/huggingface_above_v4_33.py | 10 +++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/opencompass/models/huggingface.py b/opencompass/models/huggingface.py index 649bfc499..923830020 100644 --- a/opencompass/models/huggingface.py +++ b/opencompass/models/huggingface.py @@ -295,11 +295,12 @@ def _batch_generate(self, conv.append_message(conv.roles[1], None) inputs[i] = conv.get_prompt() - # step-1: tokenize the input with batch_encode_plus - tokens = self.tokenizer.batch_encode_plus(inputs, - padding=True, - truncation=True, - max_length=self.max_seq_len) + # step-1: tokenize the input + # (tokenizer.__call__ instead of batch_encode_plus, which was removed in transformers 5.x) + tokens = self.tokenizer(inputs, + padding=True, + truncation=True, + max_length=self.max_seq_len) tokens = { k: torch.tensor(np.array(tokens[k]), device=self.model.device) for k in tokens if k in ['input_ids', 'attention_mask'] diff --git a/opencompass/models/huggingface_above_v4_33.py b/opencompass/models/huggingface_above_v4_33.py index 8e9bfe943..019b81f37 100644 --- a/opencompass/models/huggingface_above_v4_33.py +++ b/opencompass/models/huggingface_above_v4_33.py @@ -302,7 +302,7 @@ def get_ppl_tokenwise(self, inputs: List[str], label: List[List[int]], mask_leng self.tokenizer.padding_side = 'right' self.tokenizer.truncation_side = 'right' - tokens = self.tokenizer.batch_encode_plus(messages, **tokenize_kwargs) + tokens = self.tokenizer(messages, **tokenize_kwargs) tokens = {k: v.to(self.model.device) for k, v in tokens.items()} outputs = self.model(**tokens)[0] @@ -466,11 +466,11 @@ def generate(self, ) if self.fastchat_template: messages = _format_with_fast_chat_template(messages, self.fastchat_template) - tokens = self.tokenizer.batch_encode_plus(messages, **tokenize_kwargs) + tokens = self.tokenizer(messages, **tokenize_kwargs) else: messages = [self.tokenizer.apply_chat_template(m, add_generation_prompt=True, tokenize=False) for m in messages] tokenize_kwargs['add_special_tokens'] = False - tokens = self.tokenizer.batch_encode_plus(messages, **tokenize_kwargs) + tokens = self.tokenizer(messages, **tokenize_kwargs) tokens = {k: v.to(self.model.device) for k, v in tokens.items()} @@ -592,7 +592,7 @@ def generate(self, input_ids = torch.cat([input_ids[:, : self.max_seq_len // 2], input_ids[:, - self.max_seq_len // 2:]], dim=-1) tokens = {'input_ids': input_ids, } else: - tokens = self.tokenizer.batch_encode_plus(messages, **tokenize_kwargs) + tokens = self.tokenizer(messages, **tokenize_kwargs) tokens = {k: v.to(self.model.device) for k, v in tokens.items()} @@ -654,7 +654,7 @@ def get_ppl(self, inputs: List[str], mask_length: Optional[List[int]] = None) -> input_ids = torch.cat([input_ids[:, : self.max_seq_len // 2], input_ids[:, - self.max_seq_len // 2:]], dim=-1) tokens = {'input_ids': input_ids, } else: - tokens = self.tokenizer.batch_encode_plus(messages, **tokenize_kwargs) + tokens = self.tokenizer(messages, **tokenize_kwargs) tokens = {k: v.to(self.model.device) for k, v in tokens.items()} outputs = self.model(**tokens)[0] From 71e0b1f076682300a75cca6dce6c3df68f491c66 Mon Sep 17 00:00:00 2001 From: rrrxxx0510 <132631746+rrrxxx0510@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:29:10 +0800 Subject: [PATCH 2/2] Update test mocks for the direct tokenizer call The tests mocked tokenizer.batch_encode_plus, which the previous commit replaced with a direct tokenizer call, so the mocked dict was never returned and three tests failed. Mock tokenizer.return_value instead. Verified: 15 passed with this change, 3 failed without it. --- tests/models/test_huggingface.py | 2 +- tests/models/test_huggingface_above_v4_33.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/models/test_huggingface.py b/tests/models/test_huggingface.py index 24cde4636..0bd85d051 100644 --- a/tests/models/test_huggingface.py +++ b/tests/models/test_huggingface.py @@ -238,7 +238,7 @@ def test_generate_with_batch_padding(self, mock_transformers, """Test generate with batch_padding=True.""" mock_tokenizer = MagicMock() mock_tokenizer.pad_token_id = 0 - mock_tokenizer.batch_encode_plus.return_value = { + mock_tokenizer.return_value = { 'input_ids': [[1, 2, 3], [4, 5, 6]], 'attention_mask': [[1, 1, 1], [1, 1, 1]] } diff --git a/tests/models/test_huggingface_above_v4_33.py b/tests/models/test_huggingface_above_v4_33.py index ed2c3c15c..d8d53dccf 100644 --- a/tests/models/test_huggingface_above_v4_33.py +++ b/tests/models/test_huggingface_above_v4_33.py @@ -90,7 +90,7 @@ def test_generate_basic(self, mock_get_stopping_criteria, mock_input_ids.shape = [1, 3] # batch_size=1, seq_len=3 mock_attention_mask = MagicMock() mock_attention_mask.to.return_value = mock_attention_mask - mock_tokenizer.batch_encode_plus.return_value = { + mock_tokenizer.return_value = { 'input_ids': mock_input_ids, 'attention_mask': mock_attention_mask } @@ -214,7 +214,7 @@ def test_generate_basic(self, mock_get_stopping_criteria, mock_input_ids.to.return_value = mock_input_ids mock_attention_mask = MagicMock() mock_attention_mask.to.return_value = mock_attention_mask - mock_tokenizer.batch_encode_plus.return_value = { + mock_tokenizer.return_value = { 'input_ids': mock_input_ids, 'attention_mask': mock_attention_mask }