Replace removed tokenizer.batch_encode_plus with direct call

opencompass/models/huggingface.py

@@ -295,11 +295,12 @@ def _batch_generate(self,
                 conv.append_message(conv.roles[1], None)
                 inputs[i] = conv.get_prompt()
 
-        # step-1: tokenize the input with batch_encode_plus
-        tokens = self.tokenizer.batch_encode_plus(inputs,
-                                                  padding=True,
-                                                  truncation=True,
-                                                  max_length=self.max_seq_len)
+        # step-1: tokenize the input
+        # (tokenizer.__call__ instead of batch_encode_plus, which was removed in transformers 5.x)
+        tokens = self.tokenizer(inputs,
+                                padding=True,
+                                truncation=True,
+                                max_length=self.max_seq_len)
         tokens = {
             k: torch.tensor(np.array(tokens[k]), device=self.model.device)
             for k in tokens if k in ['input_ids', 'attention_mask']

opencompass/models/huggingface_above_v4_33.py

@@ -302,7 +302,7 @@ def get_ppl_tokenwise(self, inputs: List[str], label: List[List[int]], mask_leng
         self.tokenizer.padding_side = 'right'
         self.tokenizer.truncation_side = 'right'
 
-        tokens = self.tokenizer.batch_encode_plus(messages, **tokenize_kwargs)
+        tokens = self.tokenizer(messages, **tokenize_kwargs)
 
         tokens = {k: v.to(self.model.device) for k, v in tokens.items()}
         outputs = self.model(**tokens)[0]
@@ -466,11 +466,11 @@ def generate(self,
         )
         if self.fastchat_template:
             messages = _format_with_fast_chat_template(messages, self.fastchat_template)
-            tokens = self.tokenizer.batch_encode_plus(messages, **tokenize_kwargs)
+            tokens = self.tokenizer(messages, **tokenize_kwargs)
         else:
             messages = [self.tokenizer.apply_chat_template(m, add_generation_prompt=True, tokenize=False) for m in messages]
             tokenize_kwargs['add_special_tokens'] = False
-            tokens = self.tokenizer.batch_encode_plus(messages, **tokenize_kwargs)
+            tokens = self.tokenizer(messages, **tokenize_kwargs)
 
         tokens = {k: v.to(self.model.device) for k, v in tokens.items()}
 
@@ -592,7 +592,7 @@ def generate(self,
                 input_ids = torch.cat([input_ids[:, : self.max_seq_len // 2], input_ids[:, - self.max_seq_len // 2:]], dim=-1)
             tokens = {'input_ids': input_ids, }
         else:
-            tokens = self.tokenizer.batch_encode_plus(messages, **tokenize_kwargs)
+            tokens = self.tokenizer(messages, **tokenize_kwargs)
 
         tokens = {k: v.to(self.model.device) for k, v in tokens.items()}
 
@@ -654,7 +654,7 @@ def get_ppl(self, inputs: List[str], mask_length: Optional[List[int]] = None) ->
                 input_ids = torch.cat([input_ids[:, : self.max_seq_len // 2], input_ids[:, - self.max_seq_len // 2:]], dim=-1)
             tokens = {'input_ids': input_ids, }
         else:
-            tokens = self.tokenizer.batch_encode_plus(messages, **tokenize_kwargs)
+            tokens = self.tokenizer(messages, **tokenize_kwargs)
 
         tokens = {k: v.to(self.model.device) for k, v in tokens.items()}
         outputs = self.model(**tokens)[0]

tests/models/test_huggingface.py

@@ -238,7 +238,7 @@ def test_generate_with_batch_padding(self, mock_transformers,
         """Test generate with batch_padding=True."""
         mock_tokenizer = MagicMock()
         mock_tokenizer.pad_token_id = 0
-        mock_tokenizer.batch_encode_plus.return_value = {
+        mock_tokenizer.return_value = {
             'input_ids': [[1, 2, 3], [4, 5, 6]],
             'attention_mask': [[1, 1, 1], [1, 1, 1]]
         }

tests/models/test_huggingface_above_v4_33.py

@@ -90,7 +90,7 @@ def test_generate_basic(self, mock_get_stopping_criteria,
         mock_input_ids.shape = [1, 3]  # batch_size=1, seq_len=3
         mock_attention_mask = MagicMock()
         mock_attention_mask.to.return_value = mock_attention_mask
-        mock_tokenizer.batch_encode_plus.return_value = {
+        mock_tokenizer.return_value = {
             'input_ids': mock_input_ids,
             'attention_mask': mock_attention_mask
         }
@@ -214,7 +214,7 @@ def test_generate_basic(self, mock_get_stopping_criteria,
         mock_input_ids.to.return_value = mock_input_ids
         mock_attention_mask = MagicMock()
         mock_attention_mask.to.return_value = mock_attention_mask
-        mock_tokenizer.batch_encode_plus.return_value = {
+        mock_tokenizer.return_value = {
             'input_ids': mock_input_ids,
             'attention_mask': mock_attention_mask
         }