diff --git a/opencompass/openicl/icl_dataset_reader.py b/opencompass/openicl/icl_dataset_reader.py index 5aa4d1c30..55c337ac9 100644 --- a/opencompass/openicl/icl_dataset_reader.py +++ b/opencompass/openicl/icl_dataset_reader.py @@ -266,10 +266,11 @@ def __init__(self, def init_dataset(self): for idx, data in enumerate(self.datalist): - tokenized_data = self.tokenizer.encode_plus(data, - truncation=True, - return_tensors='pt', - verbose=False) + # (tokenizer.__call__ instead of encode_plus, which was removed in transformers 5.x) + tokenized_data = self.tokenizer(data, + truncation=True, + return_tensors='pt', + verbose=False) self.encode_dataset.append({ 'input_ids': tokenized_data.input_ids[0],