diff --git a/src/molgen3D/training/pretraining/dataprocessing/dataloader.py b/src/molgen3D/training/pretraining/dataprocessing/dataloader.py index d96c4a8..77548ad 100644 --- a/src/molgen3D/training/pretraining/dataprocessing/dataloader.py +++ b/src/molgen3D/training/pretraining/dataprocessing/dataloader.py @@ -630,7 +630,6 @@ def _ensure_tokenizer_ready(self) -> None: tokenizer = AutoTokenizer.from_pretrained( str(self._tokenizer_path), use_fast=True, - fix_mistral_regex=True, ) sep_id = ensure_tokenizer_pad_token(tokenizer) self._tokenizer = tokenizer diff --git a/src/molgen3D/training/pretraining/torchtitan_runner.py b/src/molgen3D/training/pretraining/torchtitan_runner.py index e974867..342dec1 100644 --- a/src/molgen3D/training/pretraining/torchtitan_runner.py +++ b/src/molgen3D/training/pretraining/torchtitan_runner.py @@ -207,7 +207,6 @@ def _load_tokenizer_details(tokenizer_path: Path) -> TokenizerDetails: tokenizer = AutoTokenizer.from_pretrained( str(tokenizer_path), use_fast=True, - fix_mistral_regex=True, ) ensure_tokenizer_pad_token(tokenizer) vocab_size = len(tokenizer)