From 1c8a58f71f1b8aca5e24cfa06c7f2c47d8290907 Mon Sep 17 00:00:00 2001 From: T4ras123 Date: Fri, 20 Mar 2026 08:34:09 +0000 Subject: [PATCH] Remove `fix_mistral_regex` parameter from tokenizer loading in `torchtitan_runner.py` and `dataloader.py` to streamline tokenizer initialization. --- src/molgen3D/training/pretraining/dataprocessing/dataloader.py | 1 - src/molgen3D/training/pretraining/torchtitan_runner.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/molgen3D/training/pretraining/dataprocessing/dataloader.py b/src/molgen3D/training/pretraining/dataprocessing/dataloader.py index d96c4a8..77548ad 100644 --- a/src/molgen3D/training/pretraining/dataprocessing/dataloader.py +++ b/src/molgen3D/training/pretraining/dataprocessing/dataloader.py @@ -630,7 +630,6 @@ def _ensure_tokenizer_ready(self) -> None: tokenizer = AutoTokenizer.from_pretrained( str(self._tokenizer_path), use_fast=True, - fix_mistral_regex=True, ) sep_id = ensure_tokenizer_pad_token(tokenizer) self._tokenizer = tokenizer diff --git a/src/molgen3D/training/pretraining/torchtitan_runner.py b/src/molgen3D/training/pretraining/torchtitan_runner.py index e974867..342dec1 100644 --- a/src/molgen3D/training/pretraining/torchtitan_runner.py +++ b/src/molgen3D/training/pretraining/torchtitan_runner.py @@ -207,7 +207,6 @@ def _load_tokenizer_details(tokenizer_path: Path) -> TokenizerDetails: tokenizer = AutoTokenizer.from_pretrained( str(tokenizer_path), use_fast=True, - fix_mistral_regex=True, ) ensure_tokenizer_pad_token(tokenizer) vocab_size = len(tokenizer)