update sharegpt conversations when chatml chat template is set (#1075) [skip ci]

* update sharegpt conversations when chatml chat template is set

* add info log when updating sharegpt/chatml conversation
This commit is contained in:
Wing Lian
2024-01-10 00:49:07 -05:00
committed by GitHub
parent 043c3860cd
commit 0ce1a6594e
3 changed files with 46 additions and 2 deletions

View File

@@ -25,7 +25,11 @@ from transformers import GenerationConfig, TextIteratorStreamer, TextStreamer
from axolotl.common.cli import TrainerCliArgs, load_model_and_tokenizer
from axolotl.logging_config import configure_logging
from axolotl.train import TrainDatasetMeta
from axolotl.utils.config import normalize_config, validate_config
from axolotl.utils.config import (
normalize_cfg_datasets,
normalize_config,
validate_config,
)
from axolotl.utils.data import prepare_dataset
from axolotl.utils.dict import DictDefault
from axolotl.utils.distributed import is_main_process
@@ -289,6 +293,8 @@ def load_cfg(config: Path = Path("examples/"), **kwargs):
normalize_config(cfg)
normalize_cfg_datasets(cfg)
setup_wandb_env_vars(cfg)
setup_mlflow_env_vars(cfg)