tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

set chat_template in datasets config automatically (#1664)

Browse Source 
* set chat_template in datasets config automatically

* dynamic chat_template, not jsut chatml
This commit is contained in:
Wing Lian
2024-05-29 22:27:26 -04:00
committed by GitHub
parent f7332ac449
commit 9d4225a058
1 changed files with 9 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/axolotl/utils/config/__init__.py
View File

@@ -187,19 +187,22 @@ def normalize_cfg_datasets(cfg):
helpers for mapping chat_template to various dataset configurations as necessary helpers for mapping chat_template to various dataset configurations as necessary
""" """
if cfg.chat_template and cfg.chat_template == "chatml": if cfg.chat_template:
if cfg.datasets: if cfg.datasets:
for idx, ds_cfg in enumerate(cfg.datasets): for idx, ds_cfg in enumerate(cfg.datasets):
if ds_cfg.type == "sharegpt" and not ds_cfg.conversation: if ds_cfg.type == "sharegpt" and not ds_cfg.conversation:
LOG.info( LOG.info(
f"updating dataset {ds_cfg.path} with `conversation: chatml` to match your chat_template" f"updating dataset {ds_cfg.path} with `conversation: {cfg.chat_template}` to match your chat_template"
) )
cfg.datasets[idx].conversation = "chatml" cfg.datasets[idx].conversation = cfg.chat_template
if ds_cfg.type == "orpo.chat_template" and not ds_cfg.chat_template: if (
ds_cfg.type in ["orpo.chat_template", "chat_template"]
and not ds_cfg.chat_template
):
LOG.info( LOG.info(
f"updating dataset {ds_cfg.path} with `chat_template: chatml` to match your chat_template" f"updating dataset {ds_cfg.path} with `chat_template: {cfg.chat_template}` to match your chat_template"
) )
cfg.datasets[idx].chat_template = "chatml" cfg.datasets[idx].chat_template = cfg.chat_template
def validate_config(cfg: DictDefault, capabilities: Optional[dict] = None): def validate_config(cfg: DictDefault, capabilities: Optional[dict] = None):