update sharegpt conversations when chatml chat template is set (#1075) [skip ci]

* update sharegpt conversations when chatml chat template is set

* add info log when updating sharegpt/chatml conversation
This commit is contained in:
Wing Lian
2024-01-10 00:49:07 -05:00
committed by GitHub
parent 043c3860cd
commit 0ce1a6594e
3 changed files with 46 additions and 2 deletions

View File

@@ -3,7 +3,7 @@ Test classes for checking functionality of the cfg normalization
"""
import unittest
from axolotl.utils.config import normalize_config
from axolotl.utils.config import normalize_cfg_datasets, normalize_config
from axolotl.utils.dict import DictDefault
@@ -44,3 +44,26 @@ class NormalizeConfigTestCase(unittest.TestCase):
normalize_config(cfg)
assert cfg.base_model_config == cfg.base_model
def test_chat_template_chatml(self):
cfg = DictDefault(
{
"chat_template": "chatml",
"datasets": [
{
"path": "lorem/ipsum",
"type": "sharegpt",
"conversation": "vicuna_v1.1",
},
{
"path": "sit/amet",
"type": "sharegpt",
},
],
}
)
normalize_cfg_datasets(cfg)
assert cfg.datasets[0].conversation == "vicuna_v1.1"
assert cfg.datasets[1].conversation == "chatml"