tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

re-enable DPO for tests in modal ci (#1374)

Browse Source 
* re-enable DPO for tests in modal ci

* workaround for training args

* don't mixin AxolotlTrainingArguments

* fix mixin order so MRO doesn't result in

 TypeError: non-default argument follows default argument error

* use smaller datasets for dpo tests
This commit is contained in:
Wing Lian
2024-06-03 12:50:44 -04:00
committed by GitHub
parent 5cde06587a
commit 1f151c0d52
2 changed files with 19 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/axolotl/prompt_strategies/orpo/chat_template.py
View File

@@ -56,7 +56,9 @@ class ORPODatasetParsingStrategy:
messages: List[Message] = [] messages: List[Message] = []
if system := prompt.get("system", None): if system := prompt.get("system", None):
messages.append(Message(role="system", content=system, label=False)) messages.append(Message(role="system", content=system, label=False))
messages.append(Message(role="user", content=prompt["prompt"], label=False)) messages.append(
Message(role="user", content=prompt["chosen"][0]["content"], label=False)
)
messages.append( messages.append(
Message( Message(
role="assistant", content=prompt["chosen"][1]["content"], label=True role="assistant", content=prompt["chosen"][1]["content"], label=True
@@ -70,7 +72,9 @@ class ORPODatasetParsingStrategy:
messages: List[Message] = [] messages: List[Message] = []
if system := prompt.get("system", None): if system := prompt.get("system", None):
messages.append(Message(role="system", content=system, label=False)) messages.append(Message(role="system", content=system, label=False))
messages.append(Message(role="user", content=prompt["prompt"], label=False)) messages.append(
Message(role="user", content=prompt["rejected"][0]["content"], label=False)
)
messages.append( messages.append(
Message( Message(
role="assistant", content=prompt["rejected"][1]["content"], label=True role="assistant", content=prompt["rejected"][1]["content"], label=True
@@ -152,8 +156,8 @@ class ORPOTokenizingStrategy(PromptTokenizingStrategy):
def tokenize_prompt(self, prompt): def tokenize_prompt(self, prompt):
# pass the rejected prompt/row to the Prompter to get the formatted prompt # pass the rejected prompt/row to the Prompter to get the formatted prompt
prompt_len = 0 prompt_len = 0
rejected_message_list = self.dataset_parser.get_rejected_conversation_thread( rejected_message_list: MessageList = (
prompt self.dataset_parser.get_rejected_conversation_thread(prompt)
) )
input_ids = [] input_ids = []
labels = [] labels = []
@@ -174,7 +178,9 @@ class ORPOTokenizingStrategy(PromptTokenizingStrategy):
rejected_input_ids = input_ids rejected_input_ids = input_ids
rejected_labels = labels rejected_labels = labels
# pass the chosen prompt/row to the Prompter to get the formatted prompt # pass the chosen prompt/row to the Prompter to get the formatted prompt
chosen_message_list = self.dataset_parser.get_chosen_conversation_thread(prompt) chosen_message_list: MessageList = (
self.dataset_parser.get_chosen_conversation_thread(prompt)
)
input_ids = [] input_ids = []
labels = [] labels = []
for _, (part, label) in enumerate( for _, (part, label) in enumerate(

16
tests/e2e/test_dpo.py
View File

@@ -21,7 +21,6 @@ LOG = logging.getLogger("axolotl.tests.e2e")
os.environ["WANDB_DISABLED"] = "true" os.environ["WANDB_DISABLED"] = "true"
@pytest.mark.skip(reason="doesn't seem to work on modal")
class TestDPOLlamaLora(unittest.TestCase): class TestDPOLlamaLora(unittest.TestCase):
""" """
Test case for DPO Llama models using LoRA Test case for DPO Llama models using LoRA
@@ -45,8 +44,8 @@ class TestDPOLlamaLora(unittest.TestCase):
"rl": "dpo", "rl": "dpo",
"datasets": [ "datasets": [
{ {
"path": "Intel/orca_dpo_pairs", "path": "arcee-ai/distilabel-intel-orca-dpo-pairs-binarized",
"type": "chatml.intel", "type": "chatml.ultra",
"split": "train", "split": "train",
}, },
], ],
@@ -89,8 +88,8 @@ class TestDPOLlamaLora(unittest.TestCase):
"rl": "kto_pair", "rl": "kto_pair",
"datasets": [ "datasets": [
{ {
"path": "Intel/orca_dpo_pairs", "path": "arcee-ai/distilabel-intel-orca-dpo-pairs-binarized",
"type": "chatml.intel", "type": "chatml.ultra",
"split": "train", "split": "train",
}, },
], ],
@@ -133,8 +132,8 @@ class TestDPOLlamaLora(unittest.TestCase):
"rl": "ipo", "rl": "ipo",
"datasets": [ "datasets": [
{ {
"path": "Intel/orca_dpo_pairs", "path": "arcee-ai/distilabel-intel-orca-dpo-pairs-binarized",
"type": "chatml.intel", "type": "chatml.ultra",
"split": "train", "split": "train",
}, },
], ],
@@ -180,7 +179,7 @@ class TestDPOLlamaLora(unittest.TestCase):
"chat_template": "chatml", "chat_template": "chatml",
"datasets": [ "datasets": [
{ {
"path": "argilla/ultrafeedback-binarized-preferences-cleaned", "path": "argilla/distilabel-capybara-dpo-7k-binarized",
"type": "chat_template.argilla", "type": "chat_template.argilla",
"split": "train", "split": "train",
}, },
@@ -206,6 +205,7 @@ class TestDPOLlamaLora(unittest.TestCase):
train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta) train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
assert (Path(temp_dir) / "checkpoint-20/adapter_model.safetensors").exists() assert (Path(temp_dir) / "checkpoint-20/adapter_model.safetensors").exists()
@pytest.mark.skip(reason="Fix the implementation")
@with_temp_dir @with_temp_dir
def test_kto_lora(self, temp_dir): def test_kto_lora(self, temp_dir):
# pylint: disable=duplicate-code # pylint: disable=duplicate-code