tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

paired kto support (#1069)

Browse Source
This commit is contained in:
Wing Lian
2024-01-09 13:30:45 -05:00
committed by GitHub
parent 768d348f42
commit d7057ccd36
3 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@@ -595,6 +595,9 @@ datasets:
# For `completion` datsets only, uses the provided field instead of `text` column # For `completion` datsets only, uses the provided field instead of `text` column
field: field:
# use RL training: dpo, ipo, kto_pair
rl:
# Saves the desired chat template to the tokenizer_config.json for easier inferencing # Saves the desired chat template to the tokenizer_config.json for easier inferencing
# Currently supports chatml and inst (mistral/mixtral) # Currently supports chatml and inst (mistral/mixtral)
chat_template: chatml chat_template: chatml

2
requirements.txt
View File

@@ -40,4 +40,4 @@ s3fs
gcsfs gcsfs
# adlfs # adlfs
trl @ git+https://github.com/huggingface/trl.git@main trl>=0.7.9

2
src/axolotl/core/trainer_builder.py
View File

@@ -927,6 +927,8 @@ class HFDPOTrainerBuilder(TrainerBuilderBase):
dpo_trainer_kwargs["loss_type"] = "ipo" dpo_trainer_kwargs["loss_type"] = "ipo"
if self.cfg.dpo_label_smoothing: if self.cfg.dpo_label_smoothing:
dpo_trainer_kwargs["label_smoothing"] = self.cfg.dpo_label_smoothing dpo_trainer_kwargs["label_smoothing"] = self.cfg.dpo_label_smoothing
elif self.cfg.rl == "kto_pair":
dpo_trainer_kwargs["loss_type"] = "kto_pair"
dpo_trainer = DPOTrainer( dpo_trainer = DPOTrainer(
self.model, self.model,