tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

use kd_alpha in the correct loss method

Browse Source
This commit is contained in:
Wing Lian
2024-12-24 19:54:32 -05:00
parent 2e6422a711
commit b88128d067
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/axolotl/core/trainers/kd.py
View File

@@ -182,7 +182,8 @@ class AxolotlKDTrainer(AxolotlTrainer):
) )
if self.args.kd_ce_alpha > 0: if self.args.kd_ce_alpha > 0:
loss = self.args.kd_ce_alpha * outputs["loss"] + loss_kd kd_alpha = self.args.kd_alpha
loss = self.args.kd_ce_alpha * outputs["loss"] + kd_alpha * loss_kd
else: else:
loss = loss_kd loss = loss_kd
# Save past state if it exists # Save past state if it exists