tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

make sure to multiply against the correct loss

Browse Source
This commit is contained in:
Wing Lian
2024-12-19 01:42:57 -05:00
parent 7d281b71dc
commit e2fae47114
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/axolotl/core/trainers/kd.py
View File

@@ -121,6 +121,6 @@ class AxolotlKDTrainer(AxolotlTrainer):
] ]
if self.args.average_tokens_across_devices and self.model_accepts_loss_kwargs: if self.args.average_tokens_across_devices and self.model_accepts_loss_kwargs:
loss_kd *= self.accelerator.num_processes loss *= self.accelerator.num_processes
return (loss, outputs) if return_outputs else loss return (loss, outputs) if return_outputs else loss