tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

fix handling of ddp_find_unused_parameters

Browse Source
This commit is contained in:
Wing Lian
2024-03-14 17:45:42 -04:00
parent 8df7b888ff
commit 34eb4e1677
1 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/axolotl/core/trainer_builder.py
View File

@@ -1000,9 +1000,14 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
and self.cfg.eval_steps and self.cfg.eval_steps
and self.cfg.save_steps % self.cfg.eval_steps == 0 and self.cfg.save_steps % self.cfg.eval_steps == 0
) or False ) or False
training_arguments_kwargs["ddp_find_unused_parameters"] = ( ddp_find_unused_parameters = (
False if self.cfg.ddp else None self.cfg.ddp_find_unused_parameters
if self.cfg.ddp_find_unused_parameters is not None
else (False if self.cfg.ddp else None)
) )
training_arguments_kwargs[
"ddp_find_unused_parameters"
] = ddp_find_unused_parameters
training_arguments_kwargs["group_by_length"] = self.cfg.group_by_length training_arguments_kwargs["group_by_length"] = self.cfg.group_by_length
report_to = None report_to = None
if self.cfg.use_wandb: if self.cfg.use_wandb: