From 34eb4e1677b1be2762b34fc9b0e88a44e3f605e7 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Thu, 14 Mar 2024 17:45:42 -0400 Subject: [PATCH] fix handling of ddp_find_unused_parameters --- src/axolotl/core/trainer_builder.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/axolotl/core/trainer_builder.py b/src/axolotl/core/trainer_builder.py index d11f0c653..c6833369b 100644 --- a/src/axolotl/core/trainer_builder.py +++ b/src/axolotl/core/trainer_builder.py @@ -1000,9 +1000,14 @@ class HFCausalTrainerBuilder(TrainerBuilderBase): and self.cfg.eval_steps and self.cfg.save_steps % self.cfg.eval_steps == 0 ) or False - training_arguments_kwargs["ddp_find_unused_parameters"] = ( - False if self.cfg.ddp else None + ddp_find_unused_parameters = ( + self.cfg.ddp_find_unused_parameters + if self.cfg.ddp_find_unused_parameters is not None + else (False if self.cfg.ddp else None) ) + training_arguments_kwargs[ + "ddp_find_unused_parameters" + ] = ddp_find_unused_parameters training_arguments_kwargs["group_by_length"] = self.cfg.group_by_length report_to = None if self.cfg.use_wandb: