From e77d62933d7fabf43d96355176b454125314ff58 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Wed, 28 May 2025 08:19:52 -0400 Subject: [PATCH] Fix decay --- src/axolotl/integrations/kd/callbacks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axolotl/integrations/kd/callbacks.py b/src/axolotl/integrations/kd/callbacks.py index 521833477..911c3d517 100644 --- a/src/axolotl/integrations/kd/callbacks.py +++ b/src/axolotl/integrations/kd/callbacks.py @@ -29,7 +29,7 @@ class KDTemperatureSchedulerCallback(TrainerCallback): # This factor goes from 1 (at progress=0) to 0 (at progress=1) decay_factor = 0.5 * (1.0 + math.cos(math.pi * progress)) self.temperature = self.temperature_start - ( - (self.temperature_start - self.temperature_min) * decay_factor + (self.temperature_start - self.temperature_min) * (1.0 - decay_factor) ) if hasattr(self.trainer.data_collator, "kd_temperature"):