Fix decay

2025-05-28 08:19:52 -04:00
parent 3a0faa97ca
commit e77d62933d
1 changed files with 1 additions and 1 deletions
--- a/src/axolotl/integrations/kd/callbacks.py
+++ b/src/axolotl/integrations/kd/callbacks.py
@@ -29,7 +29,7 @@ class KDTemperatureSchedulerCallback(TrainerCallback):
        # This factor goes from 1 (at progress=0) to 0 (at progress=1)
        decay_factor = 0.5 * (1.0 + math.cos(math.pi * progress))
        self.temperature = self.temperature_start - (
-            (self.temperature_start - self.temperature_min) * decay_factor
+            (self.temperature_start - self.temperature_min) * (1.0 - decay_factor)
        )

        if hasattr(self.trainer.data_collator, "kd_temperature"):