make sure to scale too

2025-01-21 13:22:49 -05:00
parent e3393042e5
commit 59a7ac427d
1 changed files with 3 additions and 1 deletions
--- a/src/axolotl/integrations/rrt/modeling/linear.py
+++ b/src/axolotl/integrations/rrt/modeling/linear.py
@@ -98,6 +98,8 @@ class RelaxedRecursiveDoraLinear(nn.Module):
                0
            )  # shape [1, out_features]

-            result_dora = (scale_factor - 1) * base_out + scale_factor * lora_out
+            result_dora = (
+                scale_factor - 1
+            ) * base_out + scale_factor * lora_out * self.scaling
            return result_dora
        return base_out + lora_out * self.scaling