From 59a7ac427ddd9750b51907fc695e633732be2f44 Mon Sep 17 00:00:00 2001
From: Wing Lian <wing@axolotl.ai>
Date: Tue, 21 Jan 2025 13:22:49 -0500
Subject: [PATCH] make sure to scale too

---
 src/axolotl/integrations/rrt/modeling/linear.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/axolotl/integrations/rrt/modeling/linear.py b/src/axolotl/integrations/rrt/modeling/linear.py
index 009fafc4b..38516801e 100644
--- a/src/axolotl/integrations/rrt/modeling/linear.py
+++ b/src/axolotl/integrations/rrt/modeling/linear.py
@@ -98,6 +98,8 @@ class RelaxedRecursiveDoraLinear(nn.Module):
                 0
             )  # shape [1, out_features]
 
-            result_dora = (scale_factor - 1) * base_out + scale_factor * lora_out
+            result_dora = (
+                scale_factor - 1
+            ) * base_out + scale_factor * lora_out * self.scaling
             return result_dora
         return base_out + lora_out * self.scaling