tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

make sure to scale too

Browse Source
This commit is contained in:
Wing Lian
2025-01-21 13:22:49 -05:00
parent e3393042e5
commit 59a7ac427d
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/axolotl/integrations/rrt/modeling/linear.py
View File

@@ -98,6 +98,8 @@ class RelaxedRecursiveDoraLinear(nn.Module):
0 0
) # shape [1, out_features] ) # shape [1, out_features]
result_dora = (scale_factor - 1) * base_out + scale_factor * lora_out result_dora = (
scale_factor - 1
) * base_out + scale_factor * lora_out * self.scaling
return result_dora return result_dora
return base_out + lora_out * self.scaling return base_out + lora_out * self.scaling