don't scale delta before decomposing

2025-01-21 15:45:21 -05:00
parent 59a7ac427d
commit fa5efbf235
1 changed files with 1 additions and 1 deletions
--- a/src/axolotl/integrations/rrt/cli/convert.py
+++ b/src/axolotl/integrations/rrt/cli/convert.py
@@ -134,7 +134,7 @@ def decompose_delta_weight(layer_weight, avg_weight, alpha, rank):

    delta_first_pass = final_weight - base_weight

-    delta_for_svd = delta_first_pass / scaling
+    delta_for_svd = delta_first_pass

    # 3. Low-rank factorization of the delta direction
    lora_A, lora_B = low_rank_decomposition(delta_for_svd, rank)