From fa5efbf235be2ea9a60b77405bc09af1f01f39d7 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Tue, 21 Jan 2025 15:45:21 -0500 Subject: [PATCH] don't scale delta before decomposing --- src/axolotl/integrations/rrt/cli/convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axolotl/integrations/rrt/cli/convert.py b/src/axolotl/integrations/rrt/cli/convert.py index 7aceb5175..47cc66581 100644 --- a/src/axolotl/integrations/rrt/cli/convert.py +++ b/src/axolotl/integrations/rrt/cli/convert.py @@ -134,7 +134,7 @@ def decompose_delta_weight(layer_weight, avg_weight, alpha, rank): delta_first_pass = final_weight - base_weight - delta_for_svd = delta_first_pass / scaling + delta_for_svd = delta_first_pass # 3. Low-rank factorization of the delta direction lora_A, lora_B = low_rank_decomposition(delta_for_svd, rank)