tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

don't scale delta before decomposing

Browse Source
This commit is contained in:
Wing Lian
2025-01-21 15:45:21 -05:00
parent 59a7ac427d
commit fa5efbf235
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/axolotl/integrations/rrt/cli/convert.py
View File

@@ -134,7 +134,7 @@ def decompose_delta_weight(layer_weight, avg_weight, alpha, rank):
delta_first_pass = final_weight - base_weight delta_first_pass = final_weight - base_weight
delta_for_svd = delta_first_pass / scaling delta_for_svd = delta_first_pass
# 3. Low-rank factorization of the delta direction # 3. Low-rank factorization of the delta direction
lora_A, lora_B = low_rank_decomposition(delta_for_svd, rank) lora_A, lora_B = low_rank_decomposition(delta_for_svd, rank)