tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

support layer replication for peft and fix rslora integration (#1445)

Browse Source
This commit is contained in:
Wing Lian
2024-03-27 10:16:47 -04:00
committed by GitHub
parent da265dd796
commit 25afd35842
2 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/axolotl/utils/models.py
View File

@@ -849,7 +849,9 @@ def load_lora(model, cfg, inference=False, config_only=False):
if cfg.peft_use_dora:
lora_config_kwargs["use_dora"] = cfg.peft_use_dora
if cfg.peft_use_rslora:
lora_config_kwargs["use_rslora"] = cfg.use_rslora
lora_config_kwargs["use_rslora"] = cfg.peft_use_rslora
if cfg.peft_layer_replication:
lora_config_kwargs["peft_layer_replication"] = cfg.peft_layer_replication
lora_config = LoraConfig(
r=cfg.lora_r,