tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

support for rslora (#1387) [skip ci]

Browse Source
This commit is contained in:
Wing Lian
2024-03-10 20:49:45 -04:00
committed by GitHub
parent 3fd8093717
commit 7659c001aa
2 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/axolotl/utils/config/models/input/v0_4_1/__init__.py
View File

@@ -179,6 +179,7 @@ class LoraConfig(BaseModel):
peft_layers_to_transform: Optional[List[int]] = None peft_layers_to_transform: Optional[List[int]] = None
peft: Optional[PeftConfig] = None peft: Optional[PeftConfig] = None
peft_use_dora: Optional[bool] = None peft_use_dora: Optional[bool] = None
peft_use_relora: Optional[bool] = None
lora_on_cpu: Optional[bool] = None lora_on_cpu: Optional[bool] = None
gptq: Optional[bool] = None gptq: Optional[bool] = None

2
src/axolotl/utils/models.py
View File

@@ -1055,6 +1055,8 @@ def load_lora(model, cfg, inference=False, config_only=False):
lora_config_kwargs["init_lora_weights"] = "loftq" lora_config_kwargs["init_lora_weights"] = "loftq"
if cfg.peft_use_dora: if cfg.peft_use_dora:
lora_config_kwargs["use_dora"] = cfg.peft_use_dora lora_config_kwargs["use_dora"] = cfg.peft_use_dora
if cfg.peft_use_rslora:
lora_config_kwargs["use_rslora"] = cfg.use_rslora
lora_config = LoraConfig( lora_config = LoraConfig(
r=cfg.lora_r, r=cfg.lora_r,