tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

support layer replication for peft and fix rslora integration (#1445)

Browse Source
This commit is contained in:
Wing Lian
2024-03-27 10:16:47 -04:00
committed by GitHub
parent da265dd796
commit 25afd35842
2 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/axolotl/utils/config/models/input/v0_4_1/__init__.py
View File

@@ -6,7 +6,7 @@ Module for pydantic models for configuration
import logging
import os
from enum import Enum
from typing import Any, Dict, List, Literal, Optional, Union
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
from pydantic import BaseModel, Field, conlist, field_validator, model_validator
from transformers import SchedulerType
@@ -179,7 +179,8 @@ class LoraConfig(BaseModel):
peft_layers_to_transform: Optional[List[int]] = None
peft: Optional[PeftConfig] = None
peft_use_dora: Optional[bool] = None
peft_use_relora: Optional[bool] = None
peft_use_rslora: Optional[bool] = None
peft_layer_replication: Optional[List[Tuple[int, int]]] = None
lora_on_cpu: Optional[bool] = None
gptq: Optional[bool] = None

4
src/axolotl/utils/models.py
View File

@@ -849,7 +849,9 @@ def load_lora(model, cfg, inference=False, config_only=False):
if cfg.peft_use_dora:
lora_config_kwargs["use_dora"] = cfg.peft_use_dora
if cfg.peft_use_rslora:
lora_config_kwargs["use_rslora"] = cfg.use_rslora
lora_config_kwargs["use_rslora"] = cfg.peft_use_rslora
if cfg.peft_layer_replication:
lora_config_kwargs["peft_layer_replication"] = cfg.peft_layer_replication
lora_config = LoraConfig(
r=cfg.lora_r,