tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

support layer replication for peft and fix rslora integration (#1445)

Browse Source
This commit is contained in:
Wing Lian
2024-03-27 10:16:47 -04:00
committed by GitHub
parent da265dd796
commit 25afd35842
2 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/axolotl/utils/config/models/input/v0_4_1/__init__.py
View File

@@ -6,7 +6,7 @@ Module for pydantic models for configuration
import logging import logging
import os import os
from enum import Enum from enum import Enum
from typing import Any, Dict, List, Literal, Optional, Union from typing import Any, Dict, List, Literal, Optional, Tuple, Union
from pydantic import BaseModel, Field, conlist, field_validator, model_validator from pydantic import BaseModel, Field, conlist, field_validator, model_validator
from transformers import SchedulerType from transformers import SchedulerType
@@ -179,7 +179,8 @@ class LoraConfig(BaseModel):
peft_layers_to_transform: Optional[List[int]] = None peft_layers_to_transform: Optional[List[int]] = None
peft: Optional[PeftConfig] = None peft: Optional[PeftConfig] = None
peft_use_dora: Optional[bool] = None peft_use_dora: Optional[bool] = None
peft_use_relora: Optional[bool] = None peft_use_rslora: Optional[bool] = None
peft_layer_replication: Optional[List[Tuple[int, int]]] = None
lora_on_cpu: Optional[bool] = None lora_on_cpu: Optional[bool] = None
gptq: Optional[bool] = None gptq: Optional[bool] = None

4
src/axolotl/utils/models.py
View File

@@ -849,7 +849,9 @@ def load_lora(model, cfg, inference=False, config_only=False):
if cfg.peft_use_dora: if cfg.peft_use_dora:
lora_config_kwargs["use_dora"] = cfg.peft_use_dora lora_config_kwargs["use_dora"] = cfg.peft_use_dora
if cfg.peft_use_rslora: if cfg.peft_use_rslora:
lora_config_kwargs["use_rslora"] = cfg.use_rslora lora_config_kwargs["use_rslora"] = cfg.peft_use_rslora
if cfg.peft_layer_replication:
lora_config_kwargs["peft_layer_replication"] = cfg.peft_layer_replication
lora_config = LoraConfig( lora_config = LoraConfig(
r=cfg.lora_r, r=cfg.lora_r,