tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Add cfg.lora_target_linear

Browse Source
This commit is contained in:
NanoCode012
2023-05-26 14:32:30 +09:00
parent bbfc333a01
commit 919623793a
2 changed files with 13 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@@ -232,6 +232,7 @@ lora_target_modules:
# - gate_proj # - gate_proj
# - down_proj # - down_proj
# - up_proj # - up_proj
lora_target_linear: # if true, will target all linear layers
lora_modules_to_save: lora_modules_to_save:
# - embed_tokens # - embed_tokens
# - lm_head # - lm_head

20
src/axolotl/utils/models.py
View File

@@ -364,14 +364,18 @@ def load_lora(model, cfg):
PeftModel, PeftModel,
) )
bits = None lora_target_modules = list(cfg.lora_target_modules)
if cfg.load_in_4bit:
bits = 4 if cfg.lora_target_linear:
elif cfg.load_in_8bit: bits = None
bits = 8 if cfg.load_in_4bit:
linear_names = find_all_linear_names(bits, model) bits = 4
logging.info(f"found linear modules: {repr(linear_names)}") elif cfg.load_in_8bit:
lora_target_modules = list(set(list(cfg.lora_target_modules) + linear_names)) bits = 8
linear_names = find_all_linear_names(bits, model)
logging.info(f"found linear modules: {repr(linear_names)}")
lora_target_modules = list(set(lora_target_modules + linear_names))
lora_config = LoraConfig( lora_config = LoraConfig(
r=cfg.lora_r, r=cfg.lora_r,