tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

make lisa training example work on one 24gb gpu

Browse Source
This commit is contained in:
Aman Karmani
2024-04-02 03:19:54 +00:00
parent 5dd9364c00
commit dfe591435f
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
examples/llama-2/lisa.yml
View File

@@ -25,7 +25,7 @@ lora_dropout:
lora_target_linear:
lora_fan_in_fan_out:
lisa_n_layers: 4
lisa_n_layers: 2
lisa_step_interval: 20
lisa_layers_attribute: model.layers
@@ -35,7 +35,7 @@ wandb_watch:
wandb_name:
wandb_log_model:
gradient_accumulation_steps: 2
gradient_accumulation_steps: 1
micro_batch_size: 1
num_epochs: 1
optimizer: adamw_bnb_8bit