tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

falcon + qlora + xformer mbs 40 gas 2 on A6000

Browse Source
This commit is contained in:
Utensil
2023-06-01 18:29:20 +08:00
committed by GitHub
parent 72bf8aafb6
commit fb3d40f197
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
examples/falcon/config-7b-qlora.yml
View File

@@ -18,7 +18,7 @@ val_set_size: 0.01
adapter: qlora adapter: qlora
lora_model_dir: lora_model_dir:
sequence_len: 2048 sequence_len: 2048
max_packed_sequence_len: 2048 max_packed_sequence_len:
lora_r: 64 lora_r: 64
lora_alpha: 16 lora_alpha: 16
lora_dropout: 0.05 lora_dropout: 0.05
@@ -30,8 +30,8 @@ wandb_watch:
wandb_run_id: wandb_run_id:
wandb_log_model: wandb_log_model:
output_dir: ./qlora-out output_dir: ./qlora-out
batch_size: 8 micro_batch_size: 40
micro_batch_size: 4 gradient_accumulation_steps: 2
num_epochs: 3 num_epochs: 3
optimizer: paged_adamw_32bit optimizer: paged_adamw_32bit
torchdistx_path: torchdistx_path:
@@ -50,7 +50,7 @@ resume_from_checkpoint:
auto_resume_from_checkpoints: true auto_resume_from_checkpoints: true
local_rank: local_rank:
logging_steps: 1 logging_steps: 1
xformers_attention: false xformers_attention: true
flash_attention: flash_attention:
gptq_groupsize: gptq_groupsize:
gptq_model_v1: gptq_model_v1: