From fb3d40f197471f275c6c1ecfae2761189bfb36ea Mon Sep 17 00:00:00 2001
From: Utensil <utensilcandel@gmail.com>
Date: Thu, 1 Jun 2023 18:29:20 +0800
Subject: [PATCH] falcon + qlora + xformer mbs 40 gas 2 on A6000

---
 examples/falcon/config-7b-qlora.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/falcon/config-7b-qlora.yml b/examples/falcon/config-7b-qlora.yml
index c36fe9bed..f15847f5c 100644
--- a/examples/falcon/config-7b-qlora.yml
+++ b/examples/falcon/config-7b-qlora.yml
@@ -18,7 +18,7 @@ val_set_size: 0.01
 adapter: qlora
 lora_model_dir:
 sequence_len: 2048
-max_packed_sequence_len: 2048
+max_packed_sequence_len:
 lora_r: 64
 lora_alpha: 16
 lora_dropout: 0.05
@@ -30,8 +30,8 @@ wandb_watch:
 wandb_run_id:
 wandb_log_model:
 output_dir: ./qlora-out
-batch_size: 8
-micro_batch_size: 4
+micro_batch_size: 40
+gradient_accumulation_steps: 2
 num_epochs: 3
 optimizer: paged_adamw_32bit
 torchdistx_path:
@@ -50,7 +50,7 @@ resume_from_checkpoint:
 auto_resume_from_checkpoints: true
 local_rank:
 logging_steps: 1
-xformers_attention: false
+xformers_attention: true
 flash_attention:
 gptq_groupsize:
 gptq_model_v1: