From 370d0570960d47e8674e83075e6cea96a55c9e3a Mon Sep 17 00:00:00 2001 From: jphillips Date: Mon, 29 May 2023 09:07:46 -0500 Subject: [PATCH 1/3] Add qlora-openllama-3b example --- examples/qlora-openllama-3b.yml/config.yml | 61 ++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 examples/qlora-openllama-3b.yml/config.yml diff --git a/examples/qlora-openllama-3b.yml/config.yml b/examples/qlora-openllama-3b.yml/config.yml new file mode 100644 index 000000000..9499474bd --- /dev/null +++ b/examples/qlora-openllama-3b.yml/config.yml @@ -0,0 +1,61 @@ +base_model: openlm-research/open_llama_3b_600bt_preview +base_model_config: openlm-research/open_llama_3b_600bt_preview +model_type: LlamaForCausalLM +tokenizer_type: LlamaTokenizer +load_in_8bit: false +load_in_4bit: true +strict: false +push_dataset_to_hub: +datasets: + - path: teknium/GPT4-LLM-Cleaned + type: alpaca +dataset_prepared_path: last_run_prepared +val_set_size: 0.01 +adapter: qlora +lora_model_dir: +sequence_len: 2048 +max_packed_sequence_len: 2048 +lora_r: 8 +lora_alpha: 32 +lora_dropout: 0.05 +lora_target_modules: +lora_target_linear: true +lora_fan_in_fan_out: +wandb_project: +wandb_watch: +wandb_run_id: +wandb_log_model: +output_dir: ./qlora-out +batch_size: 1 +micro_batch_size: 4 # Changed from 1 to 4 +num_epochs: 2 # Changed from 4 to 2 +optimizer: paged_adamw_32bit +torchdistx_path: +lr_scheduler: cosine +learning_rate: 0.0002 +train_on_inputs: false +group_by_length: true # Changed from false to true +bf16: true +fp16: false +tf32: true +gradient_checkpointing: true +early_stopping_patience: +resume_from_checkpoint: +local_rank: +logging_steps: 1 +xformers_attention: true +flash_attention: +gptq_groupsize: +gptq_model_v1: +warmup_steps: 10 +eval_steps: 20 +save_steps: +debug: +deepspeed: +weight_decay: 0.0 +fsdp: +fsdp_config: +special_tokens: + bos_token: "" + eos_token: "" + unk_token: "" From ac85c0ed36e040c9468b7edb87cadb876b50c1ed Mon Sep 17 00:00:00 2001 From: jphillips Date: Mon, 29 May 2023 14:35:58 -0500 Subject: [PATCH 2/3] Add Readme, Clean up comments --- examples/qlora-openllama-3b/README.md | 6 ++++++ .../config.yml | 8 ++++---- 2 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 examples/qlora-openllama-3b/README.md rename examples/{qlora-openllama-3b.yml => qlora-openllama-3b}/config.yml (88%) diff --git a/examples/qlora-openllama-3b/README.md b/examples/qlora-openllama-3b/README.md new file mode 100644 index 000000000..d3276439b --- /dev/null +++ b/examples/qlora-openllama-3b/README.md @@ -0,0 +1,6 @@ +# qlora-openllama-3b + +```shell +accelerate launch scripts/finetune.py examples/lora-openllama-3b + +``` diff --git a/examples/qlora-openllama-3b.yml/config.yml b/examples/qlora-openllama-3b/config.yml similarity index 88% rename from examples/qlora-openllama-3b.yml/config.yml rename to examples/qlora-openllama-3b/config.yml index 9499474bd..87e1dfd94 100644 --- a/examples/qlora-openllama-3b.yml/config.yml +++ b/examples/qlora-openllama-3b/config.yml @@ -26,15 +26,15 @@ wandb_watch: wandb_run_id: wandb_log_model: output_dir: ./qlora-out -batch_size: 1 -micro_batch_size: 4 # Changed from 1 to 4 -num_epochs: 2 # Changed from 4 to 2 +batch_size: 4 +micro_batch_size: 4 +num_epochs: 2 optimizer: paged_adamw_32bit torchdistx_path: lr_scheduler: cosine learning_rate: 0.0002 train_on_inputs: false -group_by_length: true # Changed from false to true +group_by_length: true bf16: true fp16: false tf32: true From 6cee881d6417f127e23f2b03f6e86bcfc251fce5 Mon Sep 17 00:00:00 2001 From: jphillips <120260158+fearnworks@users.noreply.github.com> Date: Tue, 30 May 2023 09:33:33 -0500 Subject: [PATCH 3/3] Update examples/qlora-openllama-3b/README.md Co-authored-by: Wing Lian --- examples/qlora-openllama-3b/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/qlora-openllama-3b/README.md b/examples/qlora-openllama-3b/README.md index d3276439b..d79ea7f3f 100644 --- a/examples/qlora-openllama-3b/README.md +++ b/examples/qlora-openllama-3b/README.md @@ -1,6 +1,6 @@ # qlora-openllama-3b ```shell -accelerate launch scripts/finetune.py examples/lora-openllama-3b +accelerate launch scripts/finetune.py examples/qlora-openllama-3b/config.yml ```