diff --git a/examples/cerebras/qlora.yml b/examples/cerebras/qlora.yml index eec16193d..a13517f3e 100644 --- a/examples/cerebras/qlora.yml +++ b/examples/cerebras/qlora.yml @@ -7,7 +7,7 @@ push_dataset_to_hub: datasets: - path: teknium/GPT4-LLM-Cleaned type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 adapter: qlora lora_model_dir: diff --git a/examples/code-llama/13b/lora.yml b/examples/code-llama/13b/lora.yml index 071b2bc05..91807846b 100644 --- a/examples/code-llama/13b/lora.yml +++ b/examples/code-llama/13b/lora.yml @@ -11,7 +11,7 @@ strict: false datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 output_dir: ./lora-out diff --git a/examples/code-llama/13b/qlora.yml b/examples/code-llama/13b/qlora.yml index 3de4b3b4b..9fa05ffab 100644 --- a/examples/code-llama/13b/qlora.yml +++ b/examples/code-llama/13b/qlora.yml @@ -11,7 +11,7 @@ strict: false datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 output_dir: ./qlora-out diff --git a/examples/code-llama/34b/lora.yml b/examples/code-llama/34b/lora.yml index e52c11181..a342b6ebc 100644 --- a/examples/code-llama/34b/lora.yml +++ b/examples/code-llama/34b/lora.yml @@ -11,7 +11,7 @@ strict: false datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 output_dir: ./lora-out diff --git a/examples/code-llama/34b/qlora.yml b/examples/code-llama/34b/qlora.yml index 7e6da2e96..1501dd9a3 100644 --- a/examples/code-llama/34b/qlora.yml +++ b/examples/code-llama/34b/qlora.yml @@ -11,7 +11,7 @@ strict: false datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 output_dir: ./qlora-out diff --git a/examples/code-llama/7b/lora.yml b/examples/code-llama/7b/lora.yml index 5e9d12e7c..638dddc43 100644 --- a/examples/code-llama/7b/lora.yml +++ b/examples/code-llama/7b/lora.yml @@ -11,7 +11,7 @@ strict: false datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 output_dir: ./lora-out diff --git a/examples/code-llama/7b/qlora.yml b/examples/code-llama/7b/qlora.yml index b24b5c162..5b3b33822 100644 --- a/examples/code-llama/7b/qlora.yml +++ b/examples/code-llama/7b/qlora.yml @@ -11,7 +11,7 @@ strict: false datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 output_dir: ./qlora-out diff --git a/examples/falcon/config-7b-lora.yml b/examples/falcon/config-7b-lora.yml index 738068a47..f45deb643 100644 --- a/examples/falcon/config-7b-lora.yml +++ b/examples/falcon/config-7b-lora.yml @@ -12,7 +12,7 @@ push_dataset_to_hub: datasets: - path: teknium/GPT4-LLM-Cleaned type: alpaca:chat -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 adapter: lora lora_model_dir: diff --git a/examples/falcon/config-7b-qlora.yml b/examples/falcon/config-7b-qlora.yml index 554081fcb..f59341965 100644 --- a/examples/falcon/config-7b-qlora.yml +++ b/examples/falcon/config-7b-qlora.yml @@ -18,7 +18,7 @@ datasets: data_files: - Chain-of-Thought/formatted_cot_data/gsm8k_train.json type: "alpaca:chat" -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 # enable QLoRA adapter: qlora diff --git a/examples/falcon/config-7b.yml b/examples/falcon/config-7b.yml index 25e67a53b..777a97b31 100644 --- a/examples/falcon/config-7b.yml +++ b/examples/falcon/config-7b.yml @@ -12,7 +12,7 @@ push_dataset_to_hub: datasets: - path: teknium/GPT4-LLM-Cleaned type: alpaca:chat -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 adapter: lora_model_dir: diff --git a/examples/gptj/qlora.yml b/examples/gptj/qlora.yml index b41c946fc..696747dfe 100644 --- a/examples/gptj/qlora.yml +++ b/examples/gptj/qlora.yml @@ -7,7 +7,7 @@ push_dataset_to_hub: datasets: - path: teknium/GPT4-LLM-Cleaned type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 adapter: qlora lora_model_dir: diff --git a/examples/jeopardy-bot/config.yml b/examples/jeopardy-bot/config.yml index 1aaa1b557..32e7a34ee 100644 --- a/examples/jeopardy-bot/config.yml +++ b/examples/jeopardy-bot/config.yml @@ -6,7 +6,7 @@ load_in_8bit: false datasets: - path: openaccess-ai-collective/jeopardy type: jeopardy -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.02 adapter: lora_model_dir: diff --git a/examples/llama-2/gptq-lora.yml b/examples/llama-2/gptq-lora.yml index 0d96e4610..257433f26 100644 --- a/examples/llama-2/gptq-lora.yml +++ b/examples/llama-2/gptq-lora.yml @@ -15,7 +15,7 @@ hf_use_auth_token: true datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 adapter: lora lora_model_dir: diff --git a/examples/llama-2/lora.yml b/examples/llama-2/lora.yml index 106de6e32..8c0e3e910 100644 --- a/examples/llama-2/lora.yml +++ b/examples/llama-2/lora.yml @@ -11,7 +11,7 @@ strict: false datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 output_dir: ./lora-out diff --git a/examples/llama-2/qlora.yml b/examples/llama-2/qlora.yml index 870c3fa84..b8209934c 100644 --- a/examples/llama-2/qlora.yml +++ b/examples/llama-2/qlora.yml @@ -11,7 +11,7 @@ strict: false datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 output_dir: ./qlora-out diff --git a/examples/llama-2/relora.yml b/examples/llama-2/relora.yml index 2b1fad55d..9f27cafea 100644 --- a/examples/llama-2/relora.yml +++ b/examples/llama-2/relora.yml @@ -11,7 +11,7 @@ strict: false datasets: - path: teknium/GPT4-LLM-Cleaned type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 output_dir: ./relora-out diff --git a/examples/llama-2/tiny-llama.yml b/examples/llama-2/tiny-llama.yml index d556a6d71..0b56ea7d3 100644 --- a/examples/llama-2/tiny-llama.yml +++ b/examples/llama-2/tiny-llama.yml @@ -12,7 +12,7 @@ strict: false datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 output_dir: ./lora-out diff --git a/examples/mistral/config.yml b/examples/mistral/config.yml index d199f947b..e644bec50 100644 --- a/examples/mistral/config.yml +++ b/examples/mistral/config.yml @@ -11,7 +11,7 @@ strict: false datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 output_dir: ./out diff --git a/examples/mpt-7b/config.yml b/examples/mpt-7b/config.yml index 0efd28cee..8d9b429b1 100644 --- a/examples/mpt-7b/config.yml +++ b/examples/mpt-7b/config.yml @@ -6,7 +6,7 @@ load_in_8bit: false datasets: - path: vicgalle/alpaca-gpt4 type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.02 adapter: lora_model_dir: diff --git a/examples/openllama-3b/config.yml b/examples/openllama-3b/config.yml index 961aeabda..dd11d53b0 100644 --- a/examples/openllama-3b/config.yml +++ b/examples/openllama-3b/config.yml @@ -9,7 +9,7 @@ push_dataset_to_hub: datasets: - path: teknium/GPT4-LLM-Cleaned type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.02 adapter: lora_model_dir: diff --git a/examples/openllama-3b/lora.yml b/examples/openllama-3b/lora.yml index 17fa7fa8b..fad3fb551 100644 --- a/examples/openllama-3b/lora.yml +++ b/examples/openllama-3b/lora.yml @@ -9,7 +9,7 @@ push_dataset_to_hub: datasets: - path: teknium/GPT4-LLM-Cleaned type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.02 adapter: lora lora_model_dir: diff --git a/examples/openllama-3b/qlora.yml b/examples/openllama-3b/qlora.yml index deba03fd5..80d4d727b 100644 --- a/examples/openllama-3b/qlora.yml +++ b/examples/openllama-3b/qlora.yml @@ -9,7 +9,7 @@ push_dataset_to_hub: datasets: - path: teknium/GPT4-LLM-Cleaned type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 adapter: qlora lora_model_dir: diff --git a/examples/phi/phi-ft.yml b/examples/phi/phi-ft.yml index 9eb108049..668eea317 100644 --- a/examples/phi/phi-ft.yml +++ b/examples/phi/phi-ft.yml @@ -13,7 +13,7 @@ datasets: - path: garage-bAInd/Open-Platypus type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.05 output_dir: ./phi-sft-out diff --git a/examples/phi/phi-qlora.yml b/examples/phi/phi-qlora.yml index f3a06c39d..a548b3f05 100644 --- a/examples/phi/phi-qlora.yml +++ b/examples/phi/phi-qlora.yml @@ -13,7 +13,7 @@ datasets: - path: garage-bAInd/Open-Platypus type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.05 output_dir: ./phi-sft-out diff --git a/examples/pythia-12b/config.yml b/examples/pythia-12b/config.yml index dc06eb6b6..4e0e1523a 100644 --- a/examples/pythia-12b/config.yml +++ b/examples/pythia-12b/config.yml @@ -10,7 +10,7 @@ device_map: auto datasets: - path: vicgalle/alpaca-gpt4 type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.05 adapter: lora_model_dir: diff --git a/examples/pythia/lora.yml b/examples/pythia/lora.yml index dfe7ef846..f9a83c059 100644 --- a/examples/pythia/lora.yml +++ b/examples/pythia/lora.yml @@ -4,7 +4,7 @@ load_in_8bit: true datasets: - path: teknium/GPT4-LLM-Cleaned type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.05 adapter: lora lora_model_dir: diff --git a/examples/redpajama/config-3b.yml b/examples/redpajama/config-3b.yml index 10297d860..97f31c87a 100644 --- a/examples/redpajama/config-3b.yml +++ b/examples/redpajama/config-3b.yml @@ -7,7 +7,7 @@ load_in_8bit: false datasets: - path: vicgalle/alpaca-gpt4 type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.02 adapter: lora_model_dir: diff --git a/examples/replit-3b/config-lora.yml b/examples/replit-3b/config-lora.yml index 71a80d17d..d345e25a0 100644 --- a/examples/replit-3b/config-lora.yml +++ b/examples/replit-3b/config-lora.yml @@ -5,7 +5,7 @@ load_in_8bit: false datasets: - path: vicgalle/alpaca-gpt4 type: alpaca -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.05 adapter: lora lora_model_dir: diff --git a/examples/xgen-7b/xgen-7b-8k-qlora.yml b/examples/xgen-7b/xgen-7b-8k-qlora.yml index 0f215afb9..352dcb610 100644 --- a/examples/xgen-7b/xgen-7b-8k-qlora.yml +++ b/examples/xgen-7b/xgen-7b-8k-qlora.yml @@ -16,7 +16,7 @@ datasets: data_files: - openassistant_best_replies_train.jsonl type: "completion" -dataset_prepared_path: last_run_prepared +dataset_prepared_path: val_set_size: 0.01 # enable QLoRA adapter: qlora diff --git a/src/axolotl/cli/__init__.py b/src/axolotl/cli/__init__.py index c3b580391..c21d93170 100644 --- a/src/axolotl/cli/__init__.py +++ b/src/axolotl/cli/__init__.py @@ -51,7 +51,7 @@ def print_axolotl_text_art(suffix=None): def get_multi_line_input() -> Optional[str]: - print("Give me an instruction (Ctrl + D to finish): ") + print("Give me an instruction (Ctrl + D to submit): ") instruction = "" for line in sys.stdin: instruction += line # pylint: disable=consider-using-join diff --git a/src/axolotl/utils/data.py b/src/axolotl/utils/data.py index 34a5baaff..bdad21fb1 100644 --- a/src/axolotl/utils/data.py +++ b/src/axolotl/utils/data.py @@ -122,7 +122,7 @@ def load_tokenized_prepared_datasets( if dataset: ... - elif any(prepared_ds_path.glob("*")): + elif cfg.dataset_prepared_path and any(prepared_ds_path.glob("*")): LOG.info(f"Loading prepared dataset from disk at {prepared_ds_path}...") dataset = load_from_disk(str(prepared_ds_path)) LOG.info("Prepared dataset loaded from disk...") @@ -357,7 +357,7 @@ def load_tokenized_prepared_datasets( if len(datasets) > 1: LOG.info("shuffle merged datasets") dataset = dataset.shuffle(seed=seed) - if cfg.local_rank == 0: + if cfg.local_rank == 0 and cfg.dataset_prepared_path: LOG.info(f"Saving merged prepared dataset to disk... {prepared_ds_path}") dataset.save_to_disk(prepared_ds_path) if cfg.push_dataset_to_hub: @@ -425,7 +425,7 @@ def load_prepare_datasets( if dataset: ... - elif any(prepared_ds_path.glob("*")): + elif cfg.dataset_prepared_path and any(prepared_ds_path.glob("*")): LOG.info( f"Loading prepared packed dataset from disk at {prepared_ds_path}..." ) diff --git a/src/axolotl/utils/tokenization.py b/src/axolotl/utils/tokenization.py index 2b7608b63..deb9bb00e 100644 --- a/src/axolotl/utils/tokenization.py +++ b/src/axolotl/utils/tokenization.py @@ -31,7 +31,8 @@ def check_example_labels(example, tokenizer, text_only=False): ) colored_tokens.append(colored_token) - LOG.info(" ".join(colored_tokens)) + delimiter = "" if text_only else " " + LOG.info(delimiter.join(colored_tokens)) LOG.info("\n\n\n") print(" ".join(colored_tokens))