diff --git a/.nojekyll b/.nojekyll index a67a82018..3d2808dfb 100644 --- a/.nojekyll +++ b/.nojekyll @@ -1 +1 @@ -285f85e7 \ No newline at end of file +0e5e2a74 \ No newline at end of file diff --git a/docs/dataset-formats/index.html b/docs/dataset-formats/index.html index 788d472d6..d58cb214d 100644 --- a/docs/dataset-formats/index.html +++ b/docs/dataset-formats/index.html @@ -351,7 +351,7 @@ Description - + Pre-training @@ -359,7 +359,7 @@ Description Data format for a pre-training completion task. - + Instruction Tuning @@ -367,7 +367,7 @@ Description Instruction tuning formats for supervised fine-tuning. - + Conversation @@ -375,7 +375,7 @@ Description Conversation format for supervised fine-tuning. - + Template-Free @@ -383,7 +383,7 @@ Description Construct prompts without a template. - + Custom Pre-Tokenized Dataset diff --git a/examples/colab-notebooks/colab-axolotl-example.html b/examples/colab-notebooks/colab-axolotl-example.html index 5797ab9e8..ba79fa246 100644 --- a/examples/colab-notebooks/colab-axolotl-example.html +++ b/examples/colab-notebooks/colab-axolotl-example.html @@ -344,7 +344,7 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin type: alpaca dataset_prepared_path: val_set_size: 0.05 -output_dir: ./qlora-out +output_dir: ./outputs/qlora-out adapter: qlora lora_model_dir: diff --git a/examples/llama-2/qlora-fsdp.yml b/examples/llama-2/qlora-fsdp.yml index 93b3b2a60..88029f92d 100644 --- a/examples/llama-2/qlora-fsdp.yml +++ b/examples/llama-2/qlora-fsdp.yml @@ -11,7 +11,7 @@ datasets: type: alpaca dataset_prepared_path: last_run_prepared val_set_size: 0.05 -output_dir: ./qlora-out +output_dir: ./outputs/qlora-out adapter: qlora lora_model_dir: diff --git a/search.json b/search.json index 11008592d..6ebffc568 100644 --- a/search.json +++ b/search.json @@ -781,7 +781,7 @@ "href": "examples/colab-notebooks/colab-axolotl-example.html#create-an-yaml-config-file", "title": "Example notebook for running Axolotl on google colab", "section": "Create an yaml config file", - "text": "Create an yaml config file\n\nimport yaml\n\n# Your YAML string\nyaml_string = \"\"\"\nbase_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T\nmodel_type: LlamaForCausalLM\ntokenizer_type: LlamaTokenizer\nis_llama_derived_model: true\n\nload_in_8bit: false\nload_in_4bit: true\nstrict: false\n\ndatasets:\n - path: mhenrichsen/alpaca_2k_test\n type: alpaca\ndataset_prepared_path:\nval_set_size: 0.05\noutput_dir: ./qlora-out\n\nadapter: qlora\nlora_model_dir:\n\nsequence_len: 1096\nsample_packing: true\npad_to_sequence_len: true\n\nlora_r: 32\nlora_alpha: 16\nlora_dropout: 0.05\nlora_target_modules:\nlora_target_linear: true\nlora_fan_in_fan_out:\n\nwandb_project:\nwandb_entity:\nwandb_watch:\nwandb_name:\nwandb_log_model:\n\nmlflow_experiment_name: colab-example\n\ngradient_accumulation_steps: 1\nmicro_batch_size: 1\nnum_epochs: 4\nmax_steps: 20\noptimizer: paged_adamw_32bit\nlr_scheduler: cosine\nlearning_rate: 0.0002\n\ntrain_on_inputs: false\ngroup_by_length: false\nbf16: false\nfp16: true\ntf32: false\n\ngradient_checkpointing: true\nearly_stopping_patience:\nresume_from_checkpoint:\nlocal_rank:\nlogging_steps: 1\nxformers_attention:\nflash_attention: false\n\nwarmup_steps: 10\nevals_per_epoch:\nsaves_per_epoch:\ndebug:\ndeepspeed:\nweight_decay: 0.0\nfsdp:\nfsdp_config:\nspecial_tokens:\n\n\"\"\"\n\n# Convert the YAML string to a Python dictionary\nyaml_dict = yaml.safe_load(yaml_string)\n\n# Specify your file path\nfile_path = 'test_axolotl.yaml'\n\n# Write the YAML file\nwith open(file_path, 'w') as file:\n yaml.dump(yaml_dict, file)" + "text": "Create an yaml config file\n\nimport yaml\n\n# Your YAML string\nyaml_string = \"\"\"\nbase_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T\nmodel_type: LlamaForCausalLM\ntokenizer_type: LlamaTokenizer\nis_llama_derived_model: true\n\nload_in_8bit: false\nload_in_4bit: true\nstrict: false\n\ndatasets:\n - path: mhenrichsen/alpaca_2k_test\n type: alpaca\ndataset_prepared_path:\nval_set_size: 0.05\noutput_dir: ./outputs/qlora-out\n\nadapter: qlora\nlora_model_dir:\n\nsequence_len: 1096\nsample_packing: true\npad_to_sequence_len: true\n\nlora_r: 32\nlora_alpha: 16\nlora_dropout: 0.05\nlora_target_modules:\nlora_target_linear: true\nlora_fan_in_fan_out:\n\nwandb_project:\nwandb_entity:\nwandb_watch:\nwandb_name:\nwandb_log_model:\n\nmlflow_experiment_name: colab-example\n\ngradient_accumulation_steps: 1\nmicro_batch_size: 1\nnum_epochs: 4\nmax_steps: 20\noptimizer: paged_adamw_32bit\nlr_scheduler: cosine\nlearning_rate: 0.0002\n\ntrain_on_inputs: false\ngroup_by_length: false\nbf16: false\nfp16: true\ntf32: false\n\ngradient_checkpointing: true\nearly_stopping_patience:\nresume_from_checkpoint:\nlocal_rank:\nlogging_steps: 1\nxformers_attention:\nflash_attention: false\n\nwarmup_steps: 10\nevals_per_epoch:\nsaves_per_epoch:\ndebug:\ndeepspeed:\nweight_decay: 0.0\nfsdp:\nfsdp_config:\nspecial_tokens:\n\n\"\"\"\n\n# Convert the YAML string to a Python dictionary\nyaml_dict = yaml.safe_load(yaml_string)\n\n# Specify your file path\nfile_path = 'test_axolotl.yaml'\n\n# Write the YAML file\nwith open(file_path, 'w') as file:\n yaml.dump(yaml_dict, file)" }, { "objectID": "examples/colab-notebooks/colab-axolotl-example.html#launch-the-training", diff --git a/sitemap.xml b/sitemap.xml index d9944aff2..0889a50e5 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -2,90 +2,90 @@ https://OpenAccess-AI-Collective.github.io/axolotl/index.html - 2024-05-15T13:45:43.608Z + 2024-05-15T16:44:23.947Z https://OpenAccess-AI-Collective.github.io/axolotl/TODO.html - 2024-05-15T13:45:43.592Z + 2024-05-15T16:44:23.935Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/multi-node.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.939Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/rlhf.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.939Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/nccl.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.939Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/multipack.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.939Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/dataset-formats/tokenized.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.935Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/dataset-formats/inst_tune.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.935Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/dataset-formats/conversation.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.935Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/batch_vs_grad.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.935Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/input_output.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.939Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/faq.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.935Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/dataset_preprocessing.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.935Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/dataset-formats/template_free.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.935Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/dataset-formats/pretraining.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.935Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/dataset-formats/index.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.935Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/mac.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.939Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/config.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.935Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/debugging.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.935Z https://OpenAccess-AI-Collective.github.io/axolotl/docs/fsdp_qlora.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.935Z https://OpenAccess-AI-Collective.github.io/axolotl/examples/colab-notebooks/colab-axolotl-example.html - 2024-05-15T13:45:43.596Z + 2024-05-15T16:44:23.939Z https://OpenAccess-AI-Collective.github.io/axolotl/FAQS.html - 2024-05-15T13:45:43.592Z + 2024-05-15T16:44:23.935Z