tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Fix Google Colab notebook 2024-05 (#1662) [skip ci]

Browse Source 
* include mlflow installation in the colab notebook

Without explicitly installing mlflow the `accelerate launch` command fails.

* update the colab noteboko to use the latest tinyllama config
This commit is contained in:
Maciek
2024-05-28 17:23:52 +02:00
committed by GitHub
parent ef223519c9
commit 5f91064040
1 changed files with 220 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
examples/colab-notebooks/colab-axolotl-example.ipynb
View File

@@ -46,7 +46,7 @@
"!pip install torch==\"2.1.2\"\n", "!pip install torch==\"2.1.2\"\n",
"!pip install -e git+https://github.com/OpenAccess-AI-Collective/axolotl#egg=axolotl\n", "!pip install -e git+https://github.com/OpenAccess-AI-Collective/axolotl#egg=axolotl\n",
"!pip install flash-attn==\"2.5.0\"\n", "!pip install flash-attn==\"2.5.0\"\n",
"!pip install deepspeed==\"0.13.1\"" "!pip install deepspeed==\"0.13.1\"!pip install mlflow==\"2.13.0\""
] ]
}, },
{ {
@@ -73,7 +73,6 @@
"base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T\n", "base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T\n",
"model_type: LlamaForCausalLM\n", "model_type: LlamaForCausalLM\n",
"tokenizer_type: LlamaTokenizer\n", "tokenizer_type: LlamaTokenizer\n",
"is_llama_derived_model: true\n",
"\n", "\n",
"load_in_8bit: false\n", "load_in_8bit: false\n",
"load_in_4bit: true\n", "load_in_4bit: true\n",
@@ -89,8 +88,9 @@
"adapter: qlora\n", "adapter: qlora\n",
"lora_model_dir:\n", "lora_model_dir:\n",
"\n", "\n",
"sequence_len: 1096\n", "sequence_len: 4096\n",
"sample_packing: true\n", "sample_packing: true\n",
"eval_sample_packing: false\n",
"pad_to_sequence_len: true\n", "pad_to_sequence_len: true\n",
"\n", "\n",
"lora_r: 32\n", "lora_r: 32\n",
@@ -106,20 +106,17 @@
"wandb_name:\n", "wandb_name:\n",
"wandb_log_model:\n", "wandb_log_model:\n",
"\n", "\n",
"mlflow_experiment_name: colab-example\n", "gradient_accumulation_steps: 4\n",
"\n", "micro_batch_size: 2\n",
"gradient_accumulation_steps: 1\n",
"micro_batch_size: 1\n",
"num_epochs: 4\n", "num_epochs: 4\n",
"max_steps: 20\n",
"optimizer: paged_adamw_32bit\n", "optimizer: paged_adamw_32bit\n",
"lr_scheduler: cosine\n", "lr_scheduler: cosine\n",
"learning_rate: 0.0002\n", "learning_rate: 0.0002\n",
"\n", "\n",
"train_on_inputs: false\n", "train_on_inputs: false\n",
"group_by_length: false\n", "group_by_length: false\n",
"bf16: false\n", "bf16: auto\n",
"fp16: true\n", "fp16:\n",
"tf32: false\n", "tf32: false\n",
"\n", "\n",
"gradient_checkpointing: true\n", "gradient_checkpointing: true\n",
@@ -128,11 +125,11 @@
"local_rank:\n", "local_rank:\n",
"logging_steps: 1\n", "logging_steps: 1\n",
"xformers_attention:\n", "xformers_attention:\n",
"flash_attention: false\n", "flash_attention: true\n",
"\n", "\n",
"warmup_steps: 10\n", "warmup_steps: 10\n",
"evals_per_epoch:\n", "evals_per_epoch: 4\n",
"saves_per_epoch:\n", "saves_per_epoch: 1\n",
"debug:\n", "debug:\n",
"deepspeed:\n", "deepspeed:\n",
"weight_decay: 0.0\n", "weight_decay: 0.0\n",
@@ -204,13 +201,23 @@
"provenance": [] "provenance": []
}, },
"kernelspec": { "kernelspec": {
"display_name": "Python 3", "display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3" "name": "python3"
}, },
"language_info": { "language_info": {
"name": "python" "codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.1"
} }
}, },
"nbformat": 4, "nbformat": 4,
"nbformat_minor": 0 "nbformat_minor": 4
} }