diff --git a/.nojekyll b/.nojekyll index 98b401011..c02634b4c 100644 --- a/.nojekyll +++ b/.nojekyll @@ -1 +1 @@ -bb4817bf \ No newline at end of file +9f29351e \ No newline at end of file diff --git a/docs/dataset-formats/index.html b/docs/dataset-formats/index.html index 8fdb831fc..3b2213b2b 100644 --- a/docs/dataset-formats/index.html +++ b/docs/dataset-formats/index.html @@ -357,7 +357,7 @@ Description - + Pre-training @@ -365,7 +365,7 @@ Description Data format for a pre-training completion task. - + Instruction Tuning @@ -373,7 +373,7 @@ Description Instruction tuning formats for supervised fine-tuning. - + Conversation @@ -381,7 +381,7 @@ Description Conversation format for supervised fine-tuning. - + Template-Free @@ -389,7 +389,7 @@ Description Construct prompts without a template. - + Custom Pre-Tokenized Dataset diff --git a/docs/unsloth.html b/docs/unsloth.html index 9dda6b9c2..0fc85721c 100644 --- a/docs/unsloth.html +++ b/docs/unsloth.html @@ -340,9 +340,9 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin unsloth_lora_qkv: true unsloth_lora_o: true

These options are composable and can be used with multi-gpu finetuning

-
unsloth_cross_entropy_loss: true
-unsloth_rms_norm: true
-unsloth_rope: true
+
unsloth_cross_entropy_loss: true
+unsloth_rms_norm: true
+unsloth_rope: true

Limitations

diff --git a/index.html b/index.html index 0561ecab7..f52f407c6 100644 --- a/index.html +++ b/index.html @@ -535,6 +535,16 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin ✅ ❓ + +Jamba +✅ +✅ +✅ +❓ +❓ +✅ +❓ +

✅: supported ❌: not supported ❓: untested

diff --git a/search.json b/search.json index 1da176ff4..663cd733d 100644 --- a/search.json +++ b/search.json @@ -14,7 +14,7 @@ "href": "index.html#axolotl-supports", "title": "Axolotl", "section": "Axolotl supports", - "text": "Axolotl supports\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nfp16/fp32\nlora\nqlora\ngptq\ngptq w/flash attn\nflash attn\nxformers attn\n\n\n\n\nllama\n✅\n✅\n✅\n✅\n✅\n✅\n✅\n\n\nMistral\n✅\n✅\n✅\n✅\n✅\n✅\n✅\n\n\nMixtral-MoE\n✅\n✅\n✅\n❓\n❓\n❓\n❓\n\n\nMixtral8X22\n✅\n✅\n✅\n❓\n❓\n❓\n❓\n\n\nPythia\n✅\n✅\n✅\n❌\n❌\n❌\n❓\n\n\ncerebras\n✅\n✅\n✅\n❌\n❌\n❌\n❓\n\n\nbtlm\n✅\n✅\n✅\n❌\n❌\n❌\n❓\n\n\nmpt\n✅\n❌\n❓\n❌\n❌\n❌\n❓\n\n\nfalcon\n✅\n✅\n✅\n❌\n❌\n❌\n❓\n\n\ngpt-j\n✅\n✅\n✅\n❌\n❌\n❓\n❓\n\n\nXGen\n✅\n❓\n✅\n❓\n❓\n❓\n✅\n\n\nphi\n✅\n✅\n✅\n❓\n❓\n❓\n❓\n\n\nRWKV\n✅\n❓\n❓\n❓\n❓\n❓\n❓\n\n\nQwen\n✅\n✅\n✅\n❓\n❓\n❓\n❓\n\n\nGemma\n✅\n✅\n✅\n❓\n❓\n✅\n❓\n\n\n\n✅: supported ❌: not supported ❓: untested", + "text": "Axolotl supports\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nfp16/fp32\nlora\nqlora\ngptq\ngptq w/flash attn\nflash attn\nxformers attn\n\n\n\n\nllama\n✅\n✅\n✅\n✅\n✅\n✅\n✅\n\n\nMistral\n✅\n✅\n✅\n✅\n✅\n✅\n✅\n\n\nMixtral-MoE\n✅\n✅\n✅\n❓\n❓\n❓\n❓\n\n\nMixtral8X22\n✅\n✅\n✅\n❓\n❓\n❓\n❓\n\n\nPythia\n✅\n✅\n✅\n❌\n❌\n❌\n❓\n\n\ncerebras\n✅\n✅\n✅\n❌\n❌\n❌\n❓\n\n\nbtlm\n✅\n✅\n✅\n❌\n❌\n❌\n❓\n\n\nmpt\n✅\n❌\n❓\n❌\n❌\n❌\n❓\n\n\nfalcon\n✅\n✅\n✅\n❌\n❌\n❌\n❓\n\n\ngpt-j\n✅\n✅\n✅\n❌\n❌\n❓\n❓\n\n\nXGen\n✅\n❓\n✅\n❓\n❓\n❓\n✅\n\n\nphi\n✅\n✅\n✅\n❓\n❓\n❓\n❓\n\n\nRWKV\n✅\n❓\n❓\n❓\n❓\n❓\n❓\n\n\nQwen\n✅\n✅\n✅\n❓\n❓\n❓\n❓\n\n\nGemma\n✅\n✅\n✅\n❓\n❓\n✅\n❓\n\n\nJamba\n✅\n✅\n✅\n❓\n❓\n✅\n❓\n\n\n\n✅: supported ❌: not supported ❓: untested", "crumbs": [ "Home" ] diff --git a/sitemap.xml b/sitemap.xml index 929853a46..e6bf63a8a 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -2,98 +2,98 @@ https://axolotl-ai-cloud.github.io/axolotl/index.html - 2024-08-21T17:38:00.479Z + 2024-08-22T15:47:45.769Z https://axolotl-ai-cloud.github.io/axolotl/FAQS.html - 2024-08-21T17:38:00.463Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/multipack.html - 2024-08-21T17:38:00.467Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/batch_vs_grad.html - 2024-08-21T17:38:00.463Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/input_output.html - 2024-08-21T17:38:00.467Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/multi-node.html - 2024-08-21T17:38:00.467Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/index.html - 2024-08-21T17:38:00.463Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/tokenized.html - 2024-08-21T17:38:00.463Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/template_free.html - 2024-08-21T17:38:00.463Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/faq.html - 2024-08-21T17:38:00.463Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/torchao.html - 2024-08-21T17:38:00.467Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/nccl.html - 2024-08-21T17:38:00.467Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/TODO.html - 2024-08-21T17:38:00.463Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/config.html - 2024-08-21T17:38:00.463Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/dataset_preprocessing.html - 2024-08-21T17:38:00.463Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/pretraining.html - 2024-08-21T17:38:00.463Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/inst_tune.html - 2024-08-21T17:38:00.463Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/conversation.html - 2024-08-21T17:38:00.463Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/mac.html - 2024-08-21T17:38:00.467Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/debugging.html - 2024-08-21T17:38:00.463Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/fsdp_qlora.html - 2024-08-21T17:38:00.463Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/rlhf.html - 2024-08-21T17:38:00.467Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/docs/unsloth.html - 2024-08-21T17:38:00.467Z + 2024-08-22T15:47:45.757Z https://axolotl-ai-cloud.github.io/axolotl/examples/colab-notebooks/colab-axolotl-example.html - 2024-08-21T17:38:00.467Z + 2024-08-22T15:47:45.761Z