diff --git a/.nojekyll b/.nojekyll index 6b71a7f0e..3d0fd91e4 100644 --- a/.nojekyll +++ b/.nojekyll @@ -1 +1 @@ -d5d7dce8 \ No newline at end of file +1afde4ec \ No newline at end of file diff --git a/docs/custom_integrations.html b/docs/custom_integrations.html index 026b86490..b7b3348b9 100644 --- a/docs/custom_integrations.html +++ b/docs/custom_integrations.html @@ -558,56 +558,54 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); -
pip3 uninstall -y cut-cross-entropy && pip3 install "cut-cross-entropy[transformers] @ git+https://github.com/apple/ml-cross-entropy.git@bad6f7b49c75fdec69471abb71b4cddd0f0c6438"
+
pip3 uninstall -y cut-cross-entropy && pip3 install "cut-cross-entropy[transformers] @ git+https://github.com/axolotl-ai-cloud/ml-cross-entropy.git@78b2a45713a54c9bedf8b33f5e31cf07a1a57154"

Usage

-

NOTE: If you are training a VLM model, please use older version of Axolotl as upstream has applied a major VLM refactor, and our patches have not been updated yet.

-
git checkout 787880215b3ab32ccaf81c1b2e9588c6f3e6e764
-
-pip3 install --no-build-isolation -e .
-
plugins:
-  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
+
plugins:
+  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin

Supported Models

Citation

-
@article{wijmans2024cut,
-  author       = {Erik Wijmans and
-                  Brody Huval and
-                  Alexander Hertzberg and
-                  Vladlen Koltun and
-                  Philipp Kr\"ahenb\"uhl},
-  title        = {Cut Your Losses in Large-Vocabulary Language Models},
-  journal      = {arXiv},
-  year         = {2024},
-  url          = {https://arxiv.org/abs/2411.09009},
-}
+
@article{wijmans2024cut,
+  author       = {Erik Wijmans and
+                  Brody Huval and
+                  Alexander Hertzberg and
+                  Vladlen Koltun and
+                  Philipp Kr\"ahenb\"uhl},
+  title        = {Cut Your Losses in Large-Vocabulary Language Models},
+  journal      = {arXiv},
+  year         = {2024},
+  url          = {https://arxiv.org/abs/2411.09009},
+}

Please see reference here

@@ -616,20 +614,20 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});

See https://github.com/ironjr/grokfast

Usage

-
plugins:
-  - axolotl.integrations.grokfast.GrokfastPlugin
-
-grokfast_alpha: 2.0
-grokfast_lamb: 0.98
+
plugins:
+  - axolotl.integrations.grokfast.GrokfastPlugin
+
+grokfast_alpha: 2.0
+grokfast_lamb: 0.98

Citation

-
@article{lee2024grokfast,
-    title={{Grokfast}: Accelerated Grokking by Amplifying Slow Gradients},
-    author={Lee, Jaerin and Kang, Bong Gyun and Kim, Kihoon and Lee, Kyoung Mu},
-    journal={arXiv preprint arXiv:2405.20233},
-    year={2024}
-}
+
@article{lee2024grokfast,
+    title={{Grokfast}: Accelerated Grokking by Amplifying Slow Gradients},
+    author={Lee, Jaerin and Kang, Bong Gyun and Kim, Kihoon and Lee, Kyoung Mu},
+    journal={arXiv preprint arXiv:2405.20233},
+    year={2024}
+}

Please see reference here

@@ -637,21 +635,21 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});

Knowledge Distillation (KD)

Usage

-
plugins:
-  - "axolotl.integrations.kd.KDPlugin"
-
-kd_trainer: True
-kd_ce_alpha: 0.1
-kd_alpha: 0.9
-kd_temperature: 1.0
-
-torch_compile: True  # torch>=2.5.1, recommended to reduce vram
-
-datasets:
-  - path: ...
-    type: "axolotl.integrations.kd.chat_template"
-    field_messages: "messages_combined"
-    logprobs_field: "llm_text_generation_vllm_logprobs"  # for kd only, field of logprobs
+
plugins:
+  - "axolotl.integrations.kd.KDPlugin"
+
+kd_trainer: True
+kd_ce_alpha: 0.1
+kd_alpha: 0.9
+kd_temperature: 1.0
+
+torch_compile: True  # torch>=2.5.1, recommended to reduce vram
+
+datasets:
+  - path: ...
+    type: "axolotl.integrations.kd.chat_template"
+    field_messages: "messages_combined"
+    logprobs_field: "llm_text_generation_vllm_logprobs"  # for kd only, field of logprobs

An example dataset can be found at axolotl-ai-co/evolkit-logprobs-pipeline-75k-v2-sample

Please see reference here

@@ -667,13 +665,13 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});

See https://github.com/linkedin/Liger-Kernel

Usage

-
plugins:
-  - axolotl.integrations.liger.LigerPlugin
-liger_rope: true
-liger_rms_norm: true
-liger_glu_activation: true
-liger_layer_norm: true
-liger_fused_linear_cross_entropy: true
+
plugins:
+  - axolotl.integrations.liger.LigerPlugin
+liger_rope: true
+liger_rms_norm: true
+liger_glu_activation: true
+liger_layer_norm: true
+liger_fused_linear_cross_entropy: true

Supported Models

@@ -699,16 +697,16 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});

Citation

-
@article{hsu2024ligerkernelefficienttriton,
-      title={Liger Kernel: Efficient Triton Kernels for LLM Training},
-      author={Pin-Lun Hsu and Yun Dai and Vignesh Kothapalli and Qingquan Song and Shao Tang and Siyu Zhu and Steven Shimizu and Shivam Sahni and Haowen Ning and Yanning Chen},
-      year={2024},
-      eprint={2410.10989},
-      archivePrefix={arXiv},
-      primaryClass={cs.LG},
-      url={https://arxiv.org/abs/2410.10989},
-      journal={arXiv preprint arXiv:2410.10989},
-}
+
@article{hsu2024ligerkernelefficienttriton,
+      title={Liger Kernel: Efficient Triton Kernels for LLM Training},
+      author={Pin-Lun Hsu and Yun Dai and Vignesh Kothapalli and Qingquan Song and Shao Tang and Siyu Zhu and Steven Shimizu and Shivam Sahni and Haowen Ning and Yanning Chen},
+      year={2024},
+      eprint={2410.10989},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG},
+      url={https://arxiv.org/abs/2410.10989},
+      journal={arXiv preprint arXiv:2410.10989},
+}

Please see reference here

@@ -718,29 +716,29 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});

See https://github.com/EleutherAI/lm-evaluation-harness

Usage

-
plugins:
-  - axolotl.integrations.lm_eval.LMEvalPlugin
-
-lm_eval_tasks:
-  - gsm8k
-  - hellaswag
-  - arc_easy
-
-lm_eval_batch_size: # Batch size for evaluation
-output_dir: # Directory to save evaluation results
+
plugins:
+  - axolotl.integrations.lm_eval.LMEvalPlugin
+
+lm_eval_tasks:
+  - gsm8k
+  - hellaswag
+  - arc_easy
+
+lm_eval_batch_size: # Batch size for evaluation
+output_dir: # Directory to save evaluation results

Citation

-
@misc{eval-harness,
-  author       = {Gao, Leo and Tow, Jonathan and Abbasi, Baber and Biderman, Stella and Black, Sid and DiPofi, Anthony and Foster, Charles and Golding, Laurence and Hsu, Jeffrey and Le Noac'h, Alain and Li, Haonan and McDonell, Kyle and Muennighoff, Niklas and Ociepa, Chris and Phang, Jason and Reynolds, Laria and Schoelkopf, Hailey and Skowron, Aviya and Sutawika, Lintang and Tang, Eric and Thite, Anish and Wang, Ben and Wang, Kevin and Zou, Andy},
-  title        = {A framework for few-shot language model evaluation},
-  month        = 07,
-  year         = 2024,
-  publisher    = {Zenodo},
-  version      = {v0.4.3},
-  doi          = {10.5281/zenodo.12608602},
-  url          = {https://zenodo.org/records/12608602}
-}
+
@misc{eval-harness,
+  author       = {Gao, Leo and Tow, Jonathan and Abbasi, Baber and Biderman, Stella and Black, Sid and DiPofi, Anthony and Foster, Charles and Golding, Laurence and Hsu, Jeffrey and Le Noac'h, Alain and Li, Haonan and McDonell, Kyle and Muennighoff, Niklas and Ociepa, Chris and Phang, Jason and Reynolds, Laria and Schoelkopf, Hailey and Skowron, Aviya and Sutawika, Lintang and Tang, Eric and Thite, Anish and Wang, Ben and Wang, Kevin and Zou, Andy},
+  title        = {A framework for few-shot language model evaluation},
+  month        = 07,
+  year         = 2024,
+  publisher    = {Zenodo},
+  version      = {v0.4.3},
+  doi          = {10.5281/zenodo.12608602},
+  url          = {https://zenodo.org/records/12608602}
+}

Please see reference here

@@ -756,23 +754,23 @@ By identifying the top n% of layers with the highest SNR, you can optimize train

Usage

-
plugins:
-  - axolotl.integrations.spectrum.SpectrumPlugin
-
-spectrum_top_fraction: 0.5
-spectrum_model_name: meta-llama/Meta-Llama-3.1-8B
+
plugins:
+  - axolotl.integrations.spectrum.SpectrumPlugin
+
+spectrum_top_fraction: 0.5
+spectrum_model_name: meta-llama/Meta-Llama-3.1-8B

Citation

-
@misc{hartford2024spectrumtargetedtrainingsignal,
-      title={Spectrum: Targeted Training on Signal to Noise Ratio},
-      author={Eric Hartford and Lucas Atkins and Fernando Fernandes Neto and David Golchinfar},
-      year={2024},
-      eprint={2406.06623},
-      archivePrefix={arXiv},
-      primaryClass={cs.LG},
-      url={https://arxiv.org/abs/2406.06623},
-}
+
@misc{hartford2024spectrumtargetedtrainingsignal,
+      title={Spectrum: Targeted Training on Signal to Noise Ratio},
+      author={Eric Hartford and Lucas Atkins and Fernando Fernandes Neto and David Golchinfar},
+      year={2024},
+      eprint={2406.06623},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG},
+      url={https://arxiv.org/abs/2406.06623},
+}

Please see reference here

@@ -786,7 +784,7 @@ By identifying the top n% of layers with the highest SNR, you can optimize train

Requirements

This will install all necessary dependencies to fine-tune sparsified models using the integration.

@@ -795,25 +793,25 @@ By identifying the top n% of layers with the highest SNR, you can optimize train

Usage

To enable sparse fine-tuning with this integration, include the plugin in your Axolotl config:

-
plugins:
-  - axolotl.integrations.llm_compressor.LLMCompressorPlugin
-
-llmcompressor:
-  recipe:
-    finetuning_stage:
-      finetuning_modifiers:
-        ConstantPruningModifier:
-          targets: [
-            're:.*q_proj.weight',
-            're:.*k_proj.weight',
-            're:.*v_proj.weight',
-            're:.*o_proj.weight',
-            're:.*gate_proj.weight',
-            're:.*up_proj.weight',
-            're:.*down_proj.weight',
-          ]
-          start: 0
-  save_compressed: true
+
plugins:
+  - axolotl.integrations.llm_compressor.LLMCompressorPlugin
+
+llmcompressor:
+  recipe:
+    finetuning_stage:
+      finetuning_modifiers:
+        ConstantPruningModifier:
+          targets: [
+            're:.*q_proj.weight',
+            're:.*k_proj.weight',
+            're:.*v_proj.weight',
+            're:.*o_proj.weight',
+            're:.*gate_proj.weight',
+            're:.*up_proj.weight',
+            're:.*down_proj.weight',
+          ]
+          start: 0
+  save_compressed: true

This plugin does not apply pruning or sparsification itself — it is intended for fine-tuning models that have already been sparsified.

Pre-sparsified checkpoints can be: - Generated using LLMCompressor @@ -840,22 +838,22 @@ By identifying the top n% of layers with the highest SNR, you can optimize train

After fine-tuning your sparse model, you can leverage vLLM for efficient inference. You can also use LLMCompressor to apply additional quantization to your fine-tuned sparse model before inference for even greater performance benefits.:

-
from vllm import LLM, SamplingParams
-
-prompts = [
-    "Hello, my name is",
-    "The president of the United States is",
-    "The capital of France is",
-    "The future of AI is",
-]
-sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
-llm = LLM("path/to/your/sparse/model")
-outputs = llm.generate(prompts, sampling_params)
-
-for output in outputs:
-    prompt = output.prompt
-    generated_text = output.outputs[0].text
-    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
+
from vllm import LLM, SamplingParams
+
+prompts = [
+    "Hello, my name is",
+    "The president of the United States is",
+    "The capital of France is",
+    "The future of AI is",
+]
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
+llm = LLM("path/to/your/sparse/model")
+outputs = llm.generate(prompts, sampling_params)
+
+for output in outputs:
+    prompt = output.prompt
+    generated_text = output.outputs[0].text
+    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")

For more details on vLLM’s capabilities and advanced configuration options, see the official vLLM documentation.

@@ -905,10 +903,10 @@ Warning

If you could not load your integration, please ensure you are pip installing in editable mode.

-
pip install -e .
+
pip install -e .

and correctly spelled the integration name in the config file.

-
plugins:
-  - axolotl.integrations.your_integration_name.YourIntegrationPlugin
+
plugins:
+  - axolotl.integrations.your_integration_name.YourIntegrationPlugin
diff --git a/search.json b/search.json index 36689c9ed..f17e20534 100644 --- a/search.json +++ b/search.json @@ -3106,7 +3106,7 @@ "href": "docs/custom_integrations.html#cut-cross-entropy", "title": "Custom Integrations", "section": "Cut Cross Entropy", - "text": "Cut Cross Entropy\nCut Cross Entropy (CCE) reduces VRAM usage through optimization on the cross-entropy operation during loss calculation.\nSee https://github.com/apple/ml-cross-entropy\n\nRequirements\n\nPyTorch 2.4.0 or higher\n\n\n\nInstallation\nRun the following command to install cut_cross_entropy[transformers] if you don’t have it already.\n\nIf you are in dev environment\n\npython scripts/cutcrossentropy_install.py | sh\n\nIf you are installing from pip\n\npip3 uninstall -y cut-cross-entropy && pip3 install \"cut-cross-entropy[transformers] @ git+https://github.com/apple/ml-cross-entropy.git@bad6f7b49c75fdec69471abb71b4cddd0f0c6438\"\n\n\nUsage\nNOTE: If you are training a VLM model, please use older version of Axolotl as upstream has applied a major VLM refactor, and our patches have not been updated yet.\ngit checkout 787880215b3ab32ccaf81c1b2e9588c6f3e6e764\n\npip3 install --no-build-isolation -e .\nplugins:\n - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin\n\n\nSupported Models\n\nllama\nllama4\nllama4_text\nmllama\nphi3\ngemma\ngemma2\ngemma3\ngemma3_text\nmistral\nmistral3\nqwen2\nqwen2_moe\nqwen2_vl\nqwen2_5_vl\nqwen3\nqwen3_moe\ncohere\ncohere2\nglm\nglm4\n\n\n\nCitation\n@article{wijmans2024cut,\n author = {Erik Wijmans and\n Brody Huval and\n Alexander Hertzberg and\n Vladlen Koltun and\n Philipp Kr\\\"ahenb\\\"uhl},\n title = {Cut Your Losses in Large-Vocabulary Language Models},\n journal = {arXiv},\n year = {2024},\n url = {https://arxiv.org/abs/2411.09009},\n}\nPlease see reference here", + "text": "Cut Cross Entropy\nCut Cross Entropy (CCE) reduces VRAM usage through optimization on the cross-entropy operation during loss calculation.\nSee https://github.com/apple/ml-cross-entropy\n\nRequirements\n\nPyTorch 2.4.0 or higher\n\n\n\nInstallation\nRun the following command to install cut_cross_entropy[transformers] if you don’t have it already.\n\nIf you are in dev environment\n\npython scripts/cutcrossentropy_install.py | sh\n\nIf you are installing from pip\n\npip3 uninstall -y cut-cross-entropy && pip3 install \"cut-cross-entropy[transformers] @ git+https://github.com/axolotl-ai-cloud/ml-cross-entropy.git@78b2a45713a54c9bedf8b33f5e31cf07a1a57154\"\n\n\nUsage\nplugins:\n - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin\n\n\nSupported Models\n\ncohere\ncohere2\ngemma\ngemma2\ngemma3\ngemma3_text\nglm\nglm4\nllama\nllama4\nllama4_text\nmistral\nmistral3\nmllama\nphi\nphi3\nphi4_multimodal\nqwen2\nqwen2_vl\nqwen2_moe\nqwen2_5_vl\nqwen3\nqwen3_moe\n\n\n\nCitation\n@article{wijmans2024cut,\n author = {Erik Wijmans and\n Brody Huval and\n Alexander Hertzberg and\n Vladlen Koltun and\n Philipp Kr\\\"ahenb\\\"uhl},\n title = {Cut Your Losses in Large-Vocabulary Language Models},\n journal = {arXiv},\n year = {2024},\n url = {https://arxiv.org/abs/2411.09009},\n}\nPlease see reference here", "crumbs": [ "Advanced Features", "Custom Integrations" diff --git a/sitemap.xml b/sitemap.xml index 65114b4a0..09c86e90f 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -2,758 +2,758 @@ https://docs.axolotl.ai/docs/unsloth.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.359Z https://docs.axolotl.ai/docs/dataset-formats/conversation.html - 2025-06-25T12:34:07.134Z + 2025-06-25T13:50:23.355Z https://docs.axolotl.ai/docs/dataset-formats/stepwise_supervised.html - 2025-06-25T12:34:07.134Z + 2025-06-25T13:50:23.355Z https://docs.axolotl.ai/docs/dataset-formats/tokenized.html - 2025-06-25T12:34:07.135Z + 2025-06-25T13:50:23.355Z https://docs.axolotl.ai/docs/mac.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.358Z https://docs.axolotl.ai/docs/nccl.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.359Z https://docs.axolotl.ai/docs/multi-node.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.359Z https://docs.axolotl.ai/docs/docker.html - 2025-06-25T12:34:07.135Z + 2025-06-25T13:50:23.356Z https://docs.axolotl.ai/docs/lr_groups.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.358Z https://docs.axolotl.ai/docs/inference.html - 2025-06-25T12:34:07.137Z + 2025-06-25T13:50:23.358Z https://docs.axolotl.ai/docs/cli.html - 2025-06-25T12:34:07.134Z + 2025-06-25T13:50:23.355Z https://docs.axolotl.ai/docs/config-reference.html - 2025-06-25T12:37:22.831Z + 2025-06-25T13:53:53.406Z https://docs.axolotl.ai/docs/multi-gpu.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.358Z https://docs.axolotl.ai/docs/debugging.html - 2025-06-25T12:34:07.135Z + 2025-06-25T13:50:23.355Z https://docs.axolotl.ai/docs/multimodal.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.359Z https://docs.axolotl.ai/docs/api/cli.sweeps.html - 2025-06-25T12:37:08.988Z + 2025-06-25T13:53:40.584Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.llama3.html - 2025-06-25T12:37:09.321Z + 2025-06-25T13:53:40.913Z https://docs.axolotl.ai/docs/api/utils.schedulers.html - 2025-06-25T12:37:09.717Z + 2025-06-25T13:53:41.308Z https://docs.axolotl.ai/docs/api/monkeypatch.llama_attn_hijack_xformers.html - 2025-06-25T12:37:09.521Z + 2025-06-25T13:53:41.113Z https://docs.axolotl.ai/docs/api/cli.cloud.modal_.html - 2025-06-25T12:37:09.038Z + 2025-06-25T13:53:40.633Z https://docs.axolotl.ai/docs/api/kernels.geglu.html - 2025-06-25T12:37:09.474Z + 2025-06-25T13:53:41.067Z https://docs.axolotl.ai/docs/api/core.trainers.utils.html - 2025-06-25T12:37:09.111Z + 2025-06-25T13:53:40.706Z https://docs.axolotl.ai/docs/api/core.datasets.chat.html - 2025-06-25T12:37:08.836Z + 2025-06-25T13:53:40.433Z https://docs.axolotl.ai/docs/api/utils.schemas.peft.html - 2025-06-25T12:37:09.830Z + 2025-06-25T13:53:41.421Z https://docs.axolotl.ai/docs/api/monkeypatch.btlm_attn_hijack_flash.html - 2025-06-25T12:37:09.585Z + 2025-06-25T13:53:41.176Z https://docs.axolotl.ai/docs/api/prompt_strategies.chat_template.html - 2025-06-25T12:37:09.223Z + 2025-06-25T13:53:40.815Z https://docs.axolotl.ai/docs/api/prompt_strategies.kto.user_defined.html - 2025-06-25T12:37:09.353Z + 2025-06-25T13:53:40.946Z https://docs.axolotl.ai/docs/api/cli.cloud.base.html - 2025-06-25T12:37:09.031Z + 2025-06-25T13:53:40.627Z https://docs.axolotl.ai/docs/api/kernels.swiglu.html - 2025-06-25T12:37:09.484Z + 2025-06-25T13:53:41.077Z https://docs.axolotl.ai/docs/api/prompt_strategies.stepwise_supervised.html - 2025-06-25T12:37:09.287Z + 2025-06-25T13:53:40.879Z https://docs.axolotl.ai/docs/api/prompt_strategies.bradley_terry.llama3.html - 2025-06-25T12:37:09.377Z + 2025-06-25T13:53:40.970Z https://docs.axolotl.ai/docs/api/prompt_strategies.completion.html - 2025-06-25T12:37:09.277Z + 2025-06-25T13:53:40.869Z https://docs.axolotl.ai/docs/api/kernels.utils.html - 2025-06-25T12:37:09.493Z + 2025-06-25T13:53:41.086Z https://docs.axolotl.ai/docs/api/common.datasets.html - 2025-06-25T12:37:10.043Z + 2025-06-25T13:53:41.634Z https://docs.axolotl.ai/docs/api/utils.schemas.datasets.html - 2025-06-25T12:37:09.821Z + 2025-06-25T13:53:41.412Z https://docs.axolotl.ai/docs/api/core.builders.rl.html - 2025-06-25T12:37:08.791Z + 2025-06-25T13:53:40.388Z https://docs.axolotl.ai/docs/api/evaluate.html - 2025-06-25T12:37:08.696Z + 2025-06-25T13:53:40.292Z https://docs.axolotl.ai/docs/api/kernels.quantize.html - 2025-06-25T12:37:09.492Z + 2025-06-25T13:53:41.084Z https://docs.axolotl.ai/docs/api/monkeypatch.llama_attn_hijack_flash.html - 2025-06-25T12:37:09.520Z + 2025-06-25T13:53:41.112Z https://docs.axolotl.ai/docs/api/core.trainers.mixins.rng_state_loader.html - 2025-06-25T12:37:09.156Z + 2025-06-25T13:53:40.749Z https://docs.axolotl.ai/docs/api/integrations.base.html - 2025-06-25T12:37:10.003Z + 2025-06-25T13:53:41.594Z https://docs.axolotl.ai/docs/api/cli.merge_lora.html - 2025-06-25T12:37:08.961Z + 2025-06-25T13:53:40.558Z https://docs.axolotl.ai/docs/api/cli.merge_sharded_fsdp_weights.html - 2025-06-25T12:37:08.973Z + 2025-06-25T13:53:40.570Z https://docs.axolotl.ai/docs/api/monkeypatch.transformers_fa_utils.html - 2025-06-25T12:37:09.602Z + 2025-06-25T13:53:41.194Z https://docs.axolotl.ai/docs/api/prompt_strategies.llama2_chat.html - 2025-06-25T12:37:09.271Z + 2025-06-25T13:53:40.863Z https://docs.axolotl.ai/docs/api/utils.collators.mm_chat.html - 2025-06-25T12:37:10.072Z + 2025-06-25T13:53:41.664Z https://docs.axolotl.ai/docs/api/utils.data.sft.html - 2025-06-25T12:37:09.756Z + 2025-06-25T13:53:41.348Z https://docs.axolotl.ai/docs/api/prompt_strategies.alpaca_instruct.html - 2025-06-25T12:37:09.238Z + 2025-06-25T13:53:40.830Z https://docs.axolotl.ai/docs/api/integrations.liger.args.html - 2025-06-25T12:37:10.018Z + 2025-06-25T13:53:41.610Z https://docs.axolotl.ai/docs/api/monkeypatch.mistral_attn_hijack_flash.html - 2025-06-25T12:37:09.536Z + 2025-06-25T13:53:41.128Z https://docs.axolotl.ai/docs/api/cli.vllm_serve.html - 2025-06-25T12:37:09.028Z + 2025-06-25T13:53:40.624Z https://docs.axolotl.ai/docs/api/monkeypatch.utils.html - 2025-06-25T12:37:09.583Z + 2025-06-25T13:53:41.175Z https://docs.axolotl.ai/docs/api/loaders.patch_manager.html - 2025-06-25T12:37:09.146Z + 2025-06-25T13:53:40.739Z https://docs.axolotl.ai/docs/api/utils.schemas.integrations.html - 2025-06-25T12:37:09.851Z + 2025-06-25T13:53:41.442Z https://docs.axolotl.ai/docs/api/utils.callbacks.perplexity.html - 2025-06-25T12:37:10.119Z + 2025-06-25T13:53:41.711Z https://docs.axolotl.ai/docs/api/cli.utils.html - 2025-06-25T12:37:09.021Z + 2025-06-25T13:53:40.617Z https://docs.axolotl.ai/docs/api/utils.schemas.config.html - 2025-06-25T12:37:09.790Z + 2025-06-25T13:53:41.382Z https://docs.axolotl.ai/docs/api/prompt_strategies.input_output.html - 2025-06-25T12:37:09.283Z + 2025-06-25T13:53:40.875Z https://docs.axolotl.ai/docs/api/utils.distributed.html - 2025-06-25T12:37:09.737Z + 2025-06-25T13:53:41.328Z https://docs.axolotl.ai/docs/api/monkeypatch.gradient_checkpointing.offload_disk.html - 2025-06-25T12:37:09.636Z + 2025-06-25T13:53:41.228Z https://docs.axolotl.ai/docs/api/monkeypatch.trainer_fsdp_optim.html - 2025-06-25T12:37:09.595Z + 2025-06-25T13:53:41.187Z https://docs.axolotl.ai/docs/api/core.builders.base.html - 2025-06-25T12:37:08.778Z + 2025-06-25T13:53:40.375Z https://docs.axolotl.ai/docs/api/core.trainers.trl.html - 2025-06-25T12:37:09.069Z + 2025-06-25T13:53:40.665Z https://docs.axolotl.ai/docs/api/cli.evaluate.html - 2025-06-25T12:37:08.893Z + 2025-06-25T13:53:40.490Z https://docs.axolotl.ai/docs/api/utils.optimizers.adopt.html - 2025-06-25T12:37:09.748Z + 2025-06-25T13:53:41.339Z https://docs.axolotl.ai/docs/api/utils.callbacks.qat.html - 2025-06-25T12:37:10.138Z + 2025-06-25T13:53:41.730Z https://docs.axolotl.ai/docs/api/core.trainers.dpo.trainer.html - 2025-06-25T12:37:09.086Z + 2025-06-25T13:53:40.681Z https://docs.axolotl.ai/docs/api/core.chat.format.shared.html - 2025-06-25T12:37:08.831Z + 2025-06-25T13:53:40.428Z https://docs.axolotl.ai/docs/api/monkeypatch.relora.html - 2025-06-25T12:37:09.545Z + 2025-06-25T13:53:41.137Z https://docs.axolotl.ai/docs/api/cli.config.html - 2025-06-25T12:37:08.938Z + 2025-06-25T13:53:40.535Z https://docs.axolotl.ai/docs/api/cli.preprocess.html - 2025-06-25T12:37:08.982Z + 2025-06-25T13:53:40.578Z https://docs.axolotl.ai/docs/api/core.trainers.base.html - 2025-06-25T12:37:09.053Z + 2025-06-25T13:53:40.648Z https://docs.axolotl.ai/docs/api/convert.html - 2025-06-25T12:37:08.720Z + 2025-06-25T13:53:40.317Z https://docs.axolotl.ai/docs/api/prompt_strategies.pygmalion.html - 2025-06-25T12:37:09.304Z + 2025-06-25T13:53:40.897Z https://docs.axolotl.ai/docs/api/utils.schemas.trl.html - 2025-06-25T12:37:09.833Z + 2025-06-25T13:53:41.425Z https://docs.axolotl.ai/docs/api/cli.args.html - 2025-06-25T12:37:08.913Z + 2025-06-25T13:53:40.510Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.chat_template.html - 2025-06-25T12:37:09.310Z + 2025-06-25T13:53:40.903Z https://docs.axolotl.ai/docs/api/loaders.constants.html - 2025-06-25T12:37:09.147Z + 2025-06-25T13:53:40.740Z https://docs.axolotl.ai/docs/api/logging_config.html - 2025-06-25T12:37:08.772Z + 2025-06-25T13:53:40.369Z https://docs.axolotl.ai/docs/api/cli.inference.html - 2025-06-25T12:37:08.953Z + 2025-06-25T13:53:40.549Z https://docs.axolotl.ai/docs/api/utils.ctx_managers.sequence_parallel.html - 2025-06-25T12:37:09.187Z + 2025-06-25T13:53:40.780Z https://docs.axolotl.ai/docs/api/integrations.spectrum.args.html - 2025-06-25T12:37:10.025Z + 2025-06-25T13:53:41.616Z https://docs.axolotl.ai/docs/api/utils.schemas.training.html - 2025-06-25T12:37:09.803Z + 2025-06-25T13:53:41.394Z https://docs.axolotl.ai/docs/api/prompt_strategies.orcamini.html - 2025-06-25T12:37:09.298Z + 2025-06-25T13:53:40.890Z https://docs.axolotl.ai/docs/api/utils.freeze.html - 2025-06-25T12:37:09.674Z + 2025-06-25T13:53:41.266Z https://docs.axolotl.ai/docs/api/loaders.tokenizer.html - 2025-06-25T12:37:09.130Z + 2025-06-25T13:53:40.724Z https://docs.axolotl.ai/docs/api/utils.bench.html - 2025-06-25T12:37:09.667Z + 2025-06-25T13:53:41.259Z https://docs.axolotl.ai/docs/api/utils.quantization.html - 2025-06-25T12:37:09.777Z + 2025-06-25T13:53:41.368Z https://docs.axolotl.ai/docs/batch_vs_grad.html - 2025-06-25T12:34:07.134Z + 2025-06-25T13:50:23.355Z https://docs.axolotl.ai/docs/input_output.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.358Z https://docs.axolotl.ai/docs/sequence_parallelism.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.359Z https://docs.axolotl.ai/docs/reward_modelling.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.359Z https://docs.axolotl.ai/index.html - 2025-06-25T12:34:07.151Z + 2025-06-25T13:50:23.372Z https://docs.axolotl.ai/src/axolotl/integrations/LICENSE.html - 2025-06-25T12:34:07.155Z + 2025-06-25T13:50:23.376Z https://docs.axolotl.ai/FAQS.html - 2025-06-25T12:34:07.132Z + 2025-06-25T13:50:23.353Z https://docs.axolotl.ai/src/axolotl/integrations/cut_cross_entropy/ACKNOWLEDGEMENTS.html - 2025-06-25T12:34:07.156Z + 2025-06-25T13:50:23.376Z https://docs.axolotl.ai/TODO.html - 2025-06-25T12:34:07.133Z + 2025-06-25T13:50:23.354Z https://docs.axolotl.ai/examples/colab-notebooks/colab-axolotl-example.html - 2025-06-25T12:34:07.139Z + 2025-06-25T13:50:23.360Z https://docs.axolotl.ai/docs/torchao.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.359Z https://docs.axolotl.ai/docs/ray-integration.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.359Z https://docs.axolotl.ai/docs/quantize.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.359Z https://docs.axolotl.ai/docs/qat.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.359Z https://docs.axolotl.ai/docs/api/utils.lora.html - 2025-06-25T12:37:09.658Z + 2025-06-25T13:53:41.250Z https://docs.axolotl.ai/docs/api/prompt_strategies.alpaca_w_system.html - 2025-06-25T12:37:09.250Z + 2025-06-25T13:53:40.842Z https://docs.axolotl.ai/docs/api/monkeypatch.stablelm_attn_hijack_flash.html - 2025-06-25T12:37:09.592Z + 2025-06-25T13:53:41.184Z https://docs.axolotl.ai/docs/api/utils.collators.core.html - 2025-06-25T12:37:10.045Z + 2025-06-25T13:53:41.637Z https://docs.axolotl.ai/docs/api/prompt_strategies.metharme.html - 2025-06-25T12:37:09.294Z + 2025-06-25T13:53:40.886Z https://docs.axolotl.ai/docs/api/utils.callbacks.profiler.html - 2025-06-25T12:37:10.123Z + 2025-06-25T13:53:41.715Z https://docs.axolotl.ai/docs/api/utils.data.pretraining.html - 2025-06-25T12:37:09.749Z + 2025-06-25T13:53:41.341Z https://docs.axolotl.ai/docs/api/utils.callbacks.lisa.html - 2025-06-25T12:37:10.124Z + 2025-06-25T13:53:41.716Z https://docs.axolotl.ai/docs/api/utils.trainer.html - 2025-06-25T12:37:09.692Z + 2025-06-25T13:53:41.284Z https://docs.axolotl.ai/docs/api/integrations.cut_cross_entropy.args.html - 2025-06-25T12:37:10.007Z + 2025-06-25T13:53:41.598Z https://docs.axolotl.ai/docs/api/utils.schemas.model.html - 2025-06-25T12:37:09.798Z + 2025-06-25T13:53:41.389Z https://docs.axolotl.ai/docs/api/monkeypatch.data.batch_dataset_fetcher.html - 2025-06-25T12:37:09.605Z + 2025-06-25T13:53:41.197Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.zephyr.html - 2025-06-25T12:37:09.332Z + 2025-06-25T13:53:40.925Z https://docs.axolotl.ai/docs/api/datasets.html - 2025-06-25T12:37:08.707Z + 2025-06-25T13:53:40.303Z https://docs.axolotl.ai/docs/api/utils.schemas.enums.html - 2025-06-25T12:37:09.861Z + 2025-06-25T13:53:41.453Z https://docs.axolotl.ai/docs/api/integrations.kd.trainer.html - 2025-06-25T12:37:10.015Z + 2025-06-25T13:53:41.606Z https://docs.axolotl.ai/docs/api/monkeypatch.lora_kernels.html - 2025-06-25T12:37:09.575Z + 2025-06-25T13:53:41.167Z https://docs.axolotl.ai/docs/api/utils.collators.batching.html - 2025-06-25T12:37:10.064Z + 2025-06-25T13:53:41.656Z https://docs.axolotl.ai/docs/api/core.trainers.grpo.sampler.html - 2025-06-25T12:37:09.110Z + 2025-06-25T13:53:40.704Z https://docs.axolotl.ai/docs/api/prompt_strategies.base.html - 2025-06-25T12:37:09.189Z + 2025-06-25T13:53:40.781Z https://docs.axolotl.ai/docs/api/monkeypatch.multipack.html - 2025-06-25T12:37:09.538Z + 2025-06-25T13:53:41.130Z https://docs.axolotl.ai/docs/api/prompt_strategies.orpo.chat_template.html - 2025-06-25T12:37:09.374Z + 2025-06-25T13:53:40.966Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.passthrough.html - 2025-06-25T12:37:09.335Z + 2025-06-25T13:53:40.928Z https://docs.axolotl.ai/docs/api/core.chat.format.chatml.html - 2025-06-25T12:37:08.828Z + 2025-06-25T13:53:40.425Z https://docs.axolotl.ai/docs/api/core.trainers.mixins.scheduler.html - 2025-06-25T12:37:09.163Z + 2025-06-25T13:53:40.756Z https://docs.axolotl.ai/docs/api/utils.model_shard_quant.html - 2025-06-25T12:37:09.663Z + 2025-06-25T13:53:41.255Z https://docs.axolotl.ai/docs/api/prompt_strategies.kto.chatml.html - 2025-06-25T12:37:09.352Z + 2025-06-25T13:53:40.944Z https://docs.axolotl.ai/docs/api/utils.tokenization.html - 2025-06-25T12:37:09.642Z + 2025-06-25T13:53:41.235Z https://docs.axolotl.ai/docs/api/loaders.model.html - 2025-06-25T12:37:09.121Z + 2025-06-25T13:53:40.716Z https://docs.axolotl.ai/docs/api/utils.callbacks.mlflow_.html - 2025-06-25T12:37:10.128Z + 2025-06-25T13:53:41.720Z https://docs.axolotl.ai/docs/api/core.trainers.grpo.trainer.html - 2025-06-25T12:37:09.097Z + 2025-06-25T13:53:40.692Z https://docs.axolotl.ai/docs/api/cli.main.html - 2025-06-25T12:37:08.876Z + 2025-06-25T13:53:40.473Z https://docs.axolotl.ai/docs/api/utils.callbacks.comet_.html - 2025-06-25T12:37:10.131Z + 2025-06-25T13:53:41.723Z https://docs.axolotl.ai/docs/api/utils.chat_templates.html - 2025-06-25T12:37:09.652Z + 2025-06-25T13:53:41.245Z https://docs.axolotl.ai/docs/api/utils.schemas.utils.html - 2025-06-25T12:37:09.867Z + 2025-06-25T13:53:41.459Z https://docs.axolotl.ai/docs/api/common.architectures.html - 2025-06-25T12:37:10.026Z + 2025-06-25T13:53:41.618Z https://docs.axolotl.ai/docs/api/monkeypatch.llama_expand_mask.html - 2025-06-25T12:37:09.546Z + 2025-06-25T13:53:41.138Z https://docs.axolotl.ai/docs/api/prompt_strategies.alpaca_chat.html - 2025-06-25T12:37:09.236Z + 2025-06-25T13:53:40.829Z https://docs.axolotl.ai/docs/api/utils.samplers.multipack.html - 2025-06-25T12:37:10.113Z + 2025-06-25T13:53:41.705Z https://docs.axolotl.ai/docs/api/integrations.grokfast.optimizer.html - 2025-06-25T12:37:10.008Z + 2025-06-25T13:53:41.599Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.chatml.html - 2025-06-25T12:37:09.331Z + 2025-06-25T13:53:40.923Z https://docs.axolotl.ai/docs/api/monkeypatch.mixtral.html - 2025-06-25T12:37:09.606Z + 2025-06-25T13:53:41.198Z https://docs.axolotl.ai/docs/api/train.html - 2025-06-25T12:37:08.685Z + 2025-06-25T13:53:40.282Z https://docs.axolotl.ai/docs/api/monkeypatch.llama_patch_multipack.html - 2025-06-25T12:37:09.586Z + 2025-06-25T13:53:41.178Z https://docs.axolotl.ai/docs/api/index.html - 2025-06-25T12:37:08.622Z + 2025-06-25T13:53:40.219Z https://docs.axolotl.ai/docs/api/loaders.adapter.html - 2025-06-25T12:37:09.137Z + 2025-06-25T13:53:40.731Z https://docs.axolotl.ai/docs/api/utils.schemas.multimodal.html - 2025-06-25T12:37:09.838Z + 2025-06-25T13:53:41.430Z https://docs.axolotl.ai/docs/api/kernels.lora.html - 2025-06-25T12:37:09.463Z + 2025-06-25T13:53:41.056Z https://docs.axolotl.ai/docs/api/prompt_strategies.kto.llama3.html - 2025-06-25T12:37:09.343Z + 2025-06-25T13:53:40.936Z https://docs.axolotl.ai/docs/api/cli.checks.html - 2025-06-25T12:37:08.920Z + 2025-06-25T13:53:40.517Z https://docs.axolotl.ai/docs/api/cli.quantize.html - 2025-06-25T12:37:09.043Z + 2025-06-25T13:53:40.638Z https://docs.axolotl.ai/docs/api/integrations.lm_eval.args.html - 2025-06-25T12:37:10.021Z + 2025-06-25T13:53:41.613Z https://docs.axolotl.ai/docs/api/core.chat.messages.html - 2025-06-25T12:37:08.826Z + 2025-06-25T13:53:40.423Z https://docs.axolotl.ai/docs/api/core.builders.causal.html - 2025-06-25T12:37:08.783Z + 2025-06-25T13:53:40.380Z https://docs.axolotl.ai/docs/api/core.trainers.relora.html - 2025-06-25T12:37:09.079Z + 2025-06-25T13:53:40.674Z https://docs.axolotl.ai/docs/api/models.mamba.modeling_mamba.html - 2025-06-25T12:37:10.044Z + 2025-06-25T13:53:41.635Z https://docs.axolotl.ai/docs/api/monkeypatch.gradient_checkpointing.offload_cpu.html - 2025-06-25T12:37:09.610Z + 2025-06-25T13:53:41.202Z https://docs.axolotl.ai/docs/api/core.trainers.mamba.html - 2025-06-25T12:37:09.074Z + 2025-06-25T13:53:40.670Z https://docs.axolotl.ai/docs/api/core.datasets.transforms.chat_builder.html - 2025-06-25T12:37:08.844Z + 2025-06-25T13:53:40.441Z https://docs.axolotl.ai/docs/api/loaders.processor.html - 2025-06-25T12:37:09.131Z + 2025-06-25T13:53:40.725Z https://docs.axolotl.ai/docs/api/core.chat.format.llama3x.html - 2025-06-25T12:37:08.829Z + 2025-06-25T13:53:40.426Z https://docs.axolotl.ai/docs/api/prompt_strategies.messages.chat.html - 2025-06-25T12:37:09.309Z + 2025-06-25T13:53:40.901Z https://docs.axolotl.ai/docs/api/cli.train.html - 2025-06-25T12:37:08.885Z + 2025-06-25T13:53:40.482Z https://docs.axolotl.ai/docs/api/core.trainers.mixins.optimizer.html - 2025-06-25T12:37:09.153Z + 2025-06-25T13:53:40.746Z https://docs.axolotl.ai/docs/api/utils.collators.mamba.html - 2025-06-25T12:37:10.067Z + 2025-06-25T13:53:41.659Z https://docs.axolotl.ai/docs/api/monkeypatch.unsloth_.html - 2025-06-25T12:37:09.603Z + 2025-06-25T13:53:41.195Z https://docs.axolotl.ai/docs/api/utils.dict.html - 2025-06-25T12:37:09.740Z + 2025-06-25T13:53:41.332Z https://docs.axolotl.ai/docs/api/prompt_strategies.user_defined.html - 2025-06-25T12:37:09.258Z + 2025-06-25T13:53:40.850Z https://docs.axolotl.ai/docs/api/core.training_args.html - 2025-06-25T12:37:08.803Z + 2025-06-25T13:53:40.400Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.user_defined.html - 2025-06-25T12:37:09.334Z + 2025-06-25T13:53:40.926Z https://docs.axolotl.ai/docs/api/prompt_tokenizers.html - 2025-06-25T12:37:08.762Z + 2025-06-25T13:53:40.359Z https://docs.axolotl.ai/docs/api/common.const.html - 2025-06-25T12:37:10.028Z + 2025-06-25T13:53:41.619Z https://docs.axolotl.ai/docs/fsdp_qlora.html - 2025-06-25T12:34:07.135Z + 2025-06-25T13:50:23.356Z https://docs.axolotl.ai/docs/custom_integrations.html - 2025-06-25T12:34:07.134Z + 2025-06-25T13:50:23.355Z https://docs.axolotl.ai/docs/getting-started.html - 2025-06-25T12:34:07.135Z + 2025-06-25T13:50:23.356Z https://docs.axolotl.ai/docs/faq.html - 2025-06-25T12:34:07.135Z + 2025-06-25T13:50:23.356Z https://docs.axolotl.ai/docs/lora_optims.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.358Z https://docs.axolotl.ai/docs/rlhf.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.359Z https://docs.axolotl.ai/docs/amd_hpc.html - 2025-06-25T12:34:07.134Z + 2025-06-25T13:50:23.355Z https://docs.axolotl.ai/docs/installation.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.358Z https://docs.axolotl.ai/docs/multipack.html - 2025-06-25T12:34:07.138Z + 2025-06-25T13:50:23.359Z https://docs.axolotl.ai/docs/dataset_preprocessing.html - 2025-06-25T12:34:07.135Z + 2025-06-25T13:50:23.355Z https://docs.axolotl.ai/docs/dataset_loading.html - 2025-06-25T12:34:07.135Z + 2025-06-25T13:50:23.355Z https://docs.axolotl.ai/docs/dataset-formats/inst_tune.html - 2025-06-25T12:34:07.134Z + 2025-06-25T13:50:23.355Z https://docs.axolotl.ai/docs/dataset-formats/template_free.html - 2025-06-25T12:34:07.135Z + 2025-06-25T13:50:23.355Z https://docs.axolotl.ai/docs/dataset-formats/index.html - 2025-06-25T12:34:07.134Z + 2025-06-25T13:50:23.355Z https://docs.axolotl.ai/docs/dataset-formats/pretraining.html - 2025-06-25T12:34:07.134Z + 2025-06-25T13:50:23.355Z