From d76f8c505c17dff4f360d81e028d75e7089fa651 Mon Sep 17 00:00:00 2001
From: Quarto GHA Workflow Runner <quarto-github-actions-publish@example.com>
Date: Fri, 10 Apr 2026 20:53:46 +0000
Subject: [PATCH] Built site for gh-pages

---
 .nojekyll                            |   2 +-
 docs/agents/model_architectures.html | 124 +++++--
 docs/agents/sft.html                 |  52 +++
 docs/multimodal.html                 | 155 ++++++---
 index.html                           | 100 ++++--
 search.json                          |  37 +-
 sitemap.xml                          | 494 +++++++++++++--------------
 7 files changed, 594 insertions(+), 370 deletions(-)

diff --git a/.nojekyll b/.nojekyll
index c47c88483..a3c993635 100644
--- a/.nojekyll
+++ b/.nojekyll
@@ -1 +1 @@
-a6bc28b6
\ No newline at end of file
+66b84b9d
\ No newline at end of file
diff --git a/docs/agents/model_architectures.html b/docs/agents/model_architectures.html
index 731a06c60..b0373a116 100644
--- a/docs/agents/model_architectures.html
+++ b/docs/agents/model_architectures.html
@@ -788,12 +788,19 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});
   <ul>
   <li><a href="#model-architectures-agent-reference" id="toc-model-architectures-agent-reference" class="nav-link active" data-scroll-target="#model-architectures-agent-reference">Model Architectures — Agent Reference</a>
   <ul class="collapse">
+  <li><a href="#vlm-vision-language-model-quick-start" id="toc-vlm-vision-language-model-quick-start" class="nav-link" data-scroll-target="#vlm-vision-language-model-quick-start">VLM (Vision Language Model) Quick Start</a></li>
+  <li><a href="#plugins-optimizations" id="toc-plugins-optimizations" class="nav-link" data-scroll-target="#plugins-optimizations">Plugins &amp; Optimizations</a>
+  <ul class="collapse">
+  <li><a href="#cut-cross-entropy-cce" id="toc-cut-cross-entropy-cce" class="nav-link" data-scroll-target="#cut-cross-entropy-cce">Cut Cross Entropy (CCE)</a></li>
+  <li><a href="#scattermoe-kernels" id="toc-scattermoe-kernels" class="nav-link" data-scroll-target="#scattermoe-kernels">ScatterMoE Kernels</a></li>
+  </ul></li>
   <li><a href="#gemma-4" id="toc-gemma-4" class="nav-link" data-scroll-target="#gemma-4">Gemma 4</a>
   <ul class="collapse">
   <li><a href="#required-settings" id="toc-required-settings" class="nav-link" data-scroll-target="#required-settings">Required settings</a></li>
   <li><a href="#auto-detection" id="toc-auto-detection" class="nav-link" data-scroll-target="#auto-detection">Auto-detection</a></li>
   <li><a href="#multi-gpu" id="toc-multi-gpu" class="nav-link" data-scroll-target="#multi-gpu">Multi-GPU</a></li>
   <li><a href="#moe-26b-a4b" id="toc-moe-26b-a4b" class="nav-link" data-scroll-target="#moe-26b-a4b">MoE (26B-A4B)</a></li>
+  <li><a href="#vlm-vision-training" id="toc-vlm-vision-training" class="nav-link" data-scroll-target="#vlm-vision-training">VLM (Vision) Training</a></li>
   <li><a href="#common-issues" id="toc-common-issues" class="nav-link" data-scroll-target="#common-issues">Common issues</a></li>
   <li><a href="#e2be4b-dense-models" id="toc-e2be4b-dense-models" class="nav-link" data-scroll-target="#e2be4b-dense-models">E2B/E4B dense models</a></li>
   </ul></li>
@@ -813,19 +820,63 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});
 <section id="model-architectures-agent-reference" class="level1">
 <h1>Model Architectures — Agent Reference</h1>
 <p>Model-specific quirks, required settings, and known issues. Check this before debugging training failures on specific model families.</p>
+<section id="vlm-vision-language-model-quick-start" class="level2">
+<h2 class="anchored" data-anchor-id="vlm-vision-language-model-quick-start">VLM (Vision Language Model) Quick Start</h2>
+<p>All VLM configs require these four lines:</p>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb1"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="fu">processor_type</span><span class="kw">:</span><span class="at"> AutoProcessor</span></span>
+<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="fu">skip_prepare_dataset</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
+<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="fu">remove_unused_columns</span><span class="kw">:</span><span class="at"> </span><span class="ch">false</span></span>
+<span id="cb1-4"><a href="#cb1-4" aria-hidden="true" tabindex="-1"></a><span class="fu">sample_packing</span><span class="kw">:</span><span class="at"> </span><span class="ch">false</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<p>Decision tree for VLM config:</p>
+<pre class="text"><code>Is the model multimodal (has vision/audio encoder)?
+  ├─ YES: Add `freeze_mm_modules: true` if training text only
+  │       Add `chat_template: &lt;model_template&gt;` (e.g. gemma4, qwen3_5, gemma3)
+  │       LoRA: use regex `lora_target_modules` to restrict to language model
+  └─ NO: Train as a regular text model
+
+Is the model MoE (e.g. Gemma4 26B-A4B, Qwen3.5 35B-A3B)?
+  ├─ YES: Add `lora_target_parameters` for expert LoRA
+  │       Consider ScatterMoE kernels (see Plugins section)
+  └─ NO: Standard LoRA config</code></pre>
+</section>
+<section id="plugins-optimizations" class="level2">
+<h2 class="anchored" data-anchor-id="plugins-optimizations">Plugins &amp; Optimizations</h2>
+<section id="cut-cross-entropy-cce" class="level3">
+<h3 class="anchored" data-anchor-id="cut-cross-entropy-cce">Cut Cross Entropy (CCE)</h3>
+<p>Computes loss from hidden states + lm_head weight without materializing the full logits tensor, saving significant VRAM. Install if not already present:</p>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb3"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="ex">uv</span> pip install <span class="st">"cut-cross-entropy[transformers] @ git+https://github.com/axolotl-ai-cloud/ml-cross-entropy.git@main"</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb4"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="fu">plugins</span><span class="kw">:</span></span>
+<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+</section>
+<section id="scattermoe-kernels" class="level3">
+<h3 class="anchored" data-anchor-id="scattermoe-kernels">ScatterMoE Kernels</h3>
+<p>Fuses expert + LoRA computation into a single kernel for MoE models. Significant speedup for models with many experts.</p>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb5"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="fu">plugins</span><span class="kw">:</span></span>
+<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> axolotl.integrations.kernels.KernelsPlugin</span></span>
+<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a><span class="fu">use_kernels</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
+<span id="cb5-4"><a href="#cb5-4" aria-hidden="true" tabindex="-1"></a><span class="fu">use_scattermoe</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
+<span id="cb5-5"><a href="#cb5-5" aria-hidden="true" tabindex="-1"></a><span class="fu">experts_implementation</span><span class="kw">:</span><span class="at"> scattermoe</span></span>
+<span id="cb5-6"><a href="#cb5-6" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb5-7"><a href="#cb5-7" aria-hidden="true" tabindex="-1"></a><span class="co"># Expert LoRA targets (3D parameter tensors, not nn.Linear):</span></span>
+<span id="cb5-8"><a href="#cb5-8" aria-hidden="true" tabindex="-1"></a><span class="fu">lora_target_parameters</span><span class="kw">:</span></span>
+<span id="cb5-9"><a href="#cb5-9" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> experts.gate_up_proj</span></span>
+<span id="cb5-10"><a href="#cb5-10" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> experts.down_proj</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<p>Supported: Gemma4 (<code>gemma4_text</code>), Mixtral, Qwen MoE variants. The plugin auto-detects model type and routing function. Without ScatterMoE, expert LoRA still works but runs base expert matmul and LoRA as separate operations.</p>
+</section>
+</section>
 <section id="gemma-4" class="level2">
 <h2 class="anchored" data-anchor-id="gemma-4">Gemma 4</h2>
 <p><strong>Models</strong>: <code>google/gemma-4-26B-A4B</code> (MoE), <code>google/gemma-4-31B</code> (dense), <code>google/gemma-4-E2B</code>, <code>google/gemma-4-E4B</code></p>
 <p><strong>Architecture</strong>: Multimodal wrapper (<code>Gemma4ForConditionalGeneration</code>) over a text backbone (<code>Gemma4TextModel</code>), with optional vision/audio encoders. All Gemma4 HF repos have <code>model_type: "gemma4"</code> — even text-only variants load as multimodal with a vision tower.</p>
 <section id="required-settings" class="level3">
 <h3 class="anchored" data-anchor-id="required-settings">Required settings</h3>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb1"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Always needed for Gemma4:</span></span>
-<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="fu">freeze_mm_modules</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span><span class="co">          # Freeze vision/audio encoders for text-only training</span></span>
-<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="fu">gradient_checkpointing_kwargs</span><span class="kw">:</span></span>
-<span id="cb1-4"><a href="#cb1-4" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="fu">use_reentrant</span><span class="kw">:</span><span class="at"> </span><span class="ch">false</span><span class="co">           # Shared per-layer norms cause "marked ready twice" with reentrant</span></span>
-<span id="cb1-5"><a href="#cb1-5" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb1-6"><a href="#cb1-6" aria-hidden="true" tabindex="-1"></a><span class="co"># LoRA target — restrict to language model only (DO NOT use lora_target_linear: true):</span></span>
-<span id="cb1-7"><a href="#cb1-7" aria-hidden="true" tabindex="-1"></a><span class="fu">lora_target_modules</span><span class="kw">:</span><span class="at"> </span><span class="st">'model.language_model.layers.[\d]+.(_checkpoint_wrapped_module.)?(mlp|self_attn).(up|down|gate|q|k|v|o)_proj'</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb6"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Always needed for Gemma4:</span></span>
+<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a><span class="fu">freeze_mm_modules</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span><span class="co">          # Freeze vision/audio encoders for text-only training</span></span>
+<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a><span class="fu">gradient_checkpointing_kwargs</span><span class="kw">:</span></span>
+<span id="cb6-4"><a href="#cb6-4" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="fu">use_reentrant</span><span class="kw">:</span><span class="at"> </span><span class="ch">false</span><span class="co">           # Shared per-layer norms cause "marked ready twice" with reentrant</span></span>
+<span id="cb6-5"><a href="#cb6-5" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb6-6"><a href="#cb6-6" aria-hidden="true" tabindex="-1"></a><span class="co"># LoRA target — restrict to language model only (DO NOT use lora_target_linear: true):</span></span>
+<span id="cb6-7"><a href="#cb6-7" aria-hidden="true" tabindex="-1"></a><span class="fu">lora_target_modules</span><span class="kw">:</span><span class="at"> </span><span class="st">'model.language_model.layers.[\d]+.(_checkpoint_wrapped_module.)?(mlp|self_attn).(up|down|gate|q|k|v|o)_proj'</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 </section>
 <section id="auto-detection" class="level3">
 <h3 class="anchored" data-anchor-id="auto-detection">Auto-detection</h3>
@@ -877,13 +928,13 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});
 </tbody>
 </table>
 <p>FSDP2 config:</p>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb2"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="fu">fsdp</span><span class="kw">:</span></span>
-<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> full_shard</span></span>
-<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> auto_wrap</span></span>
-<span id="cb2-4"><a href="#cb2-4" aria-hidden="true" tabindex="-1"></a><span class="fu">fsdp_config</span><span class="kw">:</span></span>
-<span id="cb2-5"><a href="#cb2-5" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="fu">fsdp_version</span><span class="kw">:</span><span class="at"> </span><span class="dv">2</span></span>
-<span id="cb2-6"><a href="#cb2-6" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="fu">fsdp_auto_wrap_policy</span><span class="kw">:</span><span class="at"> TRANSFORMER_BASED_WRAP</span></span>
-<span id="cb2-7"><a href="#cb2-7" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="fu">fsdp_transformer_layer_cls_to_wrap</span><span class="kw">:</span><span class="at"> Gemma4TextDecoderLayer</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb7"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="fu">fsdp</span><span class="kw">:</span></span>
+<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> full_shard</span></span>
+<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> auto_wrap</span></span>
+<span id="cb7-4"><a href="#cb7-4" aria-hidden="true" tabindex="-1"></a><span class="fu">fsdp_config</span><span class="kw">:</span></span>
+<span id="cb7-5"><a href="#cb7-5" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="fu">fsdp_version</span><span class="kw">:</span><span class="at"> </span><span class="dv">2</span></span>
+<span id="cb7-6"><a href="#cb7-6" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="fu">fsdp_auto_wrap_policy</span><span class="kw">:</span><span class="at"> TRANSFORMER_BASED_WRAP</span></span>
+<span id="cb7-7"><a href="#cb7-7" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="fu">fsdp_transformer_layer_cls_to_wrap</span><span class="kw">:</span><span class="at"> Gemma4TextDecoderLayer</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 </section>
 <section id="moe-26b-a4b" class="level3">
 <h3 class="anchored" data-anchor-id="moe-26b-a4b">MoE (26B-A4B)</h3>
@@ -891,17 +942,40 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});
 <li><p><code>enable_moe_block: true</code>, 256 experts, top-k routing</p></li>
 <li><p>No separate <code>SparseMoeBlock</code> — MoE is embedded in each decoder layer</p></li>
 <li><p>Expert LoRA targets 3D parameter tensors:</p>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb3"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="fu">lora_target_parameters</span><span class="kw">:</span></span>
-<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> experts.gate_up_proj</span></span>
-<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> experts.down_proj</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div></li>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb8"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="fu">lora_target_parameters</span><span class="kw">:</span></span>
+<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> experts.gate_up_proj</span></span>
+<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> experts.down_proj</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div></li>
 <li><p>ScatterMoE kernel acceleration:</p>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb4"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="fu">plugins</span><span class="kw">:</span></span>
-<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> axolotl.integrations.kernels.KernelsPlugin</span></span>
-<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a><span class="fu">use_kernels</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
-<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a><span class="fu">use_scattermoe</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
-<span id="cb4-5"><a href="#cb4-5" aria-hidden="true" tabindex="-1"></a><span class="fu">experts_implementation</span><span class="kw">:</span><span class="at"> scattermoe</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div></li>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb9"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a><span class="fu">plugins</span><span class="kw">:</span></span>
+<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> axolotl.integrations.kernels.KernelsPlugin</span></span>
+<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a><span class="fu">use_kernels</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
+<span id="cb9-4"><a href="#cb9-4" aria-hidden="true" tabindex="-1"></a><span class="fu">use_scattermoe</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
+<span id="cb9-5"><a href="#cb9-5" aria-hidden="true" tabindex="-1"></a><span class="fu">experts_implementation</span><span class="kw">:</span><span class="at"> scattermoe</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div></li>
 </ul>
 </section>
+<section id="vlm-vision-training" class="level3">
+<h3 class="anchored" data-anchor-id="vlm-vision-training">VLM (Vision) Training</h3>
+<p>All Gemma4 models load as <code>Gemma4ForConditionalGeneration</code> with a vision tower. No custom <code>ProcessingStrategy</code> needed — the base class auto-detects the image token.</p>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb10"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> google/gemma-4-E2B-it</span><span class="co">   # or E4B-it, 26B-A4B</span></span>
+<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a><span class="fu">processor_type</span><span class="kw">:</span><span class="at"> AutoProcessor</span></span>
+<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a><span class="fu">freeze_mm_modules</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
+<span id="cb10-4"><a href="#cb10-4" aria-hidden="true" tabindex="-1"></a><span class="fu">chat_template</span><span class="kw">:</span><span class="at"> gemma4</span></span>
+<span id="cb10-5"><a href="#cb10-5" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb10-6"><a href="#cb10-6" aria-hidden="true" tabindex="-1"></a><span class="fu">skip_prepare_dataset</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
+<span id="cb10-7"><a href="#cb10-7" aria-hidden="true" tabindex="-1"></a><span class="fu">remove_unused_columns</span><span class="kw">:</span><span class="at"> </span><span class="ch">false</span></span>
+<span id="cb10-8"><a href="#cb10-8" aria-hidden="true" tabindex="-1"></a><span class="fu">sample_packing</span><span class="kw">:</span><span class="at"> </span><span class="ch">false</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<p>A starting VLM loss of ~8-15 is typical. In most runs, loss converges below 1.0 within ~30-50 steps, though results may vary across configurations.</p>
+<p>For the 26B-A4B MoE variant with ScatterMoE + expert LoRA + CCE, add:</p>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb11"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="fu">plugins</span><span class="kw">:</span></span>
+<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin</span></span>
+<span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> axolotl.integrations.kernels.KernelsPlugin</span></span>
+<span id="cb11-4"><a href="#cb11-4" aria-hidden="true" tabindex="-1"></a><span class="fu">use_kernels</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
+<span id="cb11-5"><a href="#cb11-5" aria-hidden="true" tabindex="-1"></a><span class="fu">use_scattermoe</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
+<span id="cb11-6"><a href="#cb11-6" aria-hidden="true" tabindex="-1"></a><span class="fu">experts_implementation</span><span class="kw">:</span><span class="at"> scattermoe</span></span>
+<span id="cb11-7"><a href="#cb11-7" aria-hidden="true" tabindex="-1"></a><span class="fu">lora_target_parameters</span><span class="kw">:</span></span>
+<span id="cb11-8"><a href="#cb11-8" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> experts.gate_up_proj</span></span>
+<span id="cb11-9"><a href="#cb11-9" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> experts.down_proj</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+</section>
 <section id="common-issues" class="level3">
 <h3 class="anchored" data-anchor-id="common-issues">Common issues</h3>
 <table class="caption-top table">
@@ -969,9 +1043,9 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});
 <li><p>256 experts, 8 active per token</p></li>
 <li><p>Known weight scale drift in late DeltaNet layers (36-38) due to AdamW + rare expert interaction</p></li>
 <li><p>Fix: <code>normalize_weight_scales</code> config to detect and rescale outliers:</p>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb5"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="fu">normalize_weight_scales</span><span class="kw">:</span></span>
-<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> </span><span class="fu">name_pattern</span><span class="kw">:</span><span class="at"> </span><span class="st">'linear_attn\.conv1d\.weight'</span></span>
-<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">threshold</span><span class="kw">:</span><span class="at"> </span><span class="fl">1.3</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div></li>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb12"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="fu">normalize_weight_scales</span><span class="kw">:</span></span>
+<span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> </span><span class="fu">name_pattern</span><span class="kw">:</span><span class="at"> </span><span class="st">'linear_attn\.conv1d\.weight'</span></span>
+<span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">threshold</span><span class="kw">:</span><span class="at"> </span><span class="fl">1.3</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div></li>
 </ul>
 </section>
 <section id="general-moe-notes" class="level2">
diff --git a/docs/agents/sft.html b/docs/agents/sft.html
index 612a5b136..4a6080cc2 100644
--- a/docs/agents/sft.html
+++ b/docs/agents/sft.html
@@ -23,6 +23,41 @@ ul.task-list li input[type="checkbox"] {
   margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */ 
   vertical-align: middle;
 }
+/* CSS for syntax highlighting */
+html { -webkit-text-size-adjust: 100%; }
+pre > code.sourceCode { white-space: pre; position: relative; }
+pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
+pre > code.sourceCode > span:empty { height: 1.2em; }
+.sourceCode { overflow: visible; }
+code.sourceCode > span { color: inherit; text-decoration: inherit; }
+div.sourceCode { margin: 1em 0; }
+pre.sourceCode { margin: 0; }
+@media screen {
+div.sourceCode { overflow: auto; }
+}
+@media print {
+pre > code.sourceCode { white-space: pre-wrap; }
+pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
+}
+pre.numberSource code
+  { counter-reset: source-line 0; }
+pre.numberSource code > span
+  { position: relative; left: -4em; counter-increment: source-line; }
+pre.numberSource code > span > a:first-child::before
+  { content: counter(source-line);
+    position: relative; left: -1em; text-align: right; vertical-align: baseline;
+    border: none; display: inline-block;
+    -webkit-touch-callout: none; -webkit-user-select: none;
+    -khtml-user-select: none; -moz-user-select: none;
+    -ms-user-select: none; user-select: none;
+    padding: 0 4px; width: 4em;
+  }
+pre.numberSource { margin-left: 3em;  padding-left: 4px; }
+div.sourceCode
+  {   }
+@media screen {
+pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
+}
 </style>
 
 
@@ -760,6 +795,7 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});
   <li><a href="#hyperparameter-ranges" id="toc-hyperparameter-ranges" class="nav-link" data-scroll-target="#hyperparameter-ranges">Hyperparameter Ranges</a></li>
   <li><a href="#healthy-training-indicators" id="toc-healthy-training-indicators" class="nav-link" data-scroll-target="#healthy-training-indicators">Healthy Training Indicators</a></li>
   <li><a href="#known-issues" id="toc-known-issues" class="nav-link" data-scroll-target="#known-issues">Known Issues</a></li>
+  <li><a href="#profiling" id="toc-profiling" class="nav-link" data-scroll-target="#profiling">Profiling</a></li>
   <li><a href="#file-map" id="toc-file-map" class="nav-link" data-scroll-target="#file-map">File Map</a></li>
   </ul></li>
   </ul>
@@ -1009,6 +1045,22 @@ Multi-GPU: FSDP or DeepSpeed shards model across GPUs automatically.</code></pre
 </tr>
 </tbody>
 </table>
+</section>
+<section id="profiling" class="level2">
+<h2 class="anchored" data-anchor-id="profiling">Profiling</h2>
+<p>To profile training and identify optimization opportunities:</p>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb3"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Profile steps 3-7 (after warmup/autotuning settles)</span></span>
+<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a><span class="fu">profiler_steps_start</span><span class="kw">:</span><span class="at"> </span><span class="dv">3</span></span>
+<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a><span class="fu">profiler_steps</span><span class="kw">:</span><span class="at"> </span><span class="dv">5</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<p>This produces <code>profiler_trace.json</code> (Chrome trace) and <code>snapshot.pickle</code> (memory snapshot) in <code>output_dir</code>.
+View the Chrome trace at <code>chrome://tracing</code>.</p>
+<p>To programmatically inspect the trace:</p>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb4"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="ex">python</span> scripts/analyze_profile.py output_dir/</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<p>The trace shows per-kernel CUDA times, memory allocations, and operator-level breakdown. Look for:
+- <strong>Large matmul kernels</strong>: candidates for fusion or quantization
+- <strong>Memory copies (H2D/D2H)</strong>: unnecessary data movement
+- <strong>Small frequent kernels</strong>: candidates for kernel fusion
+- <strong>Gaps between kernels</strong>: pipeline bubbles from CPU overhead</p>
 <p>Full troubleshooting: <a href="../../docs/training_stability.html">training_stability.qmd</a>, <a href="../../docs/debugging.html">debugging.qmd</a></p>
 </section>
 <section id="file-map" class="level2">
diff --git a/docs/multimodal.html b/docs/multimodal.html
index 250772dc7..b74ed570d 100644
--- a/docs/multimodal.html
+++ b/docs/multimodal.html
@@ -797,6 +797,7 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});
   <li><a href="#sec-mistral-small-4" id="toc-sec-mistral-small-4" class="nav-link" data-scroll-target="#sec-mistral-small-4">Mistral-Small-4</a></li>
   <li><a href="#sec-magistral-small-2509" id="toc-sec-magistral-small-2509" class="nav-link" data-scroll-target="#sec-magistral-small-2509">Magistral-Small-2509</a></li>
   <li><a href="#sec-voxtral" id="toc-sec-voxtral" class="nav-link" data-scroll-target="#sec-voxtral">Voxtral</a></li>
+  <li><a href="#sec-gemma-4" id="toc-sec-gemma-4" class="nav-link" data-scroll-target="#sec-gemma-4">Gemma-4</a></li>
   <li><a href="#sec-gemma-3" id="toc-sec-gemma-3" class="nav-link" data-scroll-target="#sec-gemma-3">Gemma-3</a></li>
   <li><a href="#sec-gemma-3n" id="toc-sec-gemma-3n" class="nav-link" data-scroll-target="#sec-gemma-3n">Gemma-3n</a></li>
   <li><a href="#sec-qwen2-vl" id="toc-sec-qwen2-vl" class="nav-link" data-scroll-target="#sec-qwen2-vl">Qwen2-VL</a></li>
@@ -844,6 +845,7 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});
 <section id="supported-models" class="level2">
 <h2 class="anchored" data-anchor-id="supported-models">Supported Models</h2>
 <ul>
+<li><a href="#sec-gemma-4">Gemma-4</a> <em>(NEW)</em></li>
 <li><a href="#sec-mllama">Mllama</a></li>
 <li><a href="#sec-llama4">Llama4</a></li>
 <li><a href="#sec-pixtral">Pixtral</a></li>
@@ -998,6 +1000,55 @@ Tip
 <span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a></span>
 <span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a><span class="fu">processor_type</span><span class="kw">:</span><span class="at"> VoxtralProcessor</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 </section>
+<section id="sec-gemma-4" class="level3">
+<h3 class="anchored" data-anchor-id="sec-gemma-4">Gemma-4</h3>
+<p>All Gemma 4 variants (E2B, E4B, 26B-A4B, 31B) load as multimodal models even for text-only training.</p>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb10"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> google/gemma-4-E2B-it</span><span class="co">  # or E4B-it, 26B-A4B, 31B</span></span>
+<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a><span class="fu">chat_template</span><span class="kw">:</span><span class="at"> gemma4</span></span>
+<span id="cb10-4"><a href="#cb10-4" aria-hidden="true" tabindex="-1"></a><span class="fu">freeze_mm_modules</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span><span class="co">  # freeze vision/audio encoders for text-only or vision LoRA</span></span>
+<span id="cb10-5"><a href="#cb10-5" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb10-6"><a href="#cb10-6" aria-hidden="true" tabindex="-1"></a><span class="co"># For the 26B-A4B MoE model, enable ScatterMoE and expert LoRA:</span></span>
+<span id="cb10-7"><a href="#cb10-7" aria-hidden="true" tabindex="-1"></a><span class="fu">plugins</span><span class="kw">:</span></span>
+<span id="cb10-8"><a href="#cb10-8" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin</span></span>
+<span id="cb10-9"><a href="#cb10-9" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> axolotl.integrations.kernels.KernelsPlugin</span></span>
+<span id="cb10-10"><a href="#cb10-10" aria-hidden="true" tabindex="-1"></a><span class="fu">use_kernels</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
+<span id="cb10-11"><a href="#cb10-11" aria-hidden="true" tabindex="-1"></a><span class="fu">use_scattermoe</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span></span>
+<span id="cb10-12"><a href="#cb10-12" aria-hidden="true" tabindex="-1"></a><span class="fu">experts_implementation</span><span class="kw">:</span><span class="at"> scattermoe</span></span>
+<span id="cb10-13"><a href="#cb10-13" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb10-14"><a href="#cb10-14" aria-hidden="true" tabindex="-1"></a><span class="fu">lora_target_modules</span><span class="kw">:</span><span class="at"> </span><span class="st">'model.language_model.layers.[\d]+.(_checkpoint_wrapped_module.)?(mlp|self_attn).(up|down|gate|q|k|v|o)_proj'</span></span>
+<span id="cb10-15"><a href="#cb10-15" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb10-16"><a href="#cb10-16" aria-hidden="true" tabindex="-1"></a><span class="co"># MoE expert LoRA (3D tensors, not nn.Linear) — only for 26B-A4B:</span></span>
+<span id="cb10-17"><a href="#cb10-17" aria-hidden="true" tabindex="-1"></a><span class="fu">lora_target_parameters</span><span class="kw">:</span></span>
+<span id="cb10-18"><a href="#cb10-18" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> experts.gate_up_proj</span></span>
+<span id="cb10-19"><a href="#cb10-19" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> experts.down_proj</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="callout callout-style-default callout-warning callout-titled">
+<div class="callout-header d-flex align-content-center">
+<div class="callout-icon-container">
+<i class="callout-icon"></i>
+</div>
+<div class="callout-title-container flex-fill">
+Warning
+</div>
+</div>
+<div class="callout-body-container callout-body">
+<p>Gemma 4 VLM training starts with high loss (~8-15). This is expected — see the <a href="../docs/training_stability.html">training stability guide</a> for details.</p>
+</div>
+</div>
+<div class="callout callout-style-default callout-tip callout-titled">
+<div class="callout-header d-flex align-content-center">
+<div class="callout-icon-container">
+<i class="callout-icon"></i>
+</div>
+<div class="callout-title-container flex-fill">
+Tip
+</div>
+</div>
+<div class="callout-body-container callout-body">
+<p>For DDP training, axolotl auto-detects Gemma4 and sets <code>use_reentrant=False</code> and <code>ddp_find_unused_parameters=True</code>. However, when <code>activation_offloading: true</code>, <code>ddp_find_unused_parameters</code> is skipped (checkpoint wrappers conflict with it); use <code>freeze_mm_modules: true</code> instead to handle unused vision/audio params. For FSDP2, use <code>fsdp_transformer_layer_cls_to_wrap: Gemma4TextDecoderLayer</code>.</p>
+</div>
+</div>
+</section>
 <section id="sec-gemma-3" class="level3">
 <h3 class="anchored" data-anchor-id="sec-gemma-3">Gemma-3</h3>
 <div class="callout callout-style-default callout-tip callout-titled">
@@ -1014,9 +1065,9 @@ Tip
 </div>
 </div>
 <p>For multi-modal 4B/12B/27B models, use the following config:</p>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb10"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> google/gemma-3-4b-it</span></span>
-<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a><span class="fu">chat_template</span><span class="kw">:</span><span class="at"> gemma3</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb11"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> google/gemma-3-4b-it</span></span>
+<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a><span class="fu">chat_template</span><span class="kw">:</span><span class="at"> gemma3</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 </section>
 <section id="sec-gemma-3n" class="level3">
 <h3 class="anchored" data-anchor-id="sec-gemma-3n">Gemma-3n</h3>
@@ -1046,42 +1097,42 @@ Tip
 <p>Please make sure to install <code>timm</code> via <code>pip3 install timm==1.0.17</code></p>
 </div>
 </div>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb11"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> google/gemma-3n-E2B-it</span></span>
-<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb11-3"><a href="#cb11-3" aria-hidden="true" tabindex="-1"></a><span class="fu">chat_template</span><span class="kw">:</span><span class="at"> gemma3n</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb12"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> google/gemma-3n-E2B-it</span></span>
+<span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a><span class="fu">chat_template</span><span class="kw">:</span><span class="at"> gemma3n</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 </section>
 <section id="sec-qwen2-vl" class="level3">
 <h3 class="anchored" data-anchor-id="sec-qwen2-vl">Qwen2-VL</h3>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb12"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> Qwen/Qwen2-VL-7B-Instruct</span></span>
-<span id="cb12-2"><a href="#cb12-2" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb12-3"><a href="#cb12-3" aria-hidden="true" tabindex="-1"></a><span class="fu">chat_template</span><span class="kw">:</span><span class="at"> qwen2_vl</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb13"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb13-1"><a href="#cb13-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> Qwen/Qwen2-VL-7B-Instruct</span></span>
+<span id="cb13-2"><a href="#cb13-2" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb13-3"><a href="#cb13-3" aria-hidden="true" tabindex="-1"></a><span class="fu">chat_template</span><span class="kw">:</span><span class="at"> qwen2_vl</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 </section>
 <section id="sec-qwen25-vl" class="level3">
 <h3 class="anchored" data-anchor-id="sec-qwen25-vl">Qwen2.5-VL</h3>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb13"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb13-1"><a href="#cb13-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> Qwen/Qwen2.5-VL-7B-Instruct</span></span>
-<span id="cb13-2"><a href="#cb13-2" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb13-3"><a href="#cb13-3" aria-hidden="true" tabindex="-1"></a><span class="fu">chat_template</span><span class="kw">:</span><span class="at"> qwen2_vl</span><span class="co">  # same as qwen2-vl</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
-</section>
-<section id="sec-qwen3-vl" class="level3">
-<h3 class="anchored" data-anchor-id="sec-qwen3-vl">Qwen3-VL</h3>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb14"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb14-1"><a href="#cb14-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> Qwen/Qwen3-VL-4B-Instruct</span></span>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb14"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb14-1"><a href="#cb14-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> Qwen/Qwen2.5-VL-7B-Instruct</span></span>
 <span id="cb14-2"><a href="#cb14-2" aria-hidden="true" tabindex="-1"></a></span>
 <span id="cb14-3"><a href="#cb14-3" aria-hidden="true" tabindex="-1"></a><span class="fu">chat_template</span><span class="kw">:</span><span class="at"> qwen2_vl</span><span class="co">  # same as qwen2-vl</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 </section>
+<section id="sec-qwen3-vl" class="level3">
+<h3 class="anchored" data-anchor-id="sec-qwen3-vl">Qwen3-VL</h3>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb15"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb15-1"><a href="#cb15-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> Qwen/Qwen3-VL-4B-Instruct</span></span>
+<span id="cb15-2"><a href="#cb15-2" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb15-3"><a href="#cb15-3" aria-hidden="true" tabindex="-1"></a><span class="fu">chat_template</span><span class="kw">:</span><span class="at"> qwen2_vl</span><span class="co">  # same as qwen2-vl</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+</section>
 <section id="sec-qwen3-5" class="level3">
 <h3 class="anchored" data-anchor-id="sec-qwen3-5">Qwen3.5</h3>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb15"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb15-1"><a href="#cb15-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> Qwen/Qwen3.5-9B</span></span>
-<span id="cb15-2"><a href="#cb15-2" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb15-3"><a href="#cb15-3" aria-hidden="true" tabindex="-1"></a><span class="fu">chat_template</span><span class="kw">:</span><span class="at"> qwen3_5</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb16"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb16-1"><a href="#cb16-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> Qwen/Qwen3.5-9B</span></span>
+<span id="cb16-2"><a href="#cb16-2" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb16-3"><a href="#cb16-3" aria-hidden="true" tabindex="-1"></a><span class="fu">chat_template</span><span class="kw">:</span><span class="at"> qwen3_5</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 </section>
 <section id="sec-glm-4-6v" class="level3">
 <h3 class="anchored" data-anchor-id="sec-glm-4-6v">GLM-4.6V</h3>
 <p>Both GLM-4.6V (106B MoE) and GLM-4.6V-Flash (9B) are supported.</p>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb16"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb16-1"><a href="#cb16-1" aria-hidden="true" tabindex="-1"></a><span class="co"># GLM-4.6V (106B MoE version)</span></span>
-<span id="cb16-2"><a href="#cb16-2" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> zai-org/GLM-4.6V</span></span>
-<span id="cb16-3"><a href="#cb16-3" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb16-4"><a href="#cb16-4" aria-hidden="true" tabindex="-1"></a><span class="co"># OR GLM-4.6V-Flash (9B version)</span></span>
-<span id="cb16-5"><a href="#cb16-5" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> zai-org/GLM-4.6V-Flash</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb17"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb17-1"><a href="#cb17-1" aria-hidden="true" tabindex="-1"></a><span class="co"># GLM-4.6V (106B MoE version)</span></span>
+<span id="cb17-2"><a href="#cb17-2" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> zai-org/GLM-4.6V</span></span>
+<span id="cb17-3"><a href="#cb17-3" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb17-4"><a href="#cb17-4" aria-hidden="true" tabindex="-1"></a><span class="co"># OR GLM-4.6V-Flash (9B version)</span></span>
+<span id="cb17-5"><a href="#cb17-5" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> zai-org/GLM-4.6V-Flash</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 </section>
 <section id="sec-smolvlm2" class="level3">
 <h3 class="anchored" data-anchor-id="sec-smolvlm2">SmolVLM2</h3>
@@ -1098,7 +1149,7 @@ Tip
 <p>Please make sure to install <code>num2words</code> via <code>pip3 install num2words==0.5.14</code></p>
 </div>
 </div>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb17"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb17-1"><a href="#cb17-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> HuggingFaceTB/SmolVLM2-500M-Video-Instruct</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb18"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb18-1"><a href="#cb18-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> HuggingFaceTB/SmolVLM2-500M-Video-Instruct</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 </section>
 <section id="sec-lfm2-vl" class="level3">
 <h3 class="anchored" data-anchor-id="sec-lfm2-vl">LFM2-VL</h3>
@@ -1115,7 +1166,7 @@ Warning
 <p>Please uninstall <code>causal-conv1d</code> via <code>pip3 uninstall -y causal-conv1d</code></p>
 </div>
 </div>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb18"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb18-1"><a href="#cb18-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> LiquidAI/LFM2-VL-450M</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb19"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb19-1"><a href="#cb19-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> LiquidAI/LFM2-VL-450M</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 </section>
 <section id="sec-intern-vl" class="level3">
 <h3 class="anchored" data-anchor-id="sec-intern-vl">Intern-VL</h3>
@@ -1132,7 +1183,7 @@ Tip
 <p>Please make sure to install <code>timm</code> via <code>pip3 install timm==1.0.19</code></p>
 </div>
 </div>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb19"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb19-1"><a href="#cb19-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> OpenGVLab/InternVL3_5-8B</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb20"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb20-1"><a href="#cb20-1" aria-hidden="true" tabindex="-1"></a><span class="fu">base_model</span><span class="kw">:</span><span class="at"> OpenGVLab/InternVL3_5-8B</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 </section>
 </section>
 <section id="dataset-format" class="level2">
@@ -1217,31 +1268,31 @@ Warning
 <section id="example" class="level3">
 <h3 class="anchored" data-anchor-id="example">Example</h3>
 <p>Here is an example of a multi-modal dataset:</p>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb20"><pre class="sourceCode json code-with-copy"><code class="sourceCode json"><span id="cb20-1"><a href="#cb20-1" aria-hidden="true" tabindex="-1"></a><span class="ot">[</span></span>
-<span id="cb20-2"><a href="#cb20-2" aria-hidden="true" tabindex="-1"></a>  <span class="fu">{</span></span>
-<span id="cb20-3"><a href="#cb20-3" aria-hidden="true" tabindex="-1"></a>    <span class="dt">"messages"</span><span class="fu">:</span> <span class="ot">[</span></span>
-<span id="cb20-4"><a href="#cb20-4" aria-hidden="true" tabindex="-1"></a>        <span class="fu">{</span></span>
-<span id="cb20-5"><a href="#cb20-5" aria-hidden="true" tabindex="-1"></a>            <span class="dt">"role"</span><span class="fu">:</span> <span class="st">"system"</span><span class="fu">,</span></span>
-<span id="cb20-6"><a href="#cb20-6" aria-hidden="true" tabindex="-1"></a>            <span class="dt">"content"</span><span class="fu">:</span> <span class="ot">[</span></span>
-<span id="cb20-7"><a href="#cb20-7" aria-hidden="true" tabindex="-1"></a>              <span class="fu">{</span><span class="dt">"type"</span><span class="fu">:</span> <span class="st">"text"</span><span class="fu">,</span> <span class="dt">"text"</span><span class="fu">:</span> <span class="st">"You are a helpful assistant."</span><span class="fu">}</span></span>
-<span id="cb20-8"><a href="#cb20-8" aria-hidden="true" tabindex="-1"></a>              <span class="ot">]</span></span>
-<span id="cb20-9"><a href="#cb20-9" aria-hidden="true" tabindex="-1"></a>        <span class="fu">}</span><span class="ot">,</span></span>
-<span id="cb20-10"><a href="#cb20-10" aria-hidden="true" tabindex="-1"></a>        <span class="fu">{</span></span>
-<span id="cb20-11"><a href="#cb20-11" aria-hidden="true" tabindex="-1"></a>            <span class="dt">"role"</span><span class="fu">:</span> <span class="st">"user"</span><span class="fu">,</span></span>
-<span id="cb20-12"><a href="#cb20-12" aria-hidden="true" tabindex="-1"></a>            <span class="dt">"content"</span><span class="fu">:</span> <span class="ot">[</span></span>
-<span id="cb20-13"><a href="#cb20-13" aria-hidden="true" tabindex="-1"></a>                <span class="fu">{</span><span class="dt">"type"</span><span class="fu">:</span> <span class="st">"image"</span><span class="fu">,</span> <span class="dt">"url"</span><span class="fu">:</span> <span class="st">"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/bee.jpg"</span><span class="fu">}</span><span class="ot">,</span></span>
-<span id="cb20-14"><a href="#cb20-14" aria-hidden="true" tabindex="-1"></a>                <span class="fu">{</span><span class="dt">"type"</span><span class="fu">:</span> <span class="st">"text"</span><span class="fu">,</span> <span class="dt">"text"</span><span class="fu">:</span> <span class="st">"Describe this image in detail."</span><span class="fu">}</span></span>
-<span id="cb20-15"><a href="#cb20-15" aria-hidden="true" tabindex="-1"></a>            <span class="ot">]</span></span>
-<span id="cb20-16"><a href="#cb20-16" aria-hidden="true" tabindex="-1"></a>        <span class="fu">}</span><span class="ot">,</span></span>
-<span id="cb20-17"><a href="#cb20-17" aria-hidden="true" tabindex="-1"></a>        <span class="fu">{</span></span>
-<span id="cb20-18"><a href="#cb20-18" aria-hidden="true" tabindex="-1"></a>            <span class="dt">"role"</span><span class="fu">:</span> <span class="st">"assistant"</span><span class="fu">,</span></span>
-<span id="cb20-19"><a href="#cb20-19" aria-hidden="true" tabindex="-1"></a>            <span class="dt">"content"</span><span class="fu">:</span> <span class="ot">[</span></span>
-<span id="cb20-20"><a href="#cb20-20" aria-hidden="true" tabindex="-1"></a>              <span class="fu">{</span><span class="dt">"type"</span><span class="fu">:</span> <span class="st">"text"</span><span class="fu">,</span> <span class="dt">"text"</span><span class="fu">:</span> <span class="st">"The image is a bee."</span><span class="fu">}</span></span>
-<span id="cb20-21"><a href="#cb20-21" aria-hidden="true" tabindex="-1"></a>            <span class="ot">]</span></span>
-<span id="cb20-22"><a href="#cb20-22" aria-hidden="true" tabindex="-1"></a>        <span class="fu">}</span></span>
-<span id="cb20-23"><a href="#cb20-23" aria-hidden="true" tabindex="-1"></a>    <span class="ot">]</span></span>
-<span id="cb20-24"><a href="#cb20-24" aria-hidden="true" tabindex="-1"></a>  <span class="fu">}</span></span>
-<span id="cb20-25"><a href="#cb20-25" aria-hidden="true" tabindex="-1"></a><span class="ot">]</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb21"><pre class="sourceCode json code-with-copy"><code class="sourceCode json"><span id="cb21-1"><a href="#cb21-1" aria-hidden="true" tabindex="-1"></a><span class="ot">[</span></span>
+<span id="cb21-2"><a href="#cb21-2" aria-hidden="true" tabindex="-1"></a>  <span class="fu">{</span></span>
+<span id="cb21-3"><a href="#cb21-3" aria-hidden="true" tabindex="-1"></a>    <span class="dt">"messages"</span><span class="fu">:</span> <span class="ot">[</span></span>
+<span id="cb21-4"><a href="#cb21-4" aria-hidden="true" tabindex="-1"></a>        <span class="fu">{</span></span>
+<span id="cb21-5"><a href="#cb21-5" aria-hidden="true" tabindex="-1"></a>            <span class="dt">"role"</span><span class="fu">:</span> <span class="st">"system"</span><span class="fu">,</span></span>
+<span id="cb21-6"><a href="#cb21-6" aria-hidden="true" tabindex="-1"></a>            <span class="dt">"content"</span><span class="fu">:</span> <span class="ot">[</span></span>
+<span id="cb21-7"><a href="#cb21-7" aria-hidden="true" tabindex="-1"></a>              <span class="fu">{</span><span class="dt">"type"</span><span class="fu">:</span> <span class="st">"text"</span><span class="fu">,</span> <span class="dt">"text"</span><span class="fu">:</span> <span class="st">"You are a helpful assistant."</span><span class="fu">}</span></span>
+<span id="cb21-8"><a href="#cb21-8" aria-hidden="true" tabindex="-1"></a>              <span class="ot">]</span></span>
+<span id="cb21-9"><a href="#cb21-9" aria-hidden="true" tabindex="-1"></a>        <span class="fu">}</span><span class="ot">,</span></span>
+<span id="cb21-10"><a href="#cb21-10" aria-hidden="true" tabindex="-1"></a>        <span class="fu">{</span></span>
+<span id="cb21-11"><a href="#cb21-11" aria-hidden="true" tabindex="-1"></a>            <span class="dt">"role"</span><span class="fu">:</span> <span class="st">"user"</span><span class="fu">,</span></span>
+<span id="cb21-12"><a href="#cb21-12" aria-hidden="true" tabindex="-1"></a>            <span class="dt">"content"</span><span class="fu">:</span> <span class="ot">[</span></span>
+<span id="cb21-13"><a href="#cb21-13" aria-hidden="true" tabindex="-1"></a>                <span class="fu">{</span><span class="dt">"type"</span><span class="fu">:</span> <span class="st">"image"</span><span class="fu">,</span> <span class="dt">"url"</span><span class="fu">:</span> <span class="st">"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/bee.jpg"</span><span class="fu">}</span><span class="ot">,</span></span>
+<span id="cb21-14"><a href="#cb21-14" aria-hidden="true" tabindex="-1"></a>                <span class="fu">{</span><span class="dt">"type"</span><span class="fu">:</span> <span class="st">"text"</span><span class="fu">,</span> <span class="dt">"text"</span><span class="fu">:</span> <span class="st">"Describe this image in detail."</span><span class="fu">}</span></span>
+<span id="cb21-15"><a href="#cb21-15" aria-hidden="true" tabindex="-1"></a>            <span class="ot">]</span></span>
+<span id="cb21-16"><a href="#cb21-16" aria-hidden="true" tabindex="-1"></a>        <span class="fu">}</span><span class="ot">,</span></span>
+<span id="cb21-17"><a href="#cb21-17" aria-hidden="true" tabindex="-1"></a>        <span class="fu">{</span></span>
+<span id="cb21-18"><a href="#cb21-18" aria-hidden="true" tabindex="-1"></a>            <span class="dt">"role"</span><span class="fu">:</span> <span class="st">"assistant"</span><span class="fu">,</span></span>
+<span id="cb21-19"><a href="#cb21-19" aria-hidden="true" tabindex="-1"></a>            <span class="dt">"content"</span><span class="fu">:</span> <span class="ot">[</span></span>
+<span id="cb21-20"><a href="#cb21-20" aria-hidden="true" tabindex="-1"></a>              <span class="fu">{</span><span class="dt">"type"</span><span class="fu">:</span> <span class="st">"text"</span><span class="fu">,</span> <span class="dt">"text"</span><span class="fu">:</span> <span class="st">"The image is a bee."</span><span class="fu">}</span></span>
+<span id="cb21-21"><a href="#cb21-21" aria-hidden="true" tabindex="-1"></a>            <span class="ot">]</span></span>
+<span id="cb21-22"><a href="#cb21-22" aria-hidden="true" tabindex="-1"></a>        <span class="fu">}</span></span>
+<span id="cb21-23"><a href="#cb21-23" aria-hidden="true" tabindex="-1"></a>    <span class="ot">]</span></span>
+<span id="cb21-24"><a href="#cb21-24" aria-hidden="true" tabindex="-1"></a>  <span class="fu">}</span></span>
+<span id="cb21-25"><a href="#cb21-25" aria-hidden="true" tabindex="-1"></a><span class="ot">]</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 </section>
 </section>
 <section id="faq" class="level2">
diff --git a/index.html b/index.html
index 58f6573e2..bf9367842 100644
--- a/index.html
+++ b/index.html
@@ -906,7 +906,7 @@ Expand older updates
 <p><strong>Requirements</strong>:</p>
 <ul>
 <li>NVIDIA GPU (Ampere or newer for <code>bf16</code> and Flash Attention) or AMD GPU</li>
-<li>Python 3.11</li>
+<li>Python &gt;=3.11 (3.12 recommended)</li>
 <li>PyTorch ≥2.9.1</li>
 </ul>
 <section id="google-colab" class="level3">
@@ -920,19 +920,45 @@ Expand older updates
 </section>
 <section id="installation" class="level3">
 <h3 class="anchored" data-anchor-id="installation">Installation</h3>
+<section id="using-uv-recommended" class="level4">
+<h4 class="anchored" data-anchor-id="using-uv-recommended">Using uv (recommended)</h4>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb1"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="co"># install uv if you don't already have it installed</span></span>
+<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="ex">curl</span> <span class="at">-LsSf</span> https://astral.sh/uv/install.sh <span class="kw">|</span> <span class="fu">sh</span></span>
+<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="bu">source</span> <span class="va">$HOME</span>/.local/bin/env</span>
+<span id="cb1-4"><a href="#cb1-4" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb1-5"><a href="#cb1-5" aria-hidden="true" tabindex="-1"></a><span class="co"># CUDA 12.8.1 tends to have better package compatibility</span></span>
+<span id="cb1-6"><a href="#cb1-6" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">UV_TORCH_BACKEND</span><span class="op">=</span>cu128</span>
+<span id="cb1-7"><a href="#cb1-7" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb1-8"><a href="#cb1-8" aria-hidden="true" tabindex="-1"></a><span class="co"># create a new virtual environment</span></span>
+<span id="cb1-9"><a href="#cb1-9" aria-hidden="true" tabindex="-1"></a><span class="ex">uv</span> venv <span class="at">--python</span> 3.12</span>
+<span id="cb1-10"><a href="#cb1-10" aria-hidden="true" tabindex="-1"></a><span class="bu">source</span> .venv/bin/activate</span>
+<span id="cb1-11"><a href="#cb1-11" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb1-12"><a href="#cb1-12" aria-hidden="true" tabindex="-1"></a><span class="ex">uv</span> pip install torch==2.10.0 torchvision</span>
+<span id="cb1-13"><a href="#cb1-13" aria-hidden="true" tabindex="-1"></a><span class="ex">uv</span> pip install <span class="at">--no-build-isolation</span> axolotl<span class="pp">[</span><span class="ss">deepspeed</span><span class="pp">]</span></span>
+<span id="cb1-14"><a href="#cb1-14" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb1-15"><a href="#cb1-15" aria-hidden="true" tabindex="-1"></a><span class="co"># recommended - install cut-cross-entropy</span></span>
+<span id="cb1-16"><a href="#cb1-16" aria-hidden="true" tabindex="-1"></a><span class="ex">uv</span> pip install <span class="st">"cut-cross-entropy[transformers] @ git+https://github.com/axolotl-ai-cloud/ml-cross-entropy.git@main"</span></span>
+<span id="cb1-17"><a href="#cb1-17" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb1-18"><a href="#cb1-18" aria-hidden="true" tabindex="-1"></a><span class="co"># (optional) - prefetch flash-attn2 and causal-conv1d kernels</span></span>
+<span id="cb1-19"><a href="#cb1-19" aria-hidden="true" tabindex="-1"></a><span class="ex">uv</span> run <span class="at">--python</span> 3.12 python <span class="at">-c</span> <span class="st">"from kernels import get_kernel; get_kernel('kernels-community/flash-attn2'); get_kernel('kernels-community/causal-conv1d')"</span></span>
+<span id="cb1-20"><a href="#cb1-20" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb1-21"><a href="#cb1-21" aria-hidden="true" tabindex="-1"></a><span class="co"># Download example axolotl configs, deepspeed configs</span></span>
+<span id="cb1-22"><a href="#cb1-22" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> fetch examples</span>
+<span id="cb1-23"><a href="#cb1-23" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> fetch deepspeed_configs  <span class="co"># OPTIONAL</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+</section>
 <section id="using-pip" class="level4">
 <h4 class="anchored" data-anchor-id="using-pip">Using pip</h4>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb1"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="ex">pip3</span> install <span class="at">-U</span> packaging==26.0 setuptools==75.8.0 wheel ninja</span>
-<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="ex">pip3</span> install <span class="at">--no-build-isolation</span> axolotl<span class="pp">[</span><span class="ss">flash</span><span class="pp">-</span><span class="ss">attn,deepspeed</span><span class="pp">]</span></span>
-<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb1-4"><a href="#cb1-4" aria-hidden="true" tabindex="-1"></a><span class="co"># Download example axolotl configs, deepspeed configs</span></span>
-<span id="cb1-5"><a href="#cb1-5" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> fetch examples</span>
-<span id="cb1-6"><a href="#cb1-6" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> fetch deepspeed_configs  <span class="co"># OPTIONAL</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb2"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="ex">pip3</span> install <span class="at">-U</span> packaging==26.0 setuptools==75.8.0 wheel ninja</span>
+<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="ex">pip3</span> install <span class="at">--no-build-isolation</span> axolotl<span class="pp">[</span><span class="ss">flash</span><span class="pp">-</span><span class="ss">attn,deepspeed</span><span class="pp">]</span></span>
+<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb2-4"><a href="#cb2-4" aria-hidden="true" tabindex="-1"></a><span class="co"># Download example axolotl configs, deepspeed configs</span></span>
+<span id="cb2-5"><a href="#cb2-5" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> fetch examples</span>
+<span id="cb2-6"><a href="#cb2-6" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> fetch deepspeed_configs  <span class="co"># OPTIONAL</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 </section>
 <section id="using-docker" class="level4">
 <h4 class="anchored" data-anchor-id="using-docker">Using Docker</h4>
 <p>Installing with Docker can be less error prone than installing in your own environment.</p>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb2"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="ex">docker</span> run <span class="at">--gpus</span> <span class="st">'"all"'</span> <span class="at">--rm</span> <span class="at">-it</span> axolotlai/axolotl:main-latest</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb3"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="ex">docker</span> run <span class="at">--gpus</span> <span class="st">'"all"'</span> <span class="at">--rm</span> <span class="at">-it</span> axolotlai/axolotl:main-latest</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 <p>Other installation approaches are described <a href="https://docs.axolotl.ai/docs/installation.html">here</a>.</p>
 </section>
 <section id="cloud-providers" class="level4">
@@ -952,14 +978,14 @@ Expand older updates
 </section>
 <section id="your-first-fine-tune" class="level3">
 <h3 class="anchored" data-anchor-id="your-first-fine-tune">Your First Fine-tune</h3>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb3"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Fetch axolotl examples</span></span>
-<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> fetch examples</span>
-<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb3-4"><a href="#cb3-4" aria-hidden="true" tabindex="-1"></a><span class="co"># Or, specify a custom path</span></span>
-<span id="cb3-5"><a href="#cb3-5" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> fetch examples <span class="at">--dest</span> path/to/folder</span>
-<span id="cb3-6"><a href="#cb3-6" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb3-7"><a href="#cb3-7" aria-hidden="true" tabindex="-1"></a><span class="co"># Train a model using LoRA</span></span>
-<span id="cb3-8"><a href="#cb3-8" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> train examples/llama-3/lora-1b.yml</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb4"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Fetch axolotl examples</span></span>
+<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> fetch examples</span>
+<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a><span class="co"># Or, specify a custom path</span></span>
+<span id="cb4-5"><a href="#cb4-5" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> fetch examples <span class="at">--dest</span> path/to/folder</span>
+<span id="cb4-6"><a href="#cb4-6" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb4-7"><a href="#cb4-7" aria-hidden="true" tabindex="-1"></a><span class="co"># Train a model using LoRA</span></span>
+<span id="cb4-8"><a href="#cb4-8" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> train examples/llama-3/lora-1b.yml</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 <p>That’s it! Check out our <a href="https://docs.axolotl.ai/docs/getting-started.html">Getting Started Guide</a> for a more detailed walkthrough.</p>
 </section>
 </section>
@@ -980,20 +1006,20 @@ Expand older updates
 <section id="ai-agent-support" class="level2">
 <h2 class="anchored" data-anchor-id="ai-agent-support">AI Agent Support</h2>
 <p>Axolotl ships with built-in documentation optimized for AI coding agents (Claude Code, Cursor, Copilot, etc.). These docs are bundled with the pip package — no repo clone needed.</p>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb4"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Show overview and available training methods</span></span>
-<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> agent-docs</span>
-<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a><span class="co"># Topic-specific references</span></span>
-<span id="cb4-5"><a href="#cb4-5" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> agent-docs sft                 <span class="co"># supervised fine-tuning</span></span>
-<span id="cb4-6"><a href="#cb4-6" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> agent-docs grpo                <span class="co"># GRPO online RL</span></span>
-<span id="cb4-7"><a href="#cb4-7" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> agent-docs preference_tuning   <span class="co"># DPO, KTO, ORPO, SimPO</span></span>
-<span id="cb4-8"><a href="#cb4-8" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> agent-docs reward_modelling    <span class="co"># outcome and process reward models</span></span>
-<span id="cb4-9"><a href="#cb4-9" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> agent-docs pretraining         <span class="co"># continual pretraining</span></span>
-<span id="cb4-10"><a href="#cb4-10" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> agent-docs <span class="at">--list</span>              <span class="co"># list all topics</span></span>
-<span id="cb4-11"><a href="#cb4-11" aria-hidden="true" tabindex="-1"></a></span>
-<span id="cb4-12"><a href="#cb4-12" aria-hidden="true" tabindex="-1"></a><span class="co"># Dump config schema for programmatic use</span></span>
-<span id="cb4-13"><a href="#cb4-13" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> config-schema</span>
-<span id="cb4-14"><a href="#cb4-14" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> config-schema <span class="at">--field</span> adapter</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb5"><pre class="sourceCode bash code-with-copy"><code class="sourceCode bash"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Show overview and available training methods</span></span>
+<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> agent-docs</span>
+<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb5-4"><a href="#cb5-4" aria-hidden="true" tabindex="-1"></a><span class="co"># Topic-specific references</span></span>
+<span id="cb5-5"><a href="#cb5-5" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> agent-docs sft                 <span class="co"># supervised fine-tuning</span></span>
+<span id="cb5-6"><a href="#cb5-6" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> agent-docs grpo                <span class="co"># GRPO online RL</span></span>
+<span id="cb5-7"><a href="#cb5-7" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> agent-docs preference_tuning   <span class="co"># DPO, KTO, ORPO, SimPO</span></span>
+<span id="cb5-8"><a href="#cb5-8" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> agent-docs reward_modelling    <span class="co"># outcome and process reward models</span></span>
+<span id="cb5-9"><a href="#cb5-9" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> agent-docs pretraining         <span class="co"># continual pretraining</span></span>
+<span id="cb5-10"><a href="#cb5-10" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> agent-docs <span class="at">--list</span>              <span class="co"># list all topics</span></span>
+<span id="cb5-11"><a href="#cb5-11" aria-hidden="true" tabindex="-1"></a></span>
+<span id="cb5-12"><a href="#cb5-12" aria-hidden="true" tabindex="-1"></a><span class="co"># Dump config schema for programmatic use</span></span>
+<span id="cb5-13"><a href="#cb5-13" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> config-schema</span>
+<span id="cb5-14"><a href="#cb5-14" aria-hidden="true" tabindex="-1"></a><span class="ex">axolotl</span> config-schema <span class="at">--field</span> adapter</span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 <p>If you’re working with the source repo, agent docs are also available at <code>docs/agents/</code> and the project overview is in <code>AGENTS.md</code>.</p>
 </section>
 <section id="getting-help" class="level2">
@@ -1023,13 +1049,13 @@ disable it, set AXOLOTL_DO_NOT_TRACK=1. For more details, see our <a href="https
 <section id="citing-axolotl" class="level2">
 <h2 class="anchored" data-anchor-id="citing-axolotl">📝 Citing Axolotl</h2>
 <p>If you use Axolotl in your research or projects, please cite it as follows:</p>
-<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb5"><pre class="sourceCode bibtex code-with-copy"><code class="sourceCode bibtex"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="co">@software{axolotl,</span></span>
-<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a><span class="co">  title = {Axolotl: Open Source LLM Post-Training},</span></span>
-<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a><span class="co">  author = {{Axolotl maintainers and contributors}},</span></span>
-<span id="cb5-4"><a href="#cb5-4" aria-hidden="true" tabindex="-1"></a><span class="co">  url = {https://github.com/axolotl-ai-cloud/axolotl},</span></span>
-<span id="cb5-5"><a href="#cb5-5" aria-hidden="true" tabindex="-1"></a><span class="co">  license = {Apache-2.0},</span></span>
-<span id="cb5-6"><a href="#cb5-6" aria-hidden="true" tabindex="-1"></a><span class="co">  year = {2023}</span></span>
-<span id="cb5-7"><a href="#cb5-7" aria-hidden="true" tabindex="-1"></a><span class="co">}</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
+<div class="code-copy-outer-scaffold"><div class="sourceCode" id="cb6"><pre class="sourceCode bibtex code-with-copy"><code class="sourceCode bibtex"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="co">@software{axolotl,</span></span>
+<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a><span class="co">  title = {Axolotl: Open Source LLM Post-Training},</span></span>
+<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a><span class="co">  author = {{Axolotl maintainers and contributors}},</span></span>
+<span id="cb6-4"><a href="#cb6-4" aria-hidden="true" tabindex="-1"></a><span class="co">  url = {https://github.com/axolotl-ai-cloud/axolotl},</span></span>
+<span id="cb6-5"><a href="#cb6-5" aria-hidden="true" tabindex="-1"></a><span class="co">  license = {Apache-2.0},</span></span>
+<span id="cb6-6"><a href="#cb6-6" aria-hidden="true" tabindex="-1"></a><span class="co">  year = {2023}</span></span>
+<span id="cb6-7"><a href="#cb6-7" aria-hidden="true" tabindex="-1"></a><span class="co">}</span></span></code></pre></div><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></div>
 </section>
 <section id="license" class="level2">
 <h2 class="anchored" data-anchor-id="license">📜 License</h2>
diff --git a/search.json b/search.json
index f8babb061..f588ca744 100644
--- a/search.json
+++ b/search.json
@@ -2250,7 +2250,7 @@
     "href": "docs/multimodal.html",
     "title": "MultiModal / Vision Language Models (BETA)",
     "section": "",
-    "text": "Mllama\nLlama4\nPixtral\nLlava-1.5\nMistral-Small-3.1\nMistral-Small-4\nMagistral-Small-2509\nVoxtral\nGemma-3\nGemma-3n\nQwen2-VL\nQwen2.5-VL\nQwen3.5\nGLM-4.6V\nSmolVLM2\nLFM2-VL\nIntern-VL",
+    "text": "Gemma-4 (NEW)\nMllama\nLlama4\nPixtral\nLlava-1.5\nMistral-Small-3.1\nMistral-Small-4\nMagistral-Small-2509\nVoxtral\nGemma-3\nGemma-3n\nQwen2-VL\nQwen2.5-VL\nQwen3.5\nGLM-4.6V\nSmolVLM2\nLFM2-VL\nIntern-VL",
     "crumbs": [
       "How To Guides",
       "MultiModal / Vision Language Models (BETA)"
@@ -2261,7 +2261,7 @@
     "href": "docs/multimodal.html#supported-models",
     "title": "MultiModal / Vision Language Models (BETA)",
     "section": "",
-    "text": "Mllama\nLlama4\nPixtral\nLlava-1.5\nMistral-Small-3.1\nMistral-Small-4\nMagistral-Small-2509\nVoxtral\nGemma-3\nGemma-3n\nQwen2-VL\nQwen2.5-VL\nQwen3.5\nGLM-4.6V\nSmolVLM2\nLFM2-VL\nIntern-VL",
+    "text": "Gemma-4 (NEW)\nMllama\nLlama4\nPixtral\nLlava-1.5\nMistral-Small-3.1\nMistral-Small-4\nMagistral-Small-2509\nVoxtral\nGemma-3\nGemma-3n\nQwen2-VL\nQwen2.5-VL\nQwen3.5\nGLM-4.6V\nSmolVLM2\nLFM2-VL\nIntern-VL",
     "crumbs": [
       "How To Guides",
       "MultiModal / Vision Language Models (BETA)"
@@ -2272,7 +2272,7 @@
     "href": "docs/multimodal.html#usage",
     "title": "MultiModal / Vision Language Models (BETA)",
     "section": "Usage",
-    "text": "Usage\nMultimodal support is limited and doesn’t have full feature parity.\nHere are the hyperparams you’ll need to use to finetune a multimodal model.\nprocessor_type: AutoProcessor\n\nskip_prepare_dataset: true\nremove_unused_columns: false  # leave columns in place as they are needed to handle image embeddings during training\nsample_packing: false  # not yet supported with multimodal\n\nchat_template:  # see in next section if specified\n\n# example dataset\ndatasets:\n  - path: HuggingFaceH4/llava-instruct-mix-vsft\n    type: chat_template\n    split: train[:1%]\n\n# (optional) if doing lora, only finetune the Language model,\n# leave the vision model and vision tower frozen\n# load_in_8bit: true\nadapter: lora\nlora_target_modules: 'model.language_model.layers.[\\d]+.(mlp|cross_attn|self_attn).(up|down|gate|q|k|v|o)_proj'\n\n# (optional) if you want to resize images to a set size\nimage_size: 512\nimage_resize_algorithm: bilinear\nPlease see examples folder for full configs.\n\n\n\n\n\n\nTip\n\n\n\nSome of our chat_templates have been extended to support broader dataset types. This should not break any existing configs.\n\n\n\n\n\n\n\n\nNote\n\n\n\nAs of now, we do not truncate nor drop samples based on sequence_len as each arch has different ways to process non-text tokens. We are looking for help on this.\n\n\n\nMllama\nbase_model: meta-llama/Llama-3.2-11B-Vision-Instruct\n\nchat_template: llama3_2_vision\n\n\nLlama4\nbase_model: meta-llama/Llama-4-Scout-17B-16E-Instruct\n\nchat_template: llama4\n\n\nPixtral\nbase_model: mistralai/Pixtral-12B-2409\n\nchat_template: pixtral\n\n\nLlava-1.5\nbase_model: llava-hf/llava-1.5-7b-hf\n\nchat_template: llava\n\n\nMistral-Small-3.1\n\n\n\n\n\n\nTip\n\n\n\nPlease make sure to install vision lib via pip install 'mistral-common[opencv]==1.8.5'\n\n\nbase_model: mistralai/Mistral-Small-3.1-24B-Instruct-2503\n\n\nMistral-Small-4\nbase_model: mistralai/Mistral-Small-4-119B-2603\n\n\nMagistral-Small-2509\n\n\n\n\n\n\nTip\n\n\n\nPlease make sure to install vision lib via pip install 'mistral-common[opencv]==1.8.5'\n\n\nbase_model: mistralai/Magistral-Small-2509\n\n\nVoxtral\n\n\n\n\n\n\nTip\n\n\n\nPlease make sure to install audio lib via pip3 install librosa==0.11.0 'mistral_common[audio]==1.8.3'\n\n\nbase_model: mistralai/Voxtral-Mini-3B-2507\n\nprocessor_type: VoxtralProcessor\n\n\nGemma-3\n\n\n\n\n\n\nTip\n\n\n\nThe Gemma3-1B model is a text-only model, so please train as regular text model.\n\n\nFor multi-modal 4B/12B/27B models, use the following config:\nbase_model: google/gemma-3-4b-it\n\nchat_template: gemma3\n\n\nGemma-3n\n\n\n\n\n\n\nWarning\n\n\n\nThe model’s initial loss and grad norm will be very high. We suspect this to be due to the Conv in the vision layers.\n\n\n\n\n\n\n\n\nTip\n\n\n\nPlease make sure to install timm via pip3 install timm==1.0.17\n\n\nbase_model: google/gemma-3n-E2B-it\n\nchat_template: gemma3n\n\n\nQwen2-VL\nbase_model: Qwen/Qwen2-VL-7B-Instruct\n\nchat_template: qwen2_vl\n\n\nQwen2.5-VL\nbase_model: Qwen/Qwen2.5-VL-7B-Instruct\n\nchat_template: qwen2_vl  # same as qwen2-vl\n\n\nQwen3-VL\nbase_model: Qwen/Qwen3-VL-4B-Instruct\n\nchat_template: qwen2_vl  # same as qwen2-vl\n\n\nQwen3.5\nbase_model: Qwen/Qwen3.5-9B\n\nchat_template: qwen3_5\n\n\nGLM-4.6V\nBoth GLM-4.6V (106B MoE) and GLM-4.6V-Flash (9B) are supported.\n# GLM-4.6V (106B MoE version)\nbase_model: zai-org/GLM-4.6V\n\n# OR GLM-4.6V-Flash (9B version)\nbase_model: zai-org/GLM-4.6V-Flash\n\n\nSmolVLM2\n\n\n\n\n\n\nTip\n\n\n\nPlease make sure to install num2words via pip3 install num2words==0.5.14\n\n\nbase_model: HuggingFaceTB/SmolVLM2-500M-Video-Instruct\n\n\nLFM2-VL\n\n\n\n\n\n\nWarning\n\n\n\nPlease uninstall causal-conv1d via pip3 uninstall -y causal-conv1d\n\n\nbase_model: LiquidAI/LFM2-VL-450M\n\n\nIntern-VL\n\n\n\n\n\n\nTip\n\n\n\nPlease make sure to install timm via pip3 install timm==1.0.19\n\n\nbase_model: OpenGVLab/InternVL3_5-8B",
+    "text": "Usage\nMultimodal support is limited and doesn’t have full feature parity.\nHere are the hyperparams you’ll need to use to finetune a multimodal model.\nprocessor_type: AutoProcessor\n\nskip_prepare_dataset: true\nremove_unused_columns: false  # leave columns in place as they are needed to handle image embeddings during training\nsample_packing: false  # not yet supported with multimodal\n\nchat_template:  # see in next section if specified\n\n# example dataset\ndatasets:\n  - path: HuggingFaceH4/llava-instruct-mix-vsft\n    type: chat_template\n    split: train[:1%]\n\n# (optional) if doing lora, only finetune the Language model,\n# leave the vision model and vision tower frozen\n# load_in_8bit: true\nadapter: lora\nlora_target_modules: 'model.language_model.layers.[\\d]+.(mlp|cross_attn|self_attn).(up|down|gate|q|k|v|o)_proj'\n\n# (optional) if you want to resize images to a set size\nimage_size: 512\nimage_resize_algorithm: bilinear\nPlease see examples folder for full configs.\n\n\n\n\n\n\nTip\n\n\n\nSome of our chat_templates have been extended to support broader dataset types. This should not break any existing configs.\n\n\n\n\n\n\n\n\nNote\n\n\n\nAs of now, we do not truncate nor drop samples based on sequence_len as each arch has different ways to process non-text tokens. We are looking for help on this.\n\n\n\nMllama\nbase_model: meta-llama/Llama-3.2-11B-Vision-Instruct\n\nchat_template: llama3_2_vision\n\n\nLlama4\nbase_model: meta-llama/Llama-4-Scout-17B-16E-Instruct\n\nchat_template: llama4\n\n\nPixtral\nbase_model: mistralai/Pixtral-12B-2409\n\nchat_template: pixtral\n\n\nLlava-1.5\nbase_model: llava-hf/llava-1.5-7b-hf\n\nchat_template: llava\n\n\nMistral-Small-3.1\n\n\n\n\n\n\nTip\n\n\n\nPlease make sure to install vision lib via pip install 'mistral-common[opencv]==1.8.5'\n\n\nbase_model: mistralai/Mistral-Small-3.1-24B-Instruct-2503\n\n\nMistral-Small-4\nbase_model: mistralai/Mistral-Small-4-119B-2603\n\n\nMagistral-Small-2509\n\n\n\n\n\n\nTip\n\n\n\nPlease make sure to install vision lib via pip install 'mistral-common[opencv]==1.8.5'\n\n\nbase_model: mistralai/Magistral-Small-2509\n\n\nVoxtral\n\n\n\n\n\n\nTip\n\n\n\nPlease make sure to install audio lib via pip3 install librosa==0.11.0 'mistral_common[audio]==1.8.3'\n\n\nbase_model: mistralai/Voxtral-Mini-3B-2507\n\nprocessor_type: VoxtralProcessor\n\n\nGemma-4\nAll Gemma 4 variants (E2B, E4B, 26B-A4B, 31B) load as multimodal models even for text-only training.\nbase_model: google/gemma-4-E2B-it  # or E4B-it, 26B-A4B, 31B\n\nchat_template: gemma4\nfreeze_mm_modules: true  # freeze vision/audio encoders for text-only or vision LoRA\n\n# For the 26B-A4B MoE model, enable ScatterMoE and expert LoRA:\nplugins:\n  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin\n  - axolotl.integrations.kernels.KernelsPlugin\nuse_kernels: true\nuse_scattermoe: true\nexperts_implementation: scattermoe\n\nlora_target_modules: 'model.language_model.layers.[\\d]+.(_checkpoint_wrapped_module.)?(mlp|self_attn).(up|down|gate|q|k|v|o)_proj'\n\n# MoE expert LoRA (3D tensors, not nn.Linear) — only for 26B-A4B:\nlora_target_parameters:\n  - experts.gate_up_proj\n  - experts.down_proj\n\n\n\n\n\n\nWarning\n\n\n\nGemma 4 VLM training starts with high loss (~8-15). This is expected — see the training stability guide for details.\n\n\n\n\n\n\n\n\nTip\n\n\n\nFor DDP training, axolotl auto-detects Gemma4 and sets use_reentrant=False and ddp_find_unused_parameters=True. However, when activation_offloading: true, ddp_find_unused_parameters is skipped (checkpoint wrappers conflict with it); use freeze_mm_modules: true instead to handle unused vision/audio params. For FSDP2, use fsdp_transformer_layer_cls_to_wrap: Gemma4TextDecoderLayer.\n\n\n\n\nGemma-3\n\n\n\n\n\n\nTip\n\n\n\nThe Gemma3-1B model is a text-only model, so please train as regular text model.\n\n\nFor multi-modal 4B/12B/27B models, use the following config:\nbase_model: google/gemma-3-4b-it\n\nchat_template: gemma3\n\n\nGemma-3n\n\n\n\n\n\n\nWarning\n\n\n\nThe model’s initial loss and grad norm will be very high. We suspect this to be due to the Conv in the vision layers.\n\n\n\n\n\n\n\n\nTip\n\n\n\nPlease make sure to install timm via pip3 install timm==1.0.17\n\n\nbase_model: google/gemma-3n-E2B-it\n\nchat_template: gemma3n\n\n\nQwen2-VL\nbase_model: Qwen/Qwen2-VL-7B-Instruct\n\nchat_template: qwen2_vl\n\n\nQwen2.5-VL\nbase_model: Qwen/Qwen2.5-VL-7B-Instruct\n\nchat_template: qwen2_vl  # same as qwen2-vl\n\n\nQwen3-VL\nbase_model: Qwen/Qwen3-VL-4B-Instruct\n\nchat_template: qwen2_vl  # same as qwen2-vl\n\n\nQwen3.5\nbase_model: Qwen/Qwen3.5-9B\n\nchat_template: qwen3_5\n\n\nGLM-4.6V\nBoth GLM-4.6V (106B MoE) and GLM-4.6V-Flash (9B) are supported.\n# GLM-4.6V (106B MoE version)\nbase_model: zai-org/GLM-4.6V\n\n# OR GLM-4.6V-Flash (9B version)\nbase_model: zai-org/GLM-4.6V-Flash\n\n\nSmolVLM2\n\n\n\n\n\n\nTip\n\n\n\nPlease make sure to install num2words via pip3 install num2words==0.5.14\n\n\nbase_model: HuggingFaceTB/SmolVLM2-500M-Video-Instruct\n\n\nLFM2-VL\n\n\n\n\n\n\nWarning\n\n\n\nPlease uninstall causal-conv1d via pip3 uninstall -y causal-conv1d\n\n\nbase_model: LiquidAI/LFM2-VL-450M\n\n\nIntern-VL\n\n\n\n\n\n\nTip\n\n\n\nPlease make sure to install timm via pip3 install timm==1.0.19\n\n\nbase_model: OpenGVLab/InternVL3_5-8B",
     "crumbs": [
       "How To Guides",
       "MultiModal / Vision Language Models (BETA)"
@@ -3060,14 +3060,28 @@
     "href": "docs/agents/model_architectures.html",
     "title": "Model Architectures — Agent Reference",
     "section": "",
-    "text": "Model-specific quirks, required settings, and known issues. Check this before debugging training failures on specific model families.\n\n\nModels: google/gemma-4-26B-A4B (MoE), google/gemma-4-31B (dense), google/gemma-4-E2B, google/gemma-4-E4B\nArchitecture: Multimodal wrapper (Gemma4ForConditionalGeneration) over a text backbone (Gemma4TextModel), with optional vision/audio encoders. All Gemma4 HF repos have model_type: \"gemma4\" — even text-only variants load as multimodal with a vision tower.\n\n\n# Always needed for Gemma4:\nfreeze_mm_modules: true          # Freeze vision/audio encoders for text-only training\ngradient_checkpointing_kwargs:\n  use_reentrant: false           # Shared per-layer norms cause \"marked ready twice\" with reentrant\n\n# LoRA target — restrict to language model only (DO NOT use lora_target_linear: true):\nlora_target_modules: 'model.language_model.layers.[\\d]+.(_checkpoint_wrapped_module.)?(mlp|self_attn).(up|down|gate|q|k|v|o)_proj'\n\n\n\nAxolotl auto-detects Gemma4 and applies:\n- use_reentrant: false for gradient checkpointing\n- ddp_find_unused_parameters: true for DDP (skipped when activation_offloading: true)\n\n\n\n\n\n\n\n\n\n\n\nStrategy\nWorks?\nNotes\n\n\n\n\nDDP\nYes\nAuto-sets ddp_find_unused_parameters=True\n\n\nDDP + activation_offloading\nYes\nfind_unused_parameters is skipped (conflicts with checkpoint wrappers)\n\n\nFSDP1\nNo\nOOM during dequantization/sharding with QLoRA\n\n\nFSDP2\nYes\nUse Gemma4TextDecoderLayer (not Gemma4DecoderLayer) as wrap class\n\n\nFSDP2 + activation_offloading\nYes\nLowest VRAM (~26 GiB/GPU for 26B-A4B)\n\n\n\nFSDP2 config:\nfsdp:\n  - full_shard\n  - auto_wrap\nfsdp_config:\n  fsdp_version: 2\n  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP\n  fsdp_transformer_layer_cls_to_wrap: Gemma4TextDecoderLayer\n\n\n\n\nenable_moe_block: true, 256 experts, top-k routing\nNo separate SparseMoeBlock — MoE is embedded in each decoder layer\nExpert LoRA targets 3D parameter tensors:\nlora_target_parameters:\n  - experts.gate_up_proj\n  - experts.down_proj\nScatterMoE kernel acceleration:\nplugins:\n  - axolotl.integrations.kernels.KernelsPlugin\nuse_kernels: true\nuse_scattermoe: true\nexperts_implementation: scattermoe\n\n\n\n\n\n\n\n\n\n\n\n\nSymptom\nCause\nFix\n\n\n\n\nmm_token_type_ids is required in DDP\nmodel.config not accessible through DDP wrapper\nAlready fixed — unwrap_model() in compute_loss and prediction_step\n\n\nmarked a variable ready twice in DDP\nddp_find_unused_parameters=True + activation_offloading checkpoint wrappers\nAuto-handled — find_unused_parameters is skipped when activation_offloading: true\n\n\nLoss ~12 instead of ~0.5\nUsing lora_target_linear: true (applies LoRA to vision/audio modules)\nUse the regex lora_target_modules pattern instead\n\n\nFSDP2 Could not find Gemma4AudioLayer\nAuto-wrap detects _no_split_modules including audio layers that don’t exist\nExplicitly set fsdp_transformer_layer_cls_to_wrap: Gemma4TextDecoderLayer\n\n\nGemma4ClippableLinear not supported by PEFT\nVision tower uses a non-standard linear wrapper\nAxolotl patches this automatically via _patch_peft_clippable_linear()\n\n\n\n\n\n\nThese have hidden_size_per_layer_input: 256 (per-layer input embeddings) and attention_k_eq_v: False. Known issue: loss starts higher than expected (~12 vs ~0.5 for 26B). Root cause under investigation — may be related to the per-layer input mechanism or the Gemma4ForConditionalGeneration loss computation.\n\n\n\n\nModels: google/gemma-3-*\n\nddp_find_unused_parameters: true needed (multimodal unused params)\nuse_reentrant: false recommended\nAttention mask must be dropped for sample packing (handled automatically)\nMulti-GPU test currently skipped (tests/e2e/multigpu/test_gemma3.py)\n\n\n\n\nModels: Qwen/Qwen3.5-35B-A3B\n\nHybrid architecture: DeltaNet linear attention (30 layers) + full attention (10 layers)\n256 experts, 8 active per token\nKnown weight scale drift in late DeltaNet layers (36-38) due to AdamW + rare expert interaction\nFix: normalize_weight_scales config to detect and rescale outliers:\nnormalize_weight_scales:\n  - name_pattern: 'linear_attn\\.conv1d\\.weight'\n    threshold: 1.3\n\n\n\n\n\nlora_target_linear: true with multimodal MoE models will apply LoRA to ALL linear modules including vision/audio encoders — use regex lora_target_modules to restrict to language model only\nRare experts get larger effective learning rate from AdamW (small second-moment estimates) — can cause weight drift in recurrent/SSM components. Use normalize_weight_scales with dry_run: true to detect.\nFor ScatterMoE kernel support, set experts_implementation: scattermoe and add the KernelsPlugin"
+    "text": "Model-specific quirks, required settings, and known issues. Check this before debugging training failures on specific model families.\n\n\nAll VLM configs require these four lines:\nprocessor_type: AutoProcessor\nskip_prepare_dataset: true\nremove_unused_columns: false\nsample_packing: false\nDecision tree for VLM config:\nIs the model multimodal (has vision/audio encoder)?\n  ├─ YES: Add `freeze_mm_modules: true` if training text only\n  │       Add `chat_template: &lt;model_template&gt;` (e.g. gemma4, qwen3_5, gemma3)\n  │       LoRA: use regex `lora_target_modules` to restrict to language model\n  └─ NO: Train as a regular text model\n\nIs the model MoE (e.g. Gemma4 26B-A4B, Qwen3.5 35B-A3B)?\n  ├─ YES: Add `lora_target_parameters` for expert LoRA\n  │       Consider ScatterMoE kernels (see Plugins section)\n  └─ NO: Standard LoRA config\n\n\n\n\n\nComputes loss from hidden states + lm_head weight without materializing the full logits tensor, saving significant VRAM. Install if not already present:\nuv pip install \"cut-cross-entropy[transformers] @ git+https://github.com/axolotl-ai-cloud/ml-cross-entropy.git@main\"\nplugins:\n  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin\n\n\n\nFuses expert + LoRA computation into a single kernel for MoE models. Significant speedup for models with many experts.\nplugins:\n  - axolotl.integrations.kernels.KernelsPlugin\nuse_kernels: true\nuse_scattermoe: true\nexperts_implementation: scattermoe\n\n# Expert LoRA targets (3D parameter tensors, not nn.Linear):\nlora_target_parameters:\n  - experts.gate_up_proj\n  - experts.down_proj\nSupported: Gemma4 (gemma4_text), Mixtral, Qwen MoE variants. The plugin auto-detects model type and routing function. Without ScatterMoE, expert LoRA still works but runs base expert matmul and LoRA as separate operations.\n\n\n\n\nModels: google/gemma-4-26B-A4B (MoE), google/gemma-4-31B (dense), google/gemma-4-E2B, google/gemma-4-E4B\nArchitecture: Multimodal wrapper (Gemma4ForConditionalGeneration) over a text backbone (Gemma4TextModel), with optional vision/audio encoders. All Gemma4 HF repos have model_type: \"gemma4\" — even text-only variants load as multimodal with a vision tower.\n\n\n# Always needed for Gemma4:\nfreeze_mm_modules: true          # Freeze vision/audio encoders for text-only training\ngradient_checkpointing_kwargs:\n  use_reentrant: false           # Shared per-layer norms cause \"marked ready twice\" with reentrant\n\n# LoRA target — restrict to language model only (DO NOT use lora_target_linear: true):\nlora_target_modules: 'model.language_model.layers.[\\d]+.(_checkpoint_wrapped_module.)?(mlp|self_attn).(up|down|gate|q|k|v|o)_proj'\n\n\n\nAxolotl auto-detects Gemma4 and applies:\n- use_reentrant: false for gradient checkpointing\n- ddp_find_unused_parameters: true for DDP (skipped when activation_offloading: true)\n\n\n\n\n\n\n\n\n\n\n\nStrategy\nWorks?\nNotes\n\n\n\n\nDDP\nYes\nAuto-sets ddp_find_unused_parameters=True\n\n\nDDP + activation_offloading\nYes\nfind_unused_parameters is skipped (conflicts with checkpoint wrappers)\n\n\nFSDP1\nNo\nOOM during dequantization/sharding with QLoRA\n\n\nFSDP2\nYes\nUse Gemma4TextDecoderLayer (not Gemma4DecoderLayer) as wrap class\n\n\nFSDP2 + activation_offloading\nYes\nLowest VRAM (~26 GiB/GPU for 26B-A4B)\n\n\n\nFSDP2 config:\nfsdp:\n  - full_shard\n  - auto_wrap\nfsdp_config:\n  fsdp_version: 2\n  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP\n  fsdp_transformer_layer_cls_to_wrap: Gemma4TextDecoderLayer\n\n\n\n\nenable_moe_block: true, 256 experts, top-k routing\nNo separate SparseMoeBlock — MoE is embedded in each decoder layer\nExpert LoRA targets 3D parameter tensors:\nlora_target_parameters:\n  - experts.gate_up_proj\n  - experts.down_proj\nScatterMoE kernel acceleration:\nplugins:\n  - axolotl.integrations.kernels.KernelsPlugin\nuse_kernels: true\nuse_scattermoe: true\nexperts_implementation: scattermoe\n\n\n\n\nAll Gemma4 models load as Gemma4ForConditionalGeneration with a vision tower. No custom ProcessingStrategy needed — the base class auto-detects the image token.\nbase_model: google/gemma-4-E2B-it   # or E4B-it, 26B-A4B\nprocessor_type: AutoProcessor\nfreeze_mm_modules: true\nchat_template: gemma4\n\nskip_prepare_dataset: true\nremove_unused_columns: false\nsample_packing: false\nA starting VLM loss of ~8-15 is typical. In most runs, loss converges below 1.0 within ~30-50 steps, though results may vary across configurations.\nFor the 26B-A4B MoE variant with ScatterMoE + expert LoRA + CCE, add:\nplugins:\n  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin\n  - axolotl.integrations.kernels.KernelsPlugin\nuse_kernels: true\nuse_scattermoe: true\nexperts_implementation: scattermoe\nlora_target_parameters:\n  - experts.gate_up_proj\n  - experts.down_proj\n\n\n\n\n\n\n\n\n\n\n\nSymptom\nCause\nFix\n\n\n\n\nmm_token_type_ids is required in DDP\nmodel.config not accessible through DDP wrapper\nAlready fixed — unwrap_model() in compute_loss and prediction_step\n\n\nmarked a variable ready twice in DDP\nddp_find_unused_parameters=True + activation_offloading checkpoint wrappers\nAuto-handled — find_unused_parameters is skipped when activation_offloading: true\n\n\nLoss ~12 instead of ~0.5\nUsing lora_target_linear: true (applies LoRA to vision/audio modules)\nUse the regex lora_target_modules pattern instead\n\n\nFSDP2 Could not find Gemma4AudioLayer\nAuto-wrap detects _no_split_modules including audio layers that don’t exist\nExplicitly set fsdp_transformer_layer_cls_to_wrap: Gemma4TextDecoderLayer\n\n\nGemma4ClippableLinear not supported by PEFT\nVision tower uses a non-standard linear wrapper\nAxolotl patches this automatically via _patch_peft_clippable_linear()\n\n\n\n\n\n\nThese have hidden_size_per_layer_input: 256 (per-layer input embeddings) and attention_k_eq_v: False. Known issue: loss starts higher than expected (~12 vs ~0.5 for 26B). Root cause under investigation — may be related to the per-layer input mechanism or the Gemma4ForConditionalGeneration loss computation.\n\n\n\n\nModels: google/gemma-3-*\n\nddp_find_unused_parameters: true needed (multimodal unused params)\nuse_reentrant: false recommended\nAttention mask must be dropped for sample packing (handled automatically)\nMulti-GPU test currently skipped (tests/e2e/multigpu/test_gemma3.py)\n\n\n\n\nModels: Qwen/Qwen3.5-35B-A3B\n\nHybrid architecture: DeltaNet linear attention (30 layers) + full attention (10 layers)\n256 experts, 8 active per token\nKnown weight scale drift in late DeltaNet layers (36-38) due to AdamW + rare expert interaction\nFix: normalize_weight_scales config to detect and rescale outliers:\nnormalize_weight_scales:\n  - name_pattern: 'linear_attn\\.conv1d\\.weight'\n    threshold: 1.3\n\n\n\n\n\nlora_target_linear: true with multimodal MoE models will apply LoRA to ALL linear modules including vision/audio encoders — use regex lora_target_modules to restrict to language model only\nRare experts get larger effective learning rate from AdamW (small second-moment estimates) — can cause weight drift in recurrent/SSM components. Use normalize_weight_scales with dry_run: true to detect.\nFor ScatterMoE kernel support, set experts_implementation: scattermoe and add the KernelsPlugin"
+  },
+  {
+    "objectID": "docs/agents/model_architectures.html#vlm-vision-language-model-quick-start",
+    "href": "docs/agents/model_architectures.html#vlm-vision-language-model-quick-start",
+    "title": "Model Architectures — Agent Reference",
+    "section": "",
+    "text": "All VLM configs require these four lines:\nprocessor_type: AutoProcessor\nskip_prepare_dataset: true\nremove_unused_columns: false\nsample_packing: false\nDecision tree for VLM config:\nIs the model multimodal (has vision/audio encoder)?\n  ├─ YES: Add `freeze_mm_modules: true` if training text only\n  │       Add `chat_template: &lt;model_template&gt;` (e.g. gemma4, qwen3_5, gemma3)\n  │       LoRA: use regex `lora_target_modules` to restrict to language model\n  └─ NO: Train as a regular text model\n\nIs the model MoE (e.g. Gemma4 26B-A4B, Qwen3.5 35B-A3B)?\n  ├─ YES: Add `lora_target_parameters` for expert LoRA\n  │       Consider ScatterMoE kernels (see Plugins section)\n  └─ NO: Standard LoRA config"
+  },
+  {
+    "objectID": "docs/agents/model_architectures.html#plugins-optimizations",
+    "href": "docs/agents/model_architectures.html#plugins-optimizations",
+    "title": "Model Architectures — Agent Reference",
+    "section": "",
+    "text": "Computes loss from hidden states + lm_head weight without materializing the full logits tensor, saving significant VRAM. Install if not already present:\nuv pip install \"cut-cross-entropy[transformers] @ git+https://github.com/axolotl-ai-cloud/ml-cross-entropy.git@main\"\nplugins:\n  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin\n\n\n\nFuses expert + LoRA computation into a single kernel for MoE models. Significant speedup for models with many experts.\nplugins:\n  - axolotl.integrations.kernels.KernelsPlugin\nuse_kernels: true\nuse_scattermoe: true\nexperts_implementation: scattermoe\n\n# Expert LoRA targets (3D parameter tensors, not nn.Linear):\nlora_target_parameters:\n  - experts.gate_up_proj\n  - experts.down_proj\nSupported: Gemma4 (gemma4_text), Mixtral, Qwen MoE variants. The plugin auto-detects model type and routing function. Without ScatterMoE, expert LoRA still works but runs base expert matmul and LoRA as separate operations."
   },
   {
     "objectID": "docs/agents/model_architectures.html#gemma-4",
     "href": "docs/agents/model_architectures.html#gemma-4",
     "title": "Model Architectures — Agent Reference",
     "section": "",
-    "text": "Models: google/gemma-4-26B-A4B (MoE), google/gemma-4-31B (dense), google/gemma-4-E2B, google/gemma-4-E4B\nArchitecture: Multimodal wrapper (Gemma4ForConditionalGeneration) over a text backbone (Gemma4TextModel), with optional vision/audio encoders. All Gemma4 HF repos have model_type: \"gemma4\" — even text-only variants load as multimodal with a vision tower.\n\n\n# Always needed for Gemma4:\nfreeze_mm_modules: true          # Freeze vision/audio encoders for text-only training\ngradient_checkpointing_kwargs:\n  use_reentrant: false           # Shared per-layer norms cause \"marked ready twice\" with reentrant\n\n# LoRA target — restrict to language model only (DO NOT use lora_target_linear: true):\nlora_target_modules: 'model.language_model.layers.[\\d]+.(_checkpoint_wrapped_module.)?(mlp|self_attn).(up|down|gate|q|k|v|o)_proj'\n\n\n\nAxolotl auto-detects Gemma4 and applies:\n- use_reentrant: false for gradient checkpointing\n- ddp_find_unused_parameters: true for DDP (skipped when activation_offloading: true)\n\n\n\n\n\n\n\n\n\n\n\nStrategy\nWorks?\nNotes\n\n\n\n\nDDP\nYes\nAuto-sets ddp_find_unused_parameters=True\n\n\nDDP + activation_offloading\nYes\nfind_unused_parameters is skipped (conflicts with checkpoint wrappers)\n\n\nFSDP1\nNo\nOOM during dequantization/sharding with QLoRA\n\n\nFSDP2\nYes\nUse Gemma4TextDecoderLayer (not Gemma4DecoderLayer) as wrap class\n\n\nFSDP2 + activation_offloading\nYes\nLowest VRAM (~26 GiB/GPU for 26B-A4B)\n\n\n\nFSDP2 config:\nfsdp:\n  - full_shard\n  - auto_wrap\nfsdp_config:\n  fsdp_version: 2\n  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP\n  fsdp_transformer_layer_cls_to_wrap: Gemma4TextDecoderLayer\n\n\n\n\nenable_moe_block: true, 256 experts, top-k routing\nNo separate SparseMoeBlock — MoE is embedded in each decoder layer\nExpert LoRA targets 3D parameter tensors:\nlora_target_parameters:\n  - experts.gate_up_proj\n  - experts.down_proj\nScatterMoE kernel acceleration:\nplugins:\n  - axolotl.integrations.kernels.KernelsPlugin\nuse_kernels: true\nuse_scattermoe: true\nexperts_implementation: scattermoe\n\n\n\n\n\n\n\n\n\n\n\n\nSymptom\nCause\nFix\n\n\n\n\nmm_token_type_ids is required in DDP\nmodel.config not accessible through DDP wrapper\nAlready fixed — unwrap_model() in compute_loss and prediction_step\n\n\nmarked a variable ready twice in DDP\nddp_find_unused_parameters=True + activation_offloading checkpoint wrappers\nAuto-handled — find_unused_parameters is skipped when activation_offloading: true\n\n\nLoss ~12 instead of ~0.5\nUsing lora_target_linear: true (applies LoRA to vision/audio modules)\nUse the regex lora_target_modules pattern instead\n\n\nFSDP2 Could not find Gemma4AudioLayer\nAuto-wrap detects _no_split_modules including audio layers that don’t exist\nExplicitly set fsdp_transformer_layer_cls_to_wrap: Gemma4TextDecoderLayer\n\n\nGemma4ClippableLinear not supported by PEFT\nVision tower uses a non-standard linear wrapper\nAxolotl patches this automatically via _patch_peft_clippable_linear()\n\n\n\n\n\n\nThese have hidden_size_per_layer_input: 256 (per-layer input embeddings) and attention_k_eq_v: False. Known issue: loss starts higher than expected (~12 vs ~0.5 for 26B). Root cause under investigation — may be related to the per-layer input mechanism or the Gemma4ForConditionalGeneration loss computation."
+    "text": "Models: google/gemma-4-26B-A4B (MoE), google/gemma-4-31B (dense), google/gemma-4-E2B, google/gemma-4-E4B\nArchitecture: Multimodal wrapper (Gemma4ForConditionalGeneration) over a text backbone (Gemma4TextModel), with optional vision/audio encoders. All Gemma4 HF repos have model_type: \"gemma4\" — even text-only variants load as multimodal with a vision tower.\n\n\n# Always needed for Gemma4:\nfreeze_mm_modules: true          # Freeze vision/audio encoders for text-only training\ngradient_checkpointing_kwargs:\n  use_reentrant: false           # Shared per-layer norms cause \"marked ready twice\" with reentrant\n\n# LoRA target — restrict to language model only (DO NOT use lora_target_linear: true):\nlora_target_modules: 'model.language_model.layers.[\\d]+.(_checkpoint_wrapped_module.)?(mlp|self_attn).(up|down|gate|q|k|v|o)_proj'\n\n\n\nAxolotl auto-detects Gemma4 and applies:\n- use_reentrant: false for gradient checkpointing\n- ddp_find_unused_parameters: true for DDP (skipped when activation_offloading: true)\n\n\n\n\n\n\n\n\n\n\n\nStrategy\nWorks?\nNotes\n\n\n\n\nDDP\nYes\nAuto-sets ddp_find_unused_parameters=True\n\n\nDDP + activation_offloading\nYes\nfind_unused_parameters is skipped (conflicts with checkpoint wrappers)\n\n\nFSDP1\nNo\nOOM during dequantization/sharding with QLoRA\n\n\nFSDP2\nYes\nUse Gemma4TextDecoderLayer (not Gemma4DecoderLayer) as wrap class\n\n\nFSDP2 + activation_offloading\nYes\nLowest VRAM (~26 GiB/GPU for 26B-A4B)\n\n\n\nFSDP2 config:\nfsdp:\n  - full_shard\n  - auto_wrap\nfsdp_config:\n  fsdp_version: 2\n  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP\n  fsdp_transformer_layer_cls_to_wrap: Gemma4TextDecoderLayer\n\n\n\n\nenable_moe_block: true, 256 experts, top-k routing\nNo separate SparseMoeBlock — MoE is embedded in each decoder layer\nExpert LoRA targets 3D parameter tensors:\nlora_target_parameters:\n  - experts.gate_up_proj\n  - experts.down_proj\nScatterMoE kernel acceleration:\nplugins:\n  - axolotl.integrations.kernels.KernelsPlugin\nuse_kernels: true\nuse_scattermoe: true\nexperts_implementation: scattermoe\n\n\n\n\nAll Gemma4 models load as Gemma4ForConditionalGeneration with a vision tower. No custom ProcessingStrategy needed — the base class auto-detects the image token.\nbase_model: google/gemma-4-E2B-it   # or E4B-it, 26B-A4B\nprocessor_type: AutoProcessor\nfreeze_mm_modules: true\nchat_template: gemma4\n\nskip_prepare_dataset: true\nremove_unused_columns: false\nsample_packing: false\nA starting VLM loss of ~8-15 is typical. In most runs, loss converges below 1.0 within ~30-50 steps, though results may vary across configurations.\nFor the 26B-A4B MoE variant with ScatterMoE + expert LoRA + CCE, add:\nplugins:\n  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin\n  - axolotl.integrations.kernels.KernelsPlugin\nuse_kernels: true\nuse_scattermoe: true\nexperts_implementation: scattermoe\nlora_target_parameters:\n  - experts.gate_up_proj\n  - experts.down_proj\n\n\n\n\n\n\n\n\n\n\n\nSymptom\nCause\nFix\n\n\n\n\nmm_token_type_ids is required in DDP\nmodel.config not accessible through DDP wrapper\nAlready fixed — unwrap_model() in compute_loss and prediction_step\n\n\nmarked a variable ready twice in DDP\nddp_find_unused_parameters=True + activation_offloading checkpoint wrappers\nAuto-handled — find_unused_parameters is skipped when activation_offloading: true\n\n\nLoss ~12 instead of ~0.5\nUsing lora_target_linear: true (applies LoRA to vision/audio modules)\nUse the regex lora_target_modules pattern instead\n\n\nFSDP2 Could not find Gemma4AudioLayer\nAuto-wrap detects _no_split_modules including audio layers that don’t exist\nExplicitly set fsdp_transformer_layer_cls_to_wrap: Gemma4TextDecoderLayer\n\n\nGemma4ClippableLinear not supported by PEFT\nVision tower uses a non-standard linear wrapper\nAxolotl patches this automatically via _patch_peft_clippable_linear()\n\n\n\n\n\n\nThese have hidden_size_per_layer_input: 256 (per-layer input embeddings) and attention_k_eq_v: False. Known issue: loss starts higher than expected (~12 vs ~0.5 for 26B). Root cause under investigation — may be related to the per-layer input mechanism or the Gemma4ForConditionalGeneration loss computation."
   },
   {
     "objectID": "docs/agents/model_architectures.html#gemma-3",
@@ -3532,7 +3546,7 @@
     "href": "docs/agents/sft.html",
     "title": "SFT — Agent Reference",
     "section": "",
-    "text": "Supervised fine-tuning pipeline reference. For config templates and dataset format examples, see getting-started.qmd and dataset-formats/.\n\n\nYAML Config → axolotl train config.yaml\n\n  1. Load base model (+ quantization if QLoRA/8-bit)\n  2. Apply adapter layers (LoRA/QLoRA) if configured\n  3. Load + tokenize dataset(s)\n     - Apply prompt template (chat_template / alpaca / custom)\n     - Mask inputs (train_on_inputs: false)\n     - Pack samples into sequences (sample_packing: true)\n  4. Training loop (HuggingFace Trainer)\n     - forward → loss → backward → optimizer step → lr scheduler step\n  5. Save model / adapter weights + tokenizer\n\nMulti-GPU: FSDP or DeepSpeed shards model across GPUs automatically.\n\n\n\n\nA YAML config — model, dataset(s), adapter settings, hyperparameters\nA dataset — HuggingFace Hub, local JSONL/JSON/Parquet, or S3/GCS path\n(Optional) A custom prompt strategy — for non-standard dataset formats\n\nNo external server processes needed (unlike GRPO which requires vLLM).\n\n\n\nIs your data in chat/message format?\n  ├─ YES: OpenAI message format (role/content)?\n  │   ├─ YES ──────────────────────&gt; type: chat_template  (recommended)\n  │   └─ NO (custom field names) ──&gt; type: chat_template + message_property_mappings\n  └─ NO: Instruction/response pairs?\n      ├─ YES ──&gt; type: alpaca       (instruction, input, output)\n      └─ NO: Raw text?\n          ├─ YES with segments ─────&gt; type: input_output  (template-free masking)\n          └─ YES continuous ────────&gt; type: completion     (pretraining-style)\nFull format specs: dataset-formats/\n\n\n\n\n\n\n\n\n\n\n\n\n\nModel Size\nLoRA\nQLoRA (4-bit)\nFull Fine-Tune\nVRAM (approx)\n\n\n\n\n1-3B\nPreferred\nLow-budget option\nSingle GPU OK\n8-16 GB (LoRA)\n\n\n7-8B\nPreferred\nGood balance\nNeeds multi-GPU\n16-24 GB (LoRA)\n\n\n13-14B\nPreferred\nGood balance\nMulti-GPU required\n24-40 GB (LoRA)\n\n\n30-70B\nLoRA or QLoRA\nPreferred for single GPU\nMulti-node\n40-80 GB (QLoRA)\n\n\n\n\n\n\n\n\n\nParameter\nLoRA\nQLoRA\nFull FT\n\n\n\n\nlearning_rate\n1e-4 to 3e-4\n1e-4 to 3e-4\n1e-5 to 5e-5\n\n\nlora_r\n16-64\n16-64\nN/A\n\n\nlora_alpha\n1-2x lora_r\n1-2x lora_r\nN/A\n\n\nmicro_batch_size\n2-8\n2-4\n1-2\n\n\ngradient_accumulation_steps\n2-8\n4-16\n4-16\n\n\nnum_epochs\n1-3\n1-3\n1-3\n\n\noptimizer\nadamw_8bit\nadamw_bnb_8bit\nadamw_torch_fused\n\n\n\nEffective batch = micro_batch * grad_accum * num_gpus. Lower LR for larger models.\n\n\n\n\n\n\n\n\n\n\n\nMetric\nHealthy\nProblem\n\n\n\n\ntrain_loss\nDecreasing, starting ~2-4 for chat models\nFlat or increasing from step 1 — data or LR issue\n\n\neval_loss\nDecreasing, tracks train_loss\nIncreasing while train_loss decreases — overfitting\n\n\ngrad_norm\n0.1-10, relatively stable\nSpikes &gt;100 — instability. 0.0 — frozen weights\n\n\nlearning_rate\nFollows scheduler curve\nFlat or NaN — config issue\n\n\n\nWatch for: loss never decreasing (check train_on_inputs, dataset, LR), loss goes to 0 quickly (overfitting), eval_loss diverging (reduce epochs, add regularization). See training_stability.qmd.\n\n\n\n\n\n\n\n\n\n\nIssue\nFix\n\n\n\n\nOOM during training\nReduce micro_batch_size, enable gradient_checkpointing, reduce sequence_len\n\n\nsample_packing + SDPA + bf16 = 0.0 loss\nUse flash_attention: true or disable sample_packing\n\n\nMissing chat template error\nSet chat_template: chatml explicitly\n\n\nLabel masking wrong\nRun axolotl preprocess config.yaml --debug and inspect labels\n\n\nLoss NaN\nUse bf16: auto, lower LR, check data for empty samples\n\n\nTokenizer pad token / infinite loss\nSet special_tokens: pad_token: \"&lt;\\|end_of_text\\|&gt;\"\n\n\nFSDP save hangs\nUse fsdp_state_dict_type: FULL_STATE_DICT\n\n\nDeepSpeed CheckpointError\nSet use_reentrant: true in gradient_checkpointing_kwargs\n\n\n\nFull troubleshooting: training_stability.qmd, debugging.qmd\n\n\n\nsrc/axolotl/\n  cli/train.py                     # Entry point for `axolotl train`\n  cli/preprocess.py                # Entry point for `axolotl preprocess`\n  core/builders/causal.py          # HFCausalTrainerBuilder — wires config → SFT trainer\n  core/trainers/base.py            # AxolotlTrainer — base trainer class\n  core/trainers/mixins/            # Packing, optimizer, scheduler, checkpoints\n  prompt_strategies/               # Format handlers: chat_template, alpaca, completion, input_output\n  utils/schemas/config.py          # AxolotlInputConfig — main config schema\n  utils/schemas/datasets.py        # SFTDataset, DatasetConfig\n  utils/schemas/peft.py            # LoraConfig — LoRA parameters\n  integrations/liger/              # Liger kernel plugin\n\nexamples/llama-3/                  # LoRA, QLoRA, full FT example configs\ndocs/getting-started.qmd           # Quickstart with config templates\ndocs/optimizations.qmd             # Flash attention, gradient checkpointing, sample packing\ndocs/multi-gpu.qmd                 # FSDP and DeepSpeed setup"
+    "text": "Supervised fine-tuning pipeline reference. For config templates and dataset format examples, see getting-started.qmd and dataset-formats/.\n\n\nYAML Config → axolotl train config.yaml\n\n  1. Load base model (+ quantization if QLoRA/8-bit)\n  2. Apply adapter layers (LoRA/QLoRA) if configured\n  3. Load + tokenize dataset(s)\n     - Apply prompt template (chat_template / alpaca / custom)\n     - Mask inputs (train_on_inputs: false)\n     - Pack samples into sequences (sample_packing: true)\n  4. Training loop (HuggingFace Trainer)\n     - forward → loss → backward → optimizer step → lr scheduler step\n  5. Save model / adapter weights + tokenizer\n\nMulti-GPU: FSDP or DeepSpeed shards model across GPUs automatically.\n\n\n\n\nA YAML config — model, dataset(s), adapter settings, hyperparameters\nA dataset — HuggingFace Hub, local JSONL/JSON/Parquet, or S3/GCS path\n(Optional) A custom prompt strategy — for non-standard dataset formats\n\nNo external server processes needed (unlike GRPO which requires vLLM).\n\n\n\nIs your data in chat/message format?\n  ├─ YES: OpenAI message format (role/content)?\n  │   ├─ YES ──────────────────────&gt; type: chat_template  (recommended)\n  │   └─ NO (custom field names) ──&gt; type: chat_template + message_property_mappings\n  └─ NO: Instruction/response pairs?\n      ├─ YES ──&gt; type: alpaca       (instruction, input, output)\n      └─ NO: Raw text?\n          ├─ YES with segments ─────&gt; type: input_output  (template-free masking)\n          └─ YES continuous ────────&gt; type: completion     (pretraining-style)\nFull format specs: dataset-formats/\n\n\n\n\n\n\n\n\n\n\n\n\n\nModel Size\nLoRA\nQLoRA (4-bit)\nFull Fine-Tune\nVRAM (approx)\n\n\n\n\n1-3B\nPreferred\nLow-budget option\nSingle GPU OK\n8-16 GB (LoRA)\n\n\n7-8B\nPreferred\nGood balance\nNeeds multi-GPU\n16-24 GB (LoRA)\n\n\n13-14B\nPreferred\nGood balance\nMulti-GPU required\n24-40 GB (LoRA)\n\n\n30-70B\nLoRA or QLoRA\nPreferred for single GPU\nMulti-node\n40-80 GB (QLoRA)\n\n\n\n\n\n\n\n\n\nParameter\nLoRA\nQLoRA\nFull FT\n\n\n\n\nlearning_rate\n1e-4 to 3e-4\n1e-4 to 3e-4\n1e-5 to 5e-5\n\n\nlora_r\n16-64\n16-64\nN/A\n\n\nlora_alpha\n1-2x lora_r\n1-2x lora_r\nN/A\n\n\nmicro_batch_size\n2-8\n2-4\n1-2\n\n\ngradient_accumulation_steps\n2-8\n4-16\n4-16\n\n\nnum_epochs\n1-3\n1-3\n1-3\n\n\noptimizer\nadamw_8bit\nadamw_bnb_8bit\nadamw_torch_fused\n\n\n\nEffective batch = micro_batch * grad_accum * num_gpus. Lower LR for larger models.\n\n\n\n\n\n\n\n\n\n\n\nMetric\nHealthy\nProblem\n\n\n\n\ntrain_loss\nDecreasing, starting ~2-4 for chat models\nFlat or increasing from step 1 — data or LR issue\n\n\neval_loss\nDecreasing, tracks train_loss\nIncreasing while train_loss decreases — overfitting\n\n\ngrad_norm\n0.1-10, relatively stable\nSpikes &gt;100 — instability. 0.0 — frozen weights\n\n\nlearning_rate\nFollows scheduler curve\nFlat or NaN — config issue\n\n\n\nWatch for: loss never decreasing (check train_on_inputs, dataset, LR), loss goes to 0 quickly (overfitting), eval_loss diverging (reduce epochs, add regularization). See training_stability.qmd.\n\n\n\n\n\n\n\n\n\n\nIssue\nFix\n\n\n\n\nOOM during training\nReduce micro_batch_size, enable gradient_checkpointing, reduce sequence_len\n\n\nsample_packing + SDPA + bf16 = 0.0 loss\nUse flash_attention: true or disable sample_packing\n\n\nMissing chat template error\nSet chat_template: chatml explicitly\n\n\nLabel masking wrong\nRun axolotl preprocess config.yaml --debug and inspect labels\n\n\nLoss NaN\nUse bf16: auto, lower LR, check data for empty samples\n\n\nTokenizer pad token / infinite loss\nSet special_tokens: pad_token: \"&lt;\\|end_of_text\\|&gt;\"\n\n\nFSDP save hangs\nUse fsdp_state_dict_type: FULL_STATE_DICT\n\n\nDeepSpeed CheckpointError\nSet use_reentrant: true in gradient_checkpointing_kwargs\n\n\n\n\n\n\nTo profile training and identify optimization opportunities:\n# Profile steps 3-7 (after warmup/autotuning settles)\nprofiler_steps_start: 3\nprofiler_steps: 5\nThis produces profiler_trace.json (Chrome trace) and snapshot.pickle (memory snapshot) in output_dir.\nView the Chrome trace at chrome://tracing.\nTo programmatically inspect the trace:\npython scripts/analyze_profile.py output_dir/\nThe trace shows per-kernel CUDA times, memory allocations, and operator-level breakdown. Look for:\n- Large matmul kernels: candidates for fusion or quantization\n- Memory copies (H2D/D2H): unnecessary data movement\n- Small frequent kernels: candidates for kernel fusion\n- Gaps between kernels: pipeline bubbles from CPU overhead\nFull troubleshooting: training_stability.qmd, debugging.qmd\n\n\n\nsrc/axolotl/\n  cli/train.py                     # Entry point for `axolotl train`\n  cli/preprocess.py                # Entry point for `axolotl preprocess`\n  core/builders/causal.py          # HFCausalTrainerBuilder — wires config → SFT trainer\n  core/trainers/base.py            # AxolotlTrainer — base trainer class\n  core/trainers/mixins/            # Packing, optimizer, scheduler, checkpoints\n  prompt_strategies/               # Format handlers: chat_template, alpaca, completion, input_output\n  utils/schemas/config.py          # AxolotlInputConfig — main config schema\n  utils/schemas/datasets.py        # SFTDataset, DatasetConfig\n  utils/schemas/peft.py            # LoraConfig — LoRA parameters\n  integrations/liger/              # Liger kernel plugin\n\nexamples/llama-3/                  # LoRA, QLoRA, full FT example configs\ndocs/getting-started.qmd           # Quickstart with config templates\ndocs/optimizations.qmd             # Flash attention, gradient checkpointing, sample packing\ndocs/multi-gpu.qmd                 # FSDP and DeepSpeed setup"
   },
   {
     "objectID": "docs/agents/sft.html#architecture",
@@ -3581,7 +3595,14 @@
     "href": "docs/agents/sft.html#known-issues",
     "title": "SFT — Agent Reference",
     "section": "",
-    "text": "Issue\nFix\n\n\n\n\nOOM during training\nReduce micro_batch_size, enable gradient_checkpointing, reduce sequence_len\n\n\nsample_packing + SDPA + bf16 = 0.0 loss\nUse flash_attention: true or disable sample_packing\n\n\nMissing chat template error\nSet chat_template: chatml explicitly\n\n\nLabel masking wrong\nRun axolotl preprocess config.yaml --debug and inspect labels\n\n\nLoss NaN\nUse bf16: auto, lower LR, check data for empty samples\n\n\nTokenizer pad token / infinite loss\nSet special_tokens: pad_token: \"&lt;\\|end_of_text\\|&gt;\"\n\n\nFSDP save hangs\nUse fsdp_state_dict_type: FULL_STATE_DICT\n\n\nDeepSpeed CheckpointError\nSet use_reentrant: true in gradient_checkpointing_kwargs\n\n\n\nFull troubleshooting: training_stability.qmd, debugging.qmd"
+    "text": "Issue\nFix\n\n\n\n\nOOM during training\nReduce micro_batch_size, enable gradient_checkpointing, reduce sequence_len\n\n\nsample_packing + SDPA + bf16 = 0.0 loss\nUse flash_attention: true or disable sample_packing\n\n\nMissing chat template error\nSet chat_template: chatml explicitly\n\n\nLabel masking wrong\nRun axolotl preprocess config.yaml --debug and inspect labels\n\n\nLoss NaN\nUse bf16: auto, lower LR, check data for empty samples\n\n\nTokenizer pad token / infinite loss\nSet special_tokens: pad_token: \"&lt;\\|end_of_text\\|&gt;\"\n\n\nFSDP save hangs\nUse fsdp_state_dict_type: FULL_STATE_DICT\n\n\nDeepSpeed CheckpointError\nSet use_reentrant: true in gradient_checkpointing_kwargs"
+  },
+  {
+    "objectID": "docs/agents/sft.html#profiling",
+    "href": "docs/agents/sft.html#profiling",
+    "title": "SFT — Agent Reference",
+    "section": "",
+    "text": "To profile training and identify optimization opportunities:\n# Profile steps 3-7 (after warmup/autotuning settles)\nprofiler_steps_start: 3\nprofiler_steps: 5\nThis produces profiler_trace.json (Chrome trace) and snapshot.pickle (memory snapshot) in output_dir.\nView the Chrome trace at chrome://tracing.\nTo programmatically inspect the trace:\npython scripts/analyze_profile.py output_dir/\nThe trace shows per-kernel CUDA times, memory allocations, and operator-level breakdown. Look for:\n- Large matmul kernels: candidates for fusion or quantization\n- Memory copies (H2D/D2H): unnecessary data movement\n- Small frequent kernels: candidates for kernel fusion\n- Gaps between kernels: pipeline bubbles from CPU overhead\nFull troubleshooting: training_stability.qmd, debugging.qmd"
   },
   {
     "objectID": "docs/agents/sft.html#file-map",
@@ -6948,7 +6969,7 @@
     "href": "index.html#quick-start---llm-fine-tuning-in-minutes",
     "title": "Axolotl",
     "section": "🚀 Quick Start - LLM Fine-tuning in Minutes",
-    "text": "🚀 Quick Start - LLM Fine-tuning in Minutes\nRequirements:\n\nNVIDIA GPU (Ampere or newer for bf16 and Flash Attention) or AMD GPU\nPython 3.11\nPyTorch ≥2.9.1\n\n\nGoogle Colab\n\n\n\nOpen In Colab\n\n\n\n\nInstallation\n\nUsing pip\npip3 install -U packaging==26.0 setuptools==75.8.0 wheel ninja\npip3 install --no-build-isolation axolotl[flash-attn,deepspeed]\n\n# Download example axolotl configs, deepspeed configs\naxolotl fetch examples\naxolotl fetch deepspeed_configs  # OPTIONAL\n\n\nUsing Docker\nInstalling with Docker can be less error prone than installing in your own environment.\ndocker run --gpus '\"all\"' --rm -it axolotlai/axolotl:main-latest\nOther installation approaches are described here.\n\n\nCloud Providers\n\n\nRunPod\nVast.ai\nPRIME Intellect\nModal\nNovita\nJarvisLabs.ai\nLatitude.sh\n\n\n\n\n\nYour First Fine-tune\n# Fetch axolotl examples\naxolotl fetch examples\n\n# Or, specify a custom path\naxolotl fetch examples --dest path/to/folder\n\n# Train a model using LoRA\naxolotl train examples/llama-3/lora-1b.yml\nThat’s it! Check out our Getting Started Guide for a more detailed walkthrough.",
+    "text": "🚀 Quick Start - LLM Fine-tuning in Minutes\nRequirements:\n\nNVIDIA GPU (Ampere or newer for bf16 and Flash Attention) or AMD GPU\nPython &gt;=3.11 (3.12 recommended)\nPyTorch ≥2.9.1\n\n\nGoogle Colab\n\n\n\nOpen In Colab\n\n\n\n\nInstallation\n\nUsing uv (recommended)\n# install uv if you don't already have it installed\ncurl -LsSf https://astral.sh/uv/install.sh | sh\nsource $HOME/.local/bin/env\n\n# CUDA 12.8.1 tends to have better package compatibility\nexport UV_TORCH_BACKEND=cu128\n\n# create a new virtual environment\nuv venv --python 3.12\nsource .venv/bin/activate\n\nuv pip install torch==2.10.0 torchvision\nuv pip install --no-build-isolation axolotl[deepspeed]\n\n# recommended - install cut-cross-entropy\nuv pip install \"cut-cross-entropy[transformers] @ git+https://github.com/axolotl-ai-cloud/ml-cross-entropy.git@main\"\n\n# (optional) - prefetch flash-attn2 and causal-conv1d kernels\nuv run --python 3.12 python -c \"from kernels import get_kernel; get_kernel('kernels-community/flash-attn2'); get_kernel('kernels-community/causal-conv1d')\"\n\n# Download example axolotl configs, deepspeed configs\naxolotl fetch examples\naxolotl fetch deepspeed_configs  # OPTIONAL\n\n\nUsing pip\npip3 install -U packaging==26.0 setuptools==75.8.0 wheel ninja\npip3 install --no-build-isolation axolotl[flash-attn,deepspeed]\n\n# Download example axolotl configs, deepspeed configs\naxolotl fetch examples\naxolotl fetch deepspeed_configs  # OPTIONAL\n\n\nUsing Docker\nInstalling with Docker can be less error prone than installing in your own environment.\ndocker run --gpus '\"all\"' --rm -it axolotlai/axolotl:main-latest\nOther installation approaches are described here.\n\n\nCloud Providers\n\n\nRunPod\nVast.ai\nPRIME Intellect\nModal\nNovita\nJarvisLabs.ai\nLatitude.sh\n\n\n\n\n\nYour First Fine-tune\n# Fetch axolotl examples\naxolotl fetch examples\n\n# Or, specify a custom path\naxolotl fetch examples --dest path/to/folder\n\n# Train a model using LoRA\naxolotl train examples/llama-3/lora-1b.yml\nThat’s it! Check out our Getting Started Guide for a more detailed walkthrough.",
     "crumbs": [
       "Home"
     ]
diff --git a/sitemap.xml b/sitemap.xml
index cc118d140..1b5dc4e98 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -2,990 +2,990 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
   <url>
     <loc>https://docs.axolotl.ai/FAQS.html</loc>
-    <lastmod>2026-04-10T18:12:14.277Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.538Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/qat.html</loc>
-    <lastmod>2026-04-10T18:12:14.285Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/seed-oss.html</loc>
-    <lastmod>2026-04-10T18:16:18.521Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.273Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/internvl3_5.html</loc>
-    <lastmod>2026-04-10T18:16:18.513Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.263Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/apertus.html</loc>
-    <lastmod>2026-04-10T18:16:18.521Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.272Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/mistral.html</loc>
-    <lastmod>2026-04-10T18:16:18.519Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.270Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/smolvlm2.html</loc>
-    <lastmod>2026-04-10T18:16:18.522Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.274Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/arcee.html</loc>
-    <lastmod>2026-04-10T18:16:18.514Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.264Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/ministral3/vision.html</loc>
-    <lastmod>2026-04-10T18:16:18.515Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.266Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/kimi-linear.html</loc>
-    <lastmod>2026-04-10T18:16:18.512Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.262Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/mimo.html</loc>
-    <lastmod>2026-04-10T18:16:18.513Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.263Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/phi.html</loc>
-    <lastmod>2026-04-10T18:16:18.522Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.273Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/qwen3.html</loc>
-    <lastmod>2026-04-10T18:16:18.520Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.271Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/hunyuan.html</loc>
-    <lastmod>2026-04-10T18:16:18.523Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.275Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/ministral.html</loc>
-    <lastmod>2026-04-10T18:16:18.517Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.268Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/gemma3n.html</loc>
-    <lastmod>2026-04-10T18:16:18.520Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.271Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/devstral.html</loc>
-    <lastmod>2026-04-10T18:16:18.518Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.269Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/magistral/think.html</loc>
-    <lastmod>2026-04-10T18:16:18.517Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.267Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/qwen3-next.html</loc>
-    <lastmod>2026-04-10T18:16:18.520Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.271Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/training_stability.html</loc>
-    <lastmod>2026-04-10T18:12:14.286Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.545Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/expert_quantization.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/rlhf.html</loc>
-    <lastmod>2026-04-10T18:12:14.285Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/dataset-formats/conversation.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.540Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/dataset-formats/template_free.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/dataset-formats/tokenized.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/gradient_checkpointing.html</loc>
-    <lastmod>2026-04-10T18:12:14.281Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/checkpoint_saving.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.540Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.kto.llama3.html</loc>
-    <lastmod>2026-04-10T18:15:54.033Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.201Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.chat.format.chatml.html</loc>
-    <lastmod>2026-04-10T18:15:53.349Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.499Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.quantize.html</loc>
-    <lastmod>2026-04-10T18:15:53.560Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.720Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.lora.html</loc>
-    <lastmod>2026-04-10T18:15:54.324Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.498Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/loaders.model.html</loc>
-    <lastmod>2026-04-10T18:15:53.732Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.895Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.merge_lora.html</loc>
-    <lastmod>2026-04-10T18:15:53.530Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.689Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.trainers.dpo.trainer.html</loc>
-    <lastmod>2026-04-10T18:15:53.685Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.848Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.transformers_fa_utils.html</loc>
-    <lastmod>2026-04-10T18:15:54.255Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.427Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.completion.html</loc>
-    <lastmod>2026-04-10T18:15:53.939Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.109Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/convert.html</loc>
-    <lastmod>2026-04-10T18:15:53.217Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.359Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.schemas.datasets.html</loc>
-    <lastmod>2026-04-10T18:15:54.546Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.725Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.chat.format.llama3x.html</loc>
-    <lastmod>2026-04-10T18:15:53.351Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.502Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.messages.chat.html</loc>
-    <lastmod>2026-04-10T18:15:53.982Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.150Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.trainers.grpo.sampler.html</loc>
-    <lastmod>2026-04-10T18:15:53.718Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.881Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/kernels.quantize.html</loc>
-    <lastmod>2026-04-10T18:15:54.170Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.340Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/integrations.liger.args.html</loc>
-    <lastmod>2026-04-10T18:15:54.810Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.996Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.preprocess.html</loc>
-    <lastmod>2026-04-10T18:15:53.554Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.713Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.callbacks.comet_.html</loc>
-    <lastmod>2026-04-10T18:15:54.951Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.141Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.callbacks.profiler.html</loc>
-    <lastmod>2026-04-10T18:15:54.940Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.130Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.utils.html</loc>
-    <lastmod>2026-04-10T18:15:54.234Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.405Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.orpo.chat_template.html</loc>
-    <lastmod>2026-04-10T18:15:54.071Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.240Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.chat_templates.html</loc>
-    <lastmod>2026-04-10T18:15:54.318Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.491Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.chatml.html</loc>
-    <lastmod>2026-04-10T18:15:54.017Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.185Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.gradient_checkpointing.offload_disk.html</loc>
-    <lastmod>2026-04-10T18:15:54.308Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.481Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.checks.html</loc>
-    <lastmod>2026-04-10T18:15:53.473Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.631Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.schemas.utils.html</loc>
-    <lastmod>2026-04-10T18:15:54.606Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.786Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.lora_kernels.html</loc>
-    <lastmod>2026-04-10T18:15:54.227Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.398Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.schemas.model.html</loc>
-    <lastmod>2026-04-10T18:15:54.513Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.690Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.pygmalion.html</loc>
-    <lastmod>2026-04-10T18:15:53.975Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.144Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.bradley_terry.llama3.html</loc>
-    <lastmod>2026-04-10T18:15:54.076Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.245Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.optimizers.adopt.html</loc>
-    <lastmod>2026-04-10T18:15:54.451Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.627Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.bench.html</loc>
-    <lastmod>2026-04-10T18:15:54.336Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.509Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.callbacks.qat.html</loc>
-    <lastmod>2026-04-10T18:15:54.960Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.150Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.trainers.base.html</loc>
-    <lastmod>2026-04-10T18:15:53.651Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.813Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.zephyr.html</loc>
-    <lastmod>2026-04-10T18:15:54.019Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.187Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.chat.messages.html</loc>
-    <lastmod>2026-04-10T18:15:53.347Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.497Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/integrations.lm_eval.args.html</loc>
-    <lastmod>2026-04-10T18:15:54.814Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.000Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.trainers.mamba.html</loc>
-    <lastmod>2026-04-10T18:15:53.676Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.839Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/train.html</loc>
-    <lastmod>2026-04-10T18:15:53.179Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.320Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.orcamini.html</loc>
-    <lastmod>2026-04-10T18:15:53.966Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.136Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.inference.html</loc>
-    <lastmod>2026-04-10T18:15:53.519Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.678Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.input_output.html</loc>
-    <lastmod>2026-04-10T18:15:53.947Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.116Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.data.streaming.html</loc>
-    <lastmod>2026-04-10T18:15:54.452Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.629Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.cloud.modal_.html</loc>
-    <lastmod>2026-04-10T18:15:53.581Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.741Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.collators.mm_chat.html</loc>
-    <lastmod>2026-04-10T18:15:54.877Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.065Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.config.html</loc>
-    <lastmod>2026-04-10T18:15:53.495Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.654Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.multipack.html</loc>
-    <lastmod>2026-04-10T18:15:54.184Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.355Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/integrations.grokfast.optimizer.html</loc>
-    <lastmod>2026-04-10T18:15:54.796Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.982Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.trainers.trl.html</loc>
-    <lastmod>2026-04-10T18:15:53.669Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.832Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.alpaca_chat.html</loc>
-    <lastmod>2026-04-10T18:15:53.889Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.056Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.builders.rl.html</loc>
-    <lastmod>2026-04-10T18:15:53.301Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.450Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.schemas.trl.html</loc>
-    <lastmod>2026-04-10T18:15:54.561Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.740Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.collators.batching.html</loc>
-    <lastmod>2026-04-10T18:15:54.867Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.054Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.model_shard_quant.html</loc>
-    <lastmod>2026-04-10T18:15:54.331Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.505Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/integrations.kd.trainer.html</loc>
-    <lastmod>2026-04-10T18:15:54.806Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.992Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.schemas.peft.html</loc>
-    <lastmod>2026-04-10T18:15:54.557Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.736Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.ctx_managers.sequence_parallel.html</loc>
-    <lastmod>2026-04-10T18:15:53.825Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.992Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.vllm_serve.html</loc>
-    <lastmod>2026-04-10T18:15:53.569Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.728Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.quantization.html</loc>
-    <lastmod>2026-04-10T18:15:54.484Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.662Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.collators.mamba.html</loc>
-    <lastmod>2026-04-10T18:15:54.871Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.059Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/kernels.geglu.html</loc>
-    <lastmod>2026-04-10T18:15:54.142Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.312Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.trainers.utils.html</loc>
-    <lastmod>2026-04-10T18:15:53.720Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.883Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.llama_attn_hijack_flash.html</loc>
-    <lastmod>2026-04-10T18:15:54.179Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.349Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/common.architectures.html</loc>
-    <lastmod>2026-04-10T18:15:54.820Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.006Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.merge_sharded_fsdp_weights.html</loc>
-    <lastmod>2026-04-10T18:15:53.544Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.703Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.llama2_chat.html</loc>
-    <lastmod>2026-04-10T18:15:53.932Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.101Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/loaders.adapter.html</loc>
-    <lastmod>2026-04-10T18:15:53.752Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.916Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.utils.fetch.html</loc>
-    <lastmod>2026-04-10T18:15:53.604Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.765Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.stablelm_attn_hijack_flash.html</loc>
-    <lastmod>2026-04-10T18:15:54.243Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.415Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.main.html</loc>
-    <lastmod>2026-04-10T18:15:53.415Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.572Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.schedulers.html</loc>
-    <lastmod>2026-04-10T18:15:54.409Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.584Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.samplers.multipack.html</loc>
-    <lastmod>2026-04-10T18:15:54.928Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.117Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.chat.format.shared.html</loc>
-    <lastmod>2026-04-10T18:15:53.352Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.504Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.schemas.multimodal.html</loc>
-    <lastmod>2026-04-10T18:15:54.568Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.747Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/multimodal.html</loc>
-    <lastmod>2026-04-10T18:12:14.285Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/input_output.html</loc>
-    <lastmod>2026-04-10T18:12:14.284Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.542Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/multi-gpu.html</loc>
-    <lastmod>2026-04-10T18:12:14.284Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/lora_optims.html</loc>
-    <lastmod>2026-04-10T18:12:14.284Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/telemetry.html</loc>
-    <lastmod>2026-04-10T18:12:14.286Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.545Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/batch_vs_grad.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.540Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/custom_integrations.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.540Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/fsdp_qlora.html</loc>
-    <lastmod>2026-04-10T18:12:14.281Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/nccl.html</loc>
-    <lastmod>2026-04-10T18:12:14.285Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/vllm_serving.html</loc>
-    <lastmod>2026-04-10T18:12:14.286Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.545Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/attention.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.540Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/multipack.html</loc>
-    <lastmod>2026-04-10T18:12:14.285Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/torchao.html</loc>
-    <lastmod>2026-04-10T18:12:14.286Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.545Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/nd_parallelism.html</loc>
-    <lastmod>2026-04-10T18:12:14.285Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/mac.html</loc>
-    <lastmod>2026-04-10T18:12:14.284Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/reward_modelling.html</loc>
-    <lastmod>2026-04-10T18:12:14.285Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/agents/model_architectures.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.540Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/agents/grpo.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.540Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/agents/pretraining.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.540Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/agents/new_model_support.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.540Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/optimizers.html</loc>
-    <lastmod>2026-04-10T18:12:14.285Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/examples/colab-notebooks/colab-axolotl-example.html</loc>
-    <lastmod>2026-04-10T18:12:14.293Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.549Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/src/axolotl/integrations/LICENSE.html</loc>
-    <lastmod>2026-04-10T18:12:14.328Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.573Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/src/axolotl/integrations/cut_cross_entropy/ACKNOWLEDGEMENTS.html</loc>
-    <lastmod>2026-04-10T18:12:14.329Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.574Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/installation.html</loc>
-    <lastmod>2026-04-10T18:12:14.284Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.542Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/ebft.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/agents/reward_modelling.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.540Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/agents/sft.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.540Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/agents/preference_tuning.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.540Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/mixed_precision.html</loc>
-    <lastmod>2026-04-10T18:12:14.284Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/docker.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/grpo.html</loc>
-    <lastmod>2026-04-10T18:12:14.281Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/streaming.html</loc>
-    <lastmod>2026-04-10T18:12:14.286Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.545Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/choosing_method.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.540Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/dataset_loading.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/sequence_parallelism.html</loc>
-    <lastmod>2026-04-10T18:12:14.286Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.545Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/optimizations.html</loc>
-    <lastmod>2026-04-10T18:12:14.285Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/multi-node.html</loc>
-    <lastmod>2026-04-10T18:12:14.284Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/lr_groups.html</loc>
-    <lastmod>2026-04-10T18:12:14.284Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/quantize.html</loc>
-    <lastmod>2026-04-10T18:12:14.285Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/inference.html</loc>
-    <lastmod>2026-04-10T18:12:14.284Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.542Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/ray-integration.html</loc>
-    <lastmod>2026-04-10T18:12:14.285Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.543Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/amd_hpc.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.540Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/unsloth.html</loc>
-    <lastmod>2026-04-10T18:12:14.286Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.545Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/getting-started.html</loc>
-    <lastmod>2026-04-10T18:12:14.281Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.schemas.enums.html</loc>
-    <lastmod>2026-04-10T18:15:54.599Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.779Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.builders.base.html</loc>
-    <lastmod>2026-04-10T18:15:53.290Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.433Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/logging_config.html</loc>
-    <lastmod>2026-04-10T18:15:53.282Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.425Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.alpaca_instruct.html</loc>
-    <lastmod>2026-04-10T18:15:53.891Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.058Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.callbacks.mlflow_.html</loc>
-    <lastmod>2026-04-10T18:15:54.947Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.136Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.utils.html</loc>
-    <lastmod>2026-04-10T18:15:53.583Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.743Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/loaders.patch_manager.html</loc>
-    <lastmod>2026-04-10T18:15:53.774Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.938Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.datasets.chat.html</loc>
-    <lastmod>2026-04-10T18:15:53.359Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.512Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.trainers.mixins.scheduler.html</loc>
-    <lastmod>2026-04-10T18:15:53.796Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.961Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/common.datasets.html</loc>
-    <lastmod>2026-04-10T18:15:54.841Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.027Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.kto.chatml.html</loc>
-    <lastmod>2026-04-10T18:15:54.043Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.212Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.user_defined.html</loc>
-    <lastmod>2026-04-10T18:15:53.916Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.085Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.chat_template.html</loc>
-    <lastmod>2026-04-10T18:15:53.871Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.039Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/kernels.utils.html</loc>
-    <lastmod>2026-04-10T18:15:54.171Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.342Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.mistral_attn_hijack_flash.html</loc>
-    <lastmod>2026-04-10T18:15:54.182Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.353Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.schemas.config.html</loc>
-    <lastmod>2026-04-10T18:15:54.504Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.681Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.dict.html</loc>
-    <lastmod>2026-04-10T18:15:54.441Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.617Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/loaders.constants.html</loc>
-    <lastmod>2026-04-10T18:15:53.776Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.940Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.data.batch_dataset_fetcher.html</loc>
-    <lastmod>2026-04-10T18:15:54.269Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.441Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/kernels.lora.html</loc>
-    <lastmod>2026-04-10T18:15:54.129Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.299Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/datasets.html</loc>
-    <lastmod>2026-04-10T18:15:53.200Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.342Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/common.const.html</loc>
-    <lastmod>2026-04-10T18:15:54.822Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.008Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.trainers.grpo.trainer.html</loc>
-    <lastmod>2026-04-10T18:15:53.703Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.866Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/kernels.swiglu.html</loc>
-    <lastmod>2026-04-10T18:15:54.155Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.325Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.utils.load.html</loc>
-    <lastmod>2026-04-10T18:15:53.610Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.772Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.passthrough.html</loc>
-    <lastmod>2026-04-10T18:15:54.023Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.191Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.delinearize_llama4.html</loc>
-    <lastmod>2026-04-10T18:15:53.502Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.660Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.stepwise_supervised.html</loc>
-    <lastmod>2026-04-10T18:15:53.953Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.122Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.freeze.html</loc>
-    <lastmod>2026-04-10T18:15:54.348Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.522Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.train.html</loc>
-    <lastmod>2026-04-10T18:15:53.426Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.583Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/models.mamba.modeling_mamba.html</loc>
-    <lastmod>2026-04-10T18:15:54.842Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.029Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.alpaca_w_system.html</loc>
-    <lastmod>2026-04-10T18:15:53.906Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.074Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.kto.user_defined.html</loc>
-    <lastmod>2026-04-10T18:15:54.045Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.214Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/integrations.base.html</loc>
-    <lastmod>2026-04-10T18:15:54.791Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.976Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.callbacks.perplexity.html</loc>
-    <lastmod>2026-04-10T18:15:54.936Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.125Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/loaders.tokenizer.html</loc>
-    <lastmod>2026-04-10T18:15:53.743Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.907Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.datasets.transforms.chat_builder.html</loc>
-    <lastmod>2026-04-10T18:15:53.368Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.522Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.chat_template.html</loc>
-    <lastmod>2026-04-10T18:15:53.991Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.158Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/loaders.processor.html</loc>
-    <lastmod>2026-04-10T18:15:53.745Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.908Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.tokenization.html</loc>
-    <lastmod>2026-04-10T18:15:54.316Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.490Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.trainer_fsdp_optim.html</loc>
-    <lastmod>2026-04-10T18:15:54.247Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.419Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.data.sft.html</loc>
-    <lastmod>2026-04-10T18:15:54.460Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.637Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.user_defined.html</loc>
-    <lastmod>2026-04-10T18:15:54.021Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.189Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/integrations.spectrum.args.html</loc>
-    <lastmod>2026-04-10T18:15:54.818Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.005Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/integrations.cut_cross_entropy.args.html</loc>
-    <lastmod>2026-04-10T18:15:54.795Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.981Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.mixtral.html</loc>
-    <lastmod>2026-04-10T18:15:54.270Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.443Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.llama3.html</loc>
-    <lastmod>2026-04-10T18:15:54.004Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.172Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.gradient_checkpointing.offload_cpu.html</loc>
-    <lastmod>2026-04-10T18:15:54.275Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.447Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.llama_attn_hijack_xformers.html</loc>
-    <lastmod>2026-04-10T18:15:54.180Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.351Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.trainer.html</loc>
-    <lastmod>2026-04-10T18:15:54.370Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.544Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.btlm_attn_hijack_flash.html</loc>
-    <lastmod>2026-04-10T18:15:54.236Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.407Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.builders.causal.html</loc>
-    <lastmod>2026-04-10T18:15:53.296Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.439Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.cloud.base.html</loc>
-    <lastmod>2026-04-10T18:15:53.573Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.733Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.trainers.mixins.rng_state_loader.html</loc>
-    <lastmod>2026-04-10T18:15:53.787Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.952Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/index.html</loc>
-    <lastmod>2026-04-10T18:15:53.102Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.241Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.base.html</loc>
-    <lastmod>2026-04-10T18:15:53.827Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.994Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.unsloth_.html</loc>
-    <lastmod>2026-04-10T18:15:54.257Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.429Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.schemas.integrations.html</loc>
-    <lastmod>2026-04-10T18:15:54.588Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.767Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.training_args.html</loc>
-    <lastmod>2026-04-10T18:15:53.317Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.466Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.utils.sweeps.html</loc>
-    <lastmod>2026-04-10T18:15:53.618Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.779Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.art.html</loc>
-    <lastmod>2026-04-10T18:15:53.465Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.623Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/monkeypatch.relora.html</loc>
-    <lastmod>2026-04-10T18:15:54.189Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.359Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_tokenizers.html</loc>
-    <lastmod>2026-04-10T18:15:53.270Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.413Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.args.html</loc>
-    <lastmod>2026-04-10T18:15:53.461Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.619Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.collators.core.html</loc>
-    <lastmod>2026-04-10T18:15:54.844Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.031Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.distributed.html</loc>
-    <lastmod>2026-04-10T18:15:54.434Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.610Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.evaluate.html</loc>
-    <lastmod>2026-04-10T18:15:53.436Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.593Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/evaluate.html</loc>
-    <lastmod>2026-04-10T18:15:53.192Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.334Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/prompt_strategies.metharme.html</loc>
-    <lastmod>2026-04-10T18:15:53.962Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.131Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.callbacks.lisa.html</loc>
-    <lastmod>2026-04-10T18:15:54.942Z</lastmod>
+    <lastmod>2026-04-10T20:50:40.132Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.utils.args.html</loc>
-    <lastmod>2026-04-10T18:15:53.597Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.758Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/utils.schemas.training.html</loc>
-    <lastmod>2026-04-10T18:15:54.522Z</lastmod>
+    <lastmod>2026-04-10T20:50:39.699Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/core.trainers.mixins.optimizer.html</loc>
-    <lastmod>2026-04-10T18:15:53.783Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.948Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/api/cli.utils.train.html</loc>
-    <lastmod>2026-04-10T18:15:53.633Z</lastmod>
+    <lastmod>2026-04-10T20:50:38.794Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/faq.html</loc>
-    <lastmod>2026-04-10T18:12:14.281Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/dataset_preprocessing.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/dataset-formats/inst_tune.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/dataset-formats/pretraining.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/dataset-formats/index.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/dataset-formats/stepwise_supervised.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/cli.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.540Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/config-reference.html</loc>
-    <lastmod>2026-04-10T18:16:17.575Z</lastmod>
+    <lastmod>2026-04-10T20:51:04.227Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/debugging.html</loc>
-    <lastmod>2026-04-10T18:12:14.280Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.541Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/magistral/vision.html</loc>
-    <lastmod>2026-04-10T18:16:18.517Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.268Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/trinity.html</loc>
-    <lastmod>2026-04-10T18:16:18.514Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.264Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/gpt-oss.html</loc>
-    <lastmod>2026-04-10T18:16:18.521Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.272Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/LiquidAI.html</loc>
-    <lastmod>2026-04-10T18:16:18.523Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.274Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/granite4.html</loc>
-    <lastmod>2026-04-10T18:16:18.522Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.274Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/voxtral.html</loc>
-    <lastmod>2026-04-10T18:16:18.518Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.269Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/mistral-small.html</loc>
-    <lastmod>2026-04-10T18:16:18.518Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.268Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/llama-4.html</loc>
-    <lastmod>2026-04-10T18:16:18.519Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.270Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/llama-2.html</loc>
-    <lastmod>2026-04-10T18:16:18.519Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.270Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/jamba.html</loc>
-    <lastmod>2026-04-10T18:16:18.523Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.275Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/ministral3/think.html</loc>
-    <lastmod>2026-04-10T18:16:18.515Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.266Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/orpheus.html</loc>
-    <lastmod>2026-04-10T18:16:18.524Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.276Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/index.html</loc>
-    <lastmod>2026-04-10T18:16:18.524Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.276Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/olmo3.html</loc>
-    <lastmod>2026-04-10T18:16:18.513Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.264Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/magistral.html</loc>
-    <lastmod>2026-04-10T18:16:18.516Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.267Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/ministral3.html</loc>
-    <lastmod>2026-04-10T18:16:18.515Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.265Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/docs/models/plano.html</loc>
-    <lastmod>2026-04-10T18:16:18.512Z</lastmod>
+    <lastmod>2026-04-10T20:51:05.262Z</lastmod>
   </url>
   <url>
     <loc>https://docs.axolotl.ai/index.html</loc>
-    <lastmod>2026-04-10T18:12:14.317Z</lastmod>
+    <lastmod>2026-04-10T20:46:58.566Z</lastmod>
   </url>
 </urlset>