diff --git a/.nojekyll b/.nojekyll
index 9065ad1e5..3d627ddfd 100644
--- a/.nojekyll
+++ b/.nojekyll
@@ -1 +1 @@
-54352a43
\ No newline at end of file
+e905cd86
\ No newline at end of file
diff --git a/docs/rlhf.html b/docs/rlhf.html
index 767826029..dcfb5a36f 100644
--- a/docs/rlhf.html
+++ b/docs/rlhf.html
@@ -434,7 +434,7 @@ pre > code.sourceCode > span > a:first-child::before { text-decoration: underlin
 <div id="quarto-sidebar-glass" class="quarto-sidebar-collapse-item" data-bs-toggle="collapse" data-bs-target=".quarto-sidebar-collapse-item"></div>
 <!-- margin-sidebar -->
     <div id="quarto-margin-sidebar" class="sidebar margin-sidebar">
-        <nav id="TOC" role="doc-toc" class="toc-active">
+        <nav id="TOC" role="doc-toc" class="toc-active" data-toc-expanded="2">
     <h2 id="toc-title">On this page</h2>
    
   <ul>
@@ -962,11 +962,13 @@ Tip
 <span id="cb36-8"><a href="#cb36-8" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">vllm_gpu_memory_utilization</span><span class="kw">:</span><span class="at"> </span><span class="fl">0.15</span></span>
 <span id="cb36-9"><a href="#cb36-9" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">num_generations</span><span class="kw">:</span><span class="at"> </span><span class="dv">4</span></span>
 <span id="cb36-10"><a href="#cb36-10" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">reward_funcs</span><span class="kw">:</span><span class="at"> </span><span class="kw">[</span><span class="st">"rewards.rand_reward_func"</span><span class="kw">]</span><span class="co">    # format: '{file_name}.{fn_name}'</span></span>
-<span id="cb36-11"><a href="#cb36-11" aria-hidden="true" tabindex="-1"></a><span class="fu">datasets</span><span class="kw">:</span></span>
-<span id="cb36-12"><a href="#cb36-12" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> </span><span class="fu">path</span><span class="kw">:</span><span class="at"> openai/gsm8k</span></span>
-<span id="cb36-13"><a href="#cb36-13" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">name</span><span class="kw">:</span><span class="at"> main</span></span>
-<span id="cb36-14"><a href="#cb36-14" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">type</span><span class="kw">:</span><span class="at"> rewards.oai_gsm8k_transform</span><span class="co">  # format: '{file_name}.{fn_name}'</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
+<span id="cb36-11"><a href="#cb36-11" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">reward_weights</span><span class="kw">:</span><span class="at"> </span><span class="kw">[</span><span class="fl">1.0</span><span class="kw">]</span></span>
+<span id="cb36-12"><a href="#cb36-12" aria-hidden="true" tabindex="-1"></a><span class="fu">datasets</span><span class="kw">:</span></span>
+<span id="cb36-13"><a href="#cb36-13" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> </span><span class="fu">path</span><span class="kw">:</span><span class="at"> openai/gsm8k</span></span>
+<span id="cb36-14"><a href="#cb36-14" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">name</span><span class="kw">:</span><span class="at"> main</span></span>
+<span id="cb36-15"><a href="#cb36-15" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">type</span><span class="kw">:</span><span class="at"> rewards.oai_gsm8k_transform</span><span class="co">  # format: '{file_name}.{fn_name}'</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
 <p>To see other examples of custom reward functions, please see <a href="https://github.com/huggingface/trl/blob/main/docs/source/grpo_trainer.md#using-a-custom-reward-function">TRL GRPO Docs</a>.</p>
+<p>To see description of the configs, please see <a href="https://github.com/axolotl-ai-cloud/axolotl/blob/main/src/axolotl/utils/config/models/input/v0_4_1/trl.py">TRLConfig</a>.</p>
 </section>
 <section id="using-local-dataset-files" class="level3">
 <h3 class="anchored" data-anchor-id="using-local-dataset-files">Using local dataset files</h3>
diff --git a/search.json b/search.json
index 8f56bcbd1..7214fe3e8 100644
--- a/search.json
+++ b/search.json
@@ -878,7 +878,7 @@
     "href": "docs/rlhf.html#rlhf-using-axolotl",
     "title": "RLHF (Beta)",
     "section": "RLHF using Axolotl",
-    "text": "RLHF using Axolotl\n\n\n\n\n\n\nImportant\n\n\n\nThis is a BETA feature and many features are not fully implemented. You are encouraged to open new PRs to improve the integration and functionality.\n\n\nWe rely on the TRL library for implementations of various RL training methods, which we wrap around to expose in axolotl. Each method has their own supported ways of loading datasets and prompt formats.\n\n\n\n\n\n\nTip\n\n\n\nYou can find what each method supports by going into src/axolotl/prompt_strategies/{method} where {method} is one of our supported methods. The type: can be retrieved from {method}.{function_name}.\n\n\n\nDPO\nExample config:\nrl: dpo\ndatasets:\n  - path: Intel/orca_dpo_pairs\n    split: train\n    type: chatml.intel\n  - path: argilla/ultrafeedback-binarized-preferences\n    split: train\n    type: chatml\nDPO supports the following types with the following dataset format:\n\nchatml.argilla\n{\n    \"system\": \"...\", // optional\n    \"instruction\": \"...\",\n    \"chosen_response\": \"...\",\n    \"rejected_response\": \"...\"\n}\n\n\nchatml.argilla_chat\n{\n    \"chosen\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ],\n    \"rejected\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ]\n}\n\n\nchatml.icr\n{\n    \"system\": \"...\", // optional\n    \"input\": \"...\",\n    \"chosen\": \"...\",\n    \"rejected\": \"...\"\n}\n\n\nchatml.intel\n{\n    \"system\": \"...\", // optional\n    \"question\": \"...\",\n    \"chosen\": \"...\",\n    \"rejected\": \"...\"\n}\n\n\nchatml.prompt_pairs\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"chosen\": \"...\",\n    \"rejected\": \"...\"\n}\n\n\nchatml.ultra\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"chosen\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ],\n    \"rejected\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ]\n}\n\n\nllama3.argilla\n{\n    \"system\": \"...\", // optional\n    \"instruction\": \"...\",\n    \"chosen_response\": \"...\",\n    \"rejected_response\": \"...\"\n}\n\n\nllama3.argilla_chat\n{\n    \"chosen\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ],\n    \"rejected\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ]\n}\n\n\nllama3.icr\n{\n    \"system\": \"...\", // optional\n    \"input\": \"...\",\n    \"chosen\": \"...\",\n    \"rejected\": \"...\"\n}\n\n\nllama3.intel\n{\n    \"system\": \"...\", // optional\n    \"question\": \"...\",\n    \"chosen\": \"...\",\n    \"rejected\": \"...\"\n}\n\n\nllama3.prompt_pairs\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"chosen\": \"...\",\n    \"rejected\": \"...\"\n}\n\n\nllama3.ultra\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"chosen\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ],\n    \"rejected\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ]\n}\n\n\nzephyr.nectar\n{\n    \"prompt\": \"...\",\n    \"answers\": [\n        {\n            \"answer\": \"...\",\n            \"rank\": 1\n        },\n        {\n            \"answer\": \"...\",\n            \"rank\": 2\n        }\n        // ... more answers with ranks\n    ]\n}\n\n\nchat_template.default\nrl: dpo\ndatasets:\n  - path: ...\n    split: train\n    type: chat_template.default\n    field_messages: \"messages\"\n    field_chosen: \"chosen\"\n    field_rejected: \"rejected\"\n    message_property_mappings:\n      role: role\n      content: content\n    roles:\n      user: [\"user\"]\n      assistant: [\"assistant\"]\n      system: [\"system\"]\nSample input format:\n{\n    \"messages\": [\n        {\n            \"role\": \"system\",\n            \"content\": \"...\"\n        },\n        {\n            \"role\": \"user\",\n            \"content\": \"...\"\n        },\n        // ... more messages\n    ],\n    \"chosen\": {\n        \"role\": \"assistant\",\n        \"content\": \"...\"\n    },\n    \"rejected\": {\n        \"role\": \"assistant\",\n        \"content\": \"...\"\n    }\n}\n\n\nuser_defined.default\nFor custom behaviors,\nrl: dpo\ndatasets:\n  - path: ...\n    split: train\n    type: user_defined.default\n\n    field_prompt: \"prompt\"\n    field_system: \"system\"\n    field_chosen: \"chosen\"\n    field_rejected: \"rejected\"\n    prompt_format: \"{prompt}\"\n    chosen_format: \"{chosen}\"\n    rejected_format: \"{rejected}\"\nThe input format is a simple JSON input with customizable fields based on the above config.\n{\n    \"system\": \"...\",  // optional\n    \"prompt\": \"...\",\n    \"chosen\": \"...\",\n    \"rejected\": \"...\"\n}\n\n\n\nIPO\nAs IPO is just DPO with a different loss function, all supported options for DPO works here.\nrl: ipo\n\n\nORPO\nPaper: https://arxiv.org/abs/2403.07691\nrl: orpo\norpo_alpha: 0.1\nremove_unused_columns: false\n\nchat_template: chatml\ndatasets:\n  - path: argilla/ultrafeedback-binarized-preferences-cleaned\n    type: chat_template.argilla\nORPO supports the following types with the following dataset format:\n\nchat_template.argilla\n{\n    \"system\": \"...\",  // optional\n    \"prompt\": \"...\",  // if available, will be taken as user message for single-turn instead of from list below\n\n    // chosen/rejected should be same till last content and only even-number of alternating user/assistant turns\n    \"chosen\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ],\n    \"rejected\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ]\n}\n\n\n\nKTO\nrl: kto\nrl_beta: 0.5\nkto_desirable_weight: 0.2\n\nremove_unused_columns: false\n\ndatasets:\n  - path: argilla/ultrafeedback-binarized-preferences-cleaned-kto\n    type: llama3.ultra\n    split: train\n\ngradient_checkpointing: true\ngradient_checkpointing_kwargs:\n  use_reentrant: true\nKTO supports the following types with the following dataset format:\n\nchatml.argilla\n{\n    \"system\": \"...\", // optional\n    \"instruction\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nchatml.argilla_chat\n{\n    \"chosen\": [\n        {\"role\": \"user\", \"content\": \"...\"}\n    ],\n    \"completion\": [\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ]\n}\n\n\nchatml.intel\n{\n    \"system\": \"...\", // optional\n    \"question\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nchatml.prompt_pairs\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nchatml.ultra\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nllama3.argilla\n{\n    \"system\": \"...\", // optional\n    \"instruction\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nllama3.argilla_chat\n{\n    \"completion\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ]\n}\n\n\nllama3.intel\n{\n    \"system\": \"...\", // optional\n    \"question\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nllama3.prompt_pairs\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nllama3.ultra\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nuser_defined.default\nFor custom behaviors,\nrl: kto\ndatasets:\n  - path: ...\n    split: train\n    type: user_defined.default\n\n    field_prompt: \"prompt\"\n    field_system: \"system\"\n    field_completion: \"completion\"\n    field_label: \"label\"\n    prompt_format: \"{prompt}\"\n    completion_format: \"{completion}\"\nThe input format is a simple JSON input with customizable fields based on the above config.\n{\n    \"system\": \"...\",  // optional\n    \"prompt\": \"...\",\n    \"completion\": \"...\",\n    \"label\": \"...\"\n}\n\n\n\nGRPO\nGRPO uses custom reward functions and transformations. Please have them ready locally.\nFor ex, to load OpenAI’s GSM8K and use a random reward for completions:\n# rewards.py\nimport random\n\ndef rand_reward_func(completions, **kwargs) -&gt; list[float]:\n    return [random.uniform(0, 1) for _ in completions]\n\ndef oai_gsm8k_transform(cfg, *args, **kwargs):\n    def transform_fn(example, tokenizer=None):\n        label = example[\"answer\"].split(\"####\")[-1].strip().replace(\",\", \"\")\n        return {\n            \"prompt\": [{\"role\": \"user\", \"content\": example[\"question\"]},],\n            \"answer\": label,\n        }\n    return transform_fn, {\"remove_columns\": [\"question\"]}\nrl: grpo\n\ntrl:\n    beta: 0.001\n    max_completion_length: 256\n    use_vllm: True\n    vllm_device: auto\n    vllm_gpu_memory_utilization: 0.15\n    num_generations: 4\n    reward_funcs: [\"rewards.rand_reward_func\"]    # format: '{file_name}.{fn_name}'\ndatasets:\n  - path: openai/gsm8k\n    name: main\n    type: rewards.oai_gsm8k_transform  # format: '{file_name}.{fn_name}'\nTo see other examples of custom reward functions, please see TRL GRPO Docs.\n\n\nUsing local dataset files\ndatasets:\n  - ds_type: json\n    data_files:\n      - orca_rlhf.jsonl\n    split: train\n    type: chatml.intel\n\n\nTRL auto-unwrapping for PEFT\nTRL supports auto-unwrapping PEFT models for RL training paradigms which rely on a reference model. This significantly reduces memory pressure as an additional refreference model does not need to be loaded, and reference model log-probabilities can be obtained by disabling PEFT adapters. This is enabled by default. To turn it off, pass the following config:\n# load ref model when adapter training.\nrl_adapter_ref_model: true",
+    "text": "RLHF using Axolotl\n\n\n\n\n\n\nImportant\n\n\n\nThis is a BETA feature and many features are not fully implemented. You are encouraged to open new PRs to improve the integration and functionality.\n\n\nWe rely on the TRL library for implementations of various RL training methods, which we wrap around to expose in axolotl. Each method has their own supported ways of loading datasets and prompt formats.\n\n\n\n\n\n\nTip\n\n\n\nYou can find what each method supports by going into src/axolotl/prompt_strategies/{method} where {method} is one of our supported methods. The type: can be retrieved from {method}.{function_name}.\n\n\n\nDPO\nExample config:\nrl: dpo\ndatasets:\n  - path: Intel/orca_dpo_pairs\n    split: train\n    type: chatml.intel\n  - path: argilla/ultrafeedback-binarized-preferences\n    split: train\n    type: chatml\nDPO supports the following types with the following dataset format:\n\nchatml.argilla\n{\n    \"system\": \"...\", // optional\n    \"instruction\": \"...\",\n    \"chosen_response\": \"...\",\n    \"rejected_response\": \"...\"\n}\n\n\nchatml.argilla_chat\n{\n    \"chosen\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ],\n    \"rejected\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ]\n}\n\n\nchatml.icr\n{\n    \"system\": \"...\", // optional\n    \"input\": \"...\",\n    \"chosen\": \"...\",\n    \"rejected\": \"...\"\n}\n\n\nchatml.intel\n{\n    \"system\": \"...\", // optional\n    \"question\": \"...\",\n    \"chosen\": \"...\",\n    \"rejected\": \"...\"\n}\n\n\nchatml.prompt_pairs\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"chosen\": \"...\",\n    \"rejected\": \"...\"\n}\n\n\nchatml.ultra\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"chosen\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ],\n    \"rejected\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ]\n}\n\n\nllama3.argilla\n{\n    \"system\": \"...\", // optional\n    \"instruction\": \"...\",\n    \"chosen_response\": \"...\",\n    \"rejected_response\": \"...\"\n}\n\n\nllama3.argilla_chat\n{\n    \"chosen\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ],\n    \"rejected\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ]\n}\n\n\nllama3.icr\n{\n    \"system\": \"...\", // optional\n    \"input\": \"...\",\n    \"chosen\": \"...\",\n    \"rejected\": \"...\"\n}\n\n\nllama3.intel\n{\n    \"system\": \"...\", // optional\n    \"question\": \"...\",\n    \"chosen\": \"...\",\n    \"rejected\": \"...\"\n}\n\n\nllama3.prompt_pairs\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"chosen\": \"...\",\n    \"rejected\": \"...\"\n}\n\n\nllama3.ultra\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"chosen\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ],\n    \"rejected\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ]\n}\n\n\nzephyr.nectar\n{\n    \"prompt\": \"...\",\n    \"answers\": [\n        {\n            \"answer\": \"...\",\n            \"rank\": 1\n        },\n        {\n            \"answer\": \"...\",\n            \"rank\": 2\n        }\n        // ... more answers with ranks\n    ]\n}\n\n\nchat_template.default\nrl: dpo\ndatasets:\n  - path: ...\n    split: train\n    type: chat_template.default\n    field_messages: \"messages\"\n    field_chosen: \"chosen\"\n    field_rejected: \"rejected\"\n    message_property_mappings:\n      role: role\n      content: content\n    roles:\n      user: [\"user\"]\n      assistant: [\"assistant\"]\n      system: [\"system\"]\nSample input format:\n{\n    \"messages\": [\n        {\n            \"role\": \"system\",\n            \"content\": \"...\"\n        },\n        {\n            \"role\": \"user\",\n            \"content\": \"...\"\n        },\n        // ... more messages\n    ],\n    \"chosen\": {\n        \"role\": \"assistant\",\n        \"content\": \"...\"\n    },\n    \"rejected\": {\n        \"role\": \"assistant\",\n        \"content\": \"...\"\n    }\n}\n\n\nuser_defined.default\nFor custom behaviors,\nrl: dpo\ndatasets:\n  - path: ...\n    split: train\n    type: user_defined.default\n\n    field_prompt: \"prompt\"\n    field_system: \"system\"\n    field_chosen: \"chosen\"\n    field_rejected: \"rejected\"\n    prompt_format: \"{prompt}\"\n    chosen_format: \"{chosen}\"\n    rejected_format: \"{rejected}\"\nThe input format is a simple JSON input with customizable fields based on the above config.\n{\n    \"system\": \"...\",  // optional\n    \"prompt\": \"...\",\n    \"chosen\": \"...\",\n    \"rejected\": \"...\"\n}\n\n\n\nIPO\nAs IPO is just DPO with a different loss function, all supported options for DPO works here.\nrl: ipo\n\n\nORPO\nPaper: https://arxiv.org/abs/2403.07691\nrl: orpo\norpo_alpha: 0.1\nremove_unused_columns: false\n\nchat_template: chatml\ndatasets:\n  - path: argilla/ultrafeedback-binarized-preferences-cleaned\n    type: chat_template.argilla\nORPO supports the following types with the following dataset format:\n\nchat_template.argilla\n{\n    \"system\": \"...\",  // optional\n    \"prompt\": \"...\",  // if available, will be taken as user message for single-turn instead of from list below\n\n    // chosen/rejected should be same till last content and only even-number of alternating user/assistant turns\n    \"chosen\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ],\n    \"rejected\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ]\n}\n\n\n\nKTO\nrl: kto\nrl_beta: 0.5\nkto_desirable_weight: 0.2\n\nremove_unused_columns: false\n\ndatasets:\n  - path: argilla/ultrafeedback-binarized-preferences-cleaned-kto\n    type: llama3.ultra\n    split: train\n\ngradient_checkpointing: true\ngradient_checkpointing_kwargs:\n  use_reentrant: true\nKTO supports the following types with the following dataset format:\n\nchatml.argilla\n{\n    \"system\": \"...\", // optional\n    \"instruction\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nchatml.argilla_chat\n{\n    \"chosen\": [\n        {\"role\": \"user\", \"content\": \"...\"}\n    ],\n    \"completion\": [\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ]\n}\n\n\nchatml.intel\n{\n    \"system\": \"...\", // optional\n    \"question\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nchatml.prompt_pairs\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nchatml.ultra\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nllama3.argilla\n{\n    \"system\": \"...\", // optional\n    \"instruction\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nllama3.argilla_chat\n{\n    \"completion\": [\n        {\"role\": \"user\", \"content\": \"...\"},\n        {\"role\": \"assistant\", \"content\": \"...\"}\n    ]\n}\n\n\nllama3.intel\n{\n    \"system\": \"...\", // optional\n    \"question\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nllama3.prompt_pairs\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nllama3.ultra\n{\n    \"system\": \"...\", // optional\n    \"prompt\": \"...\",\n    \"completion\": \"...\"\n}\n\n\nuser_defined.default\nFor custom behaviors,\nrl: kto\ndatasets:\n  - path: ...\n    split: train\n    type: user_defined.default\n\n    field_prompt: \"prompt\"\n    field_system: \"system\"\n    field_completion: \"completion\"\n    field_label: \"label\"\n    prompt_format: \"{prompt}\"\n    completion_format: \"{completion}\"\nThe input format is a simple JSON input with customizable fields based on the above config.\n{\n    \"system\": \"...\",  // optional\n    \"prompt\": \"...\",\n    \"completion\": \"...\",\n    \"label\": \"...\"\n}\n\n\n\nGRPO\nGRPO uses custom reward functions and transformations. Please have them ready locally.\nFor ex, to load OpenAI’s GSM8K and use a random reward for completions:\n# rewards.py\nimport random\n\ndef rand_reward_func(completions, **kwargs) -&gt; list[float]:\n    return [random.uniform(0, 1) for _ in completions]\n\ndef oai_gsm8k_transform(cfg, *args, **kwargs):\n    def transform_fn(example, tokenizer=None):\n        label = example[\"answer\"].split(\"####\")[-1].strip().replace(\",\", \"\")\n        return {\n            \"prompt\": [{\"role\": \"user\", \"content\": example[\"question\"]},],\n            \"answer\": label,\n        }\n    return transform_fn, {\"remove_columns\": [\"question\"]}\nrl: grpo\n\ntrl:\n    beta: 0.001\n    max_completion_length: 256\n    use_vllm: True\n    vllm_device: auto\n    vllm_gpu_memory_utilization: 0.15\n    num_generations: 4\n    reward_funcs: [\"rewards.rand_reward_func\"]    # format: '{file_name}.{fn_name}'\n    reward_weights: [1.0]\ndatasets:\n  - path: openai/gsm8k\n    name: main\n    type: rewards.oai_gsm8k_transform  # format: '{file_name}.{fn_name}'\nTo see other examples of custom reward functions, please see TRL GRPO Docs.\nTo see description of the configs, please see TRLConfig.\n\n\nUsing local dataset files\ndatasets:\n  - ds_type: json\n    data_files:\n      - orca_rlhf.jsonl\n    split: train\n    type: chatml.intel\n\n\nTRL auto-unwrapping for PEFT\nTRL supports auto-unwrapping PEFT models for RL training paradigms which rely on a reference model. This significantly reduces memory pressure as an additional refreference model does not need to be loaded, and reference model log-probabilities can be obtained by disabling PEFT adapters. This is enabled by default. To turn it off, pass the following config:\n# load ref model when adapter training.\nrl_adapter_ref_model: true",
     "crumbs": [
       "How To Guides",
       "RLHF (Beta)"
diff --git a/sitemap.xml b/sitemap.xml
index 0e65903ea..f8fd10b87 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -2,162 +2,162 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/examples/colab-notebooks/colab-axolotl-example.html</loc>
-    <lastmod>2025-03-07T13:58:28.169Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.910Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/stepwise_supervised.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.906Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/template_free.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.906Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/tokenized.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.906Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/nccl.html</loc>
-    <lastmod>2025-03-07T13:58:28.168Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/amd_hpc.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.905Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/config.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.905Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/multi-gpu.html</loc>
-    <lastmod>2025-03-07T13:58:28.168Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/installation.html</loc>
-    <lastmod>2025-03-07T13:58:28.168Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/torchao.html</loc>
-    <lastmod>2025-03-07T13:58:28.169Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/reward_modelling.html</loc>
-    <lastmod>2025-03-07T13:58:28.169Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/input_output.html</loc>
-    <lastmod>2025-03-07T13:58:28.168Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/multimodal.html</loc>
-    <lastmod>2025-03-07T13:58:28.168Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/getting-started.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.906Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/inference.html</loc>
-    <lastmod>2025-03-07T13:58:28.168Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/multipack.html</loc>
-    <lastmod>2025-03-07T13:58:28.168Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/debugging.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.906Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/lr_groups.html</loc>
-    <lastmod>2025-03-07T13:58:28.168Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/TODO.html</loc>
-    <lastmod>2025-03-07T13:58:28.164Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.904Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/src/axolotl/integrations/LICENSE.html</loc>
-    <lastmod>2025-03-07T13:58:28.183Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.924Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/index.html</loc>
-    <lastmod>2025-03-07T13:58:28.180Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.921Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/src/axolotl/integrations/cut_cross_entropy/ACKNOWLEDGEMENTS.html</loc>
-    <lastmod>2025-03-07T13:58:28.184Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.924Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/FAQS.html</loc>
-    <lastmod>2025-03-07T13:58:28.163Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.904Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/multi-node.html</loc>
-    <lastmod>2025-03-07T13:58:28.168Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/faq.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.906Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/batch_vs_grad.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.905Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/lora_optims.html</loc>
-    <lastmod>2025-03-07T13:58:28.168Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/rlhf.html</loc>
-    <lastmod>2025-03-07T13:58:28.169Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/cli.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.905Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/unsloth.html</loc>
-    <lastmod>2025-03-07T13:58:28.169Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/fsdp_qlora.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.906Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/dataset_preprocessing.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.906Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/custom_integrations.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.905Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/mac.html</loc>
-    <lastmod>2025-03-07T13:58:28.168Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/docker.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.906Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/ray-integration.html</loc>
-    <lastmod>2025-03-07T13:58:28.169Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.909Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/index.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.906Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/conversation.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.905Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/pretraining.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.906Z</lastmod>
   </url>
   <url>
     <loc>https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/inst_tune.html</loc>
-    <lastmod>2025-03-07T13:58:28.165Z</lastmod>
+    <lastmod>2025-03-07T13:59:04.906Z</lastmod>
   </url>
 </urlset>