various fixes

fix loader default

.github/CONTRIBUTING.md

@@ -15,7 +15,7 @@ First of all, thank you for your interest in contributing to axolotl! We appreci
   - [Commit Messages](#commit-messages)
 - [Additional Resources](#additional-resources)
 
-## Code of Conduct
+## Code of Conductcode
 
 All contributors are expected to adhere to our [Code of Conduct](CODE_OF_CONDUCT.md). Please read it before participating in the axolotl community.
 

.github/workflows/base.yml

@@ -22,6 +22,18 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - cuda: "121"
+            cuda_version: 12.1.1
+            cudnn_version: 8
+            python_version: "3.10"
+            pytorch: 2.3.1
+            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
+          - cuda: "121"
+            cuda_version: 12.1.1
+            cudnn_version: 8
+            python_version: "3.11"
+            pytorch: 2.3.1
+            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
           - cuda: "124"
             cuda_version: 12.4.1
             cudnn_version: ""

.github/workflows/main.yml

@@ -15,6 +15,16 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - cuda: 121
+            cuda_version: 12.1.1
+            python_version: "3.10"
+            pytorch: 2.3.1
+            axolotl_extras: mamba-ssm
+          - cuda: 121
+            cuda_version: 12.1.1
+            python_version: "3.11"
+            pytorch: 2.3.1
+            axolotl_extras: mamba-ssm
           - cuda: 124
             cuda_version: 12.4.1
             python_version: "3.11"
@@ -72,6 +82,16 @@ jobs:
     strategy:
       matrix:
         include:
+          - cuda: 121
+            cuda_version: 12.1.1
+            python_version: "3.10"
+            pytorch: 2.3.1
+            axolotl_extras:
+          - cuda: 121
+            cuda_version: 12.1.1
+            python_version: "3.11"
+            pytorch: 2.3.1
+            axolotl_extras:
           - cuda: 124
             cuda_version: 12.4.1
             python_version: "3.11"
@@ -125,10 +145,10 @@ jobs:
     strategy:
       matrix:
         include:
-          - cuda: 124
-            cuda_version: 12.4.1
+          - cuda: 121
+            cuda_version: 12.1.1
             python_version: "3.11"
-            pytorch: 2.4.1
+            pytorch: 2.3.1
             axolotl_extras:
     runs-on: axolotl-gpu-runner
     steps:

.github/workflows/multi-gpu-e2e.yml

@@ -20,6 +20,12 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - cuda: 121
+            cuda_version: 12.1.1
+            python_version: "3.11"
+            pytorch: 2.3.1
+            axolotl_extras:
+            num_gpus: 2
           - cuda: 124
             cuda_version: 12.4.1
             python_version: "3.11"

.github/workflows/nightlies.yml

@@ -12,6 +12,17 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - cuda: 121
+            cuda_version: 12.1.1
+            python_version: "3.10"
+            pytorch: 2.3.1
+            axolotl_extras:
+          - cuda: 121
+            cuda_version: 12.1.1
+            python_version: "3.11"
+            pytorch: 2.3.1
+            axolotl_extras:
+            is_latest: true
           - cuda: 124
             cuda_version: 12.4.1
             python_version: "3.11"
@@ -65,6 +76,17 @@ jobs:
     strategy:
       matrix:
         include:
+          - cuda: 121
+            cuda_version: 12.1.1
+            python_version: "3.10"
+            pytorch: 2.3.1
+            axolotl_extras:
+          - cuda: 121
+            cuda_version: 12.1.1
+            python_version: "3.11"
+            pytorch: 2.3.1
+            axolotl_extras:
+            is_latest: true
           - cuda: 124
             cuda_version: 12.4.1
             python_version: "3.11"

.github/workflows/tests-nightly.yml

@@ -26,7 +26,7 @@ jobs:
       max-parallel: 2
       matrix:
         python_version: ["3.10", "3.11"]
-        pytorch_version: ["2.4.1", "2.5.1"]
+        pytorch_version: ["2.3.1", "2.4.1", "2.5.1"]
         exclude:
           - python_version: "3.10"
             pytorch_version: "2.4.1"
@@ -98,6 +98,13 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - cuda: 121
+            cuda_version: 12.1.1
+            python_version: "3.10"
+            pytorch: 2.3.1
+            num_gpus: 1
+            axolotl_extras: mamba-ssm
+            nightly_build: "true"
           - cuda: 124
             cuda_version: 12.4.1
             python_version: "3.11"

.github/workflows/tests.yml

@@ -49,7 +49,7 @@ jobs:
       max-parallel: 2
       matrix:
         python_version: ["3.10", "3.11"]
-        pytorch_version: ["2.4.1", "2.5.1"]
+        pytorch_version: ["2.3.1", "2.4.1", "2.5.1"]
         exclude:
           - python_version: "3.10"
             pytorch_version: "2.4.1"
@@ -228,7 +228,6 @@ jobs:
           echo "AXOLOTL_ARGS=${{ matrix.axolotl_args}}" >> $GITHUB_ENV
           echo "AXOLOTL_EXTRAS=${{ matrix.axolotl_extras}}" >> $GITHUB_ENV
           echo "CUDA=${{ matrix.cuda }}" >> $GITHUB_ENV
-          echo "MODAL_IMAGE_BUILDER_VERSION=2024.10" >> $GITHUB_ENV
           echo "N_GPUS=${{ matrix.num_gpus }}" >> $GITHUB_ENV
       - name: Run tests job on Modal
         run: |
@@ -245,6 +244,12 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - cuda: 121
+            cuda_version: 12.1.1
+            python_version: "3.10"
+            pytorch: 2.3.1
+            num_gpus: 1
+            axolotl_extras: mamba-ssm
           - cuda: 124
             cuda_version: 12.4.1
             python_version: "3.11"
@@ -269,7 +274,6 @@ jobs:
           echo "AXOLOTL_ARGS=${{ matrix.axolotl_args}}" >> $GITHUB_ENV
           echo "AXOLOTL_EXTRAS=${{ matrix.axolotl_extras}}" >> $GITHUB_ENV
           echo "CUDA=${{ matrix.cuda }}" >> $GITHUB_ENV
-          echo "MODAL_IMAGE_BUILDER_VERSION=2024.10" >> $GITHUB_ENV
           echo "N_GPUS=${{ matrix.num_gpus }}" >> $GITHUB_ENV
       - name: Run tests job on Modal
         run: |

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
     hooks:
       - id: isort
 -   repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
+    rev: 6.0.0
     hooks:
     - id: flake8
 -   repo: https://github.com/PyCQA/pylint

README.md

@@ -1,8 +1,8 @@
 <p align="center">
     <picture>
-        <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/887513285d98132142bf5db2a74eb5e0928787f1/image/axolotl_logo_digital_white.svg">
-        <source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/887513285d98132142bf5db2a74eb5e0928787f1/image/axolotl_logo_digital_black.svg">
-        <img alt="Axolotl" src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/887513285d98132142bf5db2a74eb5e0928787f1/image/axolotl_logo_digital_black.svg" width="400" height="104" style="max-width: 100%;">
+        <source media="(prefers-color-scheme: dark)" srcset="image/axolotl_logo_digital_white.svg">
+        <source media="(prefers-color-scheme: light)" srcset="image/axolotl_logo_digital_black.svg">
+        <img alt="Axolotl" src="image/axolotl_logo_digital_black.svg" width="400" height="104" style="max-width: 100%;">
     </picture>
 </p>
 
@@ -19,99 +19,235 @@
     <br/>
     <img src="https://github.com/axolotl-ai-cloud/axolotl/actions/workflows/tests-nightly.yml/badge.svg" alt="tests-nightly">
     <img src="https://github.com/axolotl-ai-cloud/axolotl/actions/workflows/multi-gpu-e2e.yml/badge.svg" alt="multigpu-semi-weekly tests">
-    <a href="https://www.phorm.ai/query?projectId=e315ba4a-4e14-421f-ab05-38a1f9076f25">
-    <img alt="phorm.ai" src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg?&logo=data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNSIgaGVpZ2h0PSI0IiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxwYXRoIGQ9Ik00LjQzIDEuODgyYTEuNDQgMS40NCAwIDAgMS0uMDk4LjQyNmMtLjA1LjEyMy0uMTE1LjIzLS4xOTIuMzIyLS4wNzUuMDktLjE2LjE2NS0uMjU1LjIyNmExLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxMmMtLjA5OS4wMTItLjE5Mi4wMTQtLjI3OS4wMDZsLTEuNTkzLS4xNHYtLjQwNmgxLjY1OGMuMDkuMDAxLjE3LS4xNjkuMjQ2LS4xOTFhLjYwMy42MDMgMCAwIDAgLjItLjEwNi41MjkuNTI5IDAgMCAwIC4xMzgtLjE3LjY1NC42NTQgMCAwIDAgLjA2NS0uMjRsLjAyOC0uMzJhLjkzLjkzIDAgMCAwLS4wMzYtLjI0OS41NjcuNTY3IDAgMCAwLS4xMDMtLjIuNTAyLjUwMiAwIDAgMC0uMTY4LS4xMzguNjA4LjYwOCAwIDAgMC0uMjQtLjA2N0wyLjQzNy43MjkgMS42MjUuNjcxYS4zMjIuMzIyIDAgMCAwLS4yMzIuMDU4LjM3NS4zNzUgMCAwIDAtLjExNi4yMzJsLS4xMTYgMS40NS0uMDU4LjY5Ny0uMDU4Ljc1NEwuNzA1IDRsLS4zNTctLjA3OUwuNjAyLjkwNkMuNjE3LjcyNi42NjMuNTc0LjczOS40NTRhLjk1OC45NTggMCAwIDEgLjI3NC0uMjg1Ljk3MS45NzEgMCAwIDEgLjMzNy0uMTRjLjExOS0uMDI2LjIyNy0uMDM0LjMyNS0uMDI2TDMuMjMyLjE2Yy4xNTkuMDE0LjMzNi4wMy40NTkuMDgyYTEuMTczIDEuMTczIDAgMCAxIC41NDUuNDQ3Yy4wNi4wOTQuMTA5LjE5Mi4xNDQuMjkzYTEuMzkyIDEuMzkyIDAgMCAxIC4wNzguNThsLS4wMjkuMzJaIiBmaWxsPSIjRjI3NzdBIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+Cjwvc3ZnPgo=">
-  </a>
 </p>
 
-Axolotl is a tool designed to streamline post-training for various AI models.
-Post-training refers to any modifications or additional training performed on
-pre-trained models - including full model fine-tuning, parameter-efficient tuning (like
-LoRA and QLoRA), supervised fine-tuning (SFT), instruction tuning, and alignment
-techniques. With support for multiple model architectures and training configurations,
-Axolotl makes it easy to get started with these techniques.
-
-Axolotl is designed to work with YAML config files that contain everything you need to
-preprocess a dataset, train or fine-tune a model, run model inference or evaluation,
-and much more.
+Axolotl is a tool designed to streamline the fine-tuning of various AI models, offering support for multiple configurations and architectures.
 
 Features:
-
 - Train various Huggingface models such as llama, pythia, falcon, mpt
 - Supports fullfinetune, lora, qlora, relora, and gptq
 - Customize configurations using a simple yaml file or CLI overwrite
 - Load different dataset formats, use custom formats, or bring your own tokenized datasets
-- Integrated with [xformers](https://github.com/facebookresearch/xformers), flash attention, [liger kernel](https://github.com/linkedin/Liger-Kernel), rope scaling, and multipacking
+- Integrated with xformer, flash attention, [liger kernel](https://github.com/linkedin/Liger-Kernel), rope scaling, and multipacking
 - Works with single GPU or multiple GPUs via FSDP or Deepspeed
 - Easily run with Docker locally or on the cloud
 - Log results and optionally checkpoints to wandb, mlflow or Comet
 - And more!
 
-## 🚀 Quick Start
+<a href="https://www.phorm.ai/query?projectId=e315ba4a-4e14-421f-ab05-38a1f9076f25">
+  <img alt="phorm.ai" src="https://img.shields.io/badge/Phorm-Ask_AI-%23F2777A.svg?&logo=data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNSIgaGVpZ2h0PSI0IiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxwYXRoIGQ9Ik00LjQzIDEuODgyYTEuNDQgMS40NCAwIDAgMS0uMDk4LjQyNmMtLjA1LjEyMy0uMTE1LjIzLS4xOTIuMzIyLS4wNzUuMDktLjE2LjE2NS0uMjU1LjIyNmExLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxMmMtLjA5OS4wMTItLjE5Mi4wMTQtLjI3OS4wMDZsLTEuNTkzLS4xNHYtLjQwNmgxLjY1OGMuMDkuMDAxLjE3LS4xNjkuMjQ2LS4xOTFhLjYwMy42MDMgMCAwIDAgLjItLjEwNi41MjkuNTI5IDAgMCAwIC4xMzgtLjE3LjY1NC42NTQgMCAwIDAgLjA2NS0uMjRsLjAyOC0uMzJhLjkzLjkzIDAgMCAwLS4wMzYtLjI0OS41NjcuNTY3IDAgMCAwLS4xMDMtLjIuNTAyLjUwMiAwIDAgMC0uMTY4LS4xMzguNjA4LjYwOCAwIDAgMC0uMjQtLjA2N0wyLjQzNy43MjkgMS42MjUuNjcxYS4zMjIuMzIyIDAgMCAwLS4yMzIuMDU4LjM3NS4zNzUgMCAwIDAtLjExNi4yMzJsLS4xMTYgMS40NS0uMDU4LjY5Ny0uMDU4Ljc1NEwuNzA1IDRsLS4zNTctLjA3OUwuNjAyLjkwNkMuNjE3LjcyNi42NjMuNTc0LjczOS40NTRhLjk1OC45NTggMCAwIDEgLjI3NC0uMjg1Ljk3MS45NzEgMCAwIDEgLjMzNy0uMTRjLjExOS0uMDI2LjIyNy0uMDM0LjMyNS0uMDI2TDMuMjMyLjE2Yy4xNTkuMDE0LjMzNi4wMy40NTkuMDgyYTEuMTczIDEuMTczIDAgMCAxIC41NDUuNDQ3Yy4wNi4wOTQuMTA5LjE5Mi4xNDQuMjkzYTEuMzkyIDEuMzkyIDAgMCAxIC4wNzguNThsLS4wMjkuMzJaIiBmaWxsPSIjRjI3NzdBIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+CiAgPHBhdGggZD0iTTQuMDgyIDIuMDA3YTEuNDU1IDEuNDU1IDAgMCAxLS4wOTguNDI3Yy0uMDUuMTI0LS4xMTQuMjMyLS4xOTIuMzI0YTEuMTMgMS4xMyAwIDAgMS0uMjU0LjIyNyAxLjM1MyAxLjM1MyAwIDAgMS0uNTk1LjIxNGMtLjEuMDEyLS4xOTMuMDE0LS4yOC4wMDZsLTEuNTYtLjEwOC4wMzQtLjQwNi4wMy0uMzQ4IDEuNTU5LjE1NGMuMDkgMCAuMTczLS4wMS4yNDgtLjAzM2EuNjAzLjYwMyAwIDAgMCAuMi0uMTA2LjUzMi41MzIgMCAwIDAgLjEzOS0uMTcyLjY2LjY2IDAgMCAwIC4wNjQtLjI0MWwuMDI5LS4zMjFhLjk0Ljk0IDAgMCAwLS4wMzYtLjI1LjU3LjU3IDAgMCAwLS4xMDMtLjIwMi41MDIuNTAyIDAgMCAwLS4xNjgtLjEzOC42MDUuNjA1IDAgMCAwLS4yNC0uMDY3TDEuMjczLjgyN2MtLjA5NC0uMDA4LS4xNjguMDEtLjIyMS4wNTUtLjA1My4wNDUtLjA4NC4xMTQtLjA5Mi4yMDZMLjcwNSA0IDAgMy45MzhsLjI1NS0yLjkxMUExLjAxIDEuMDEgMCAwIDEgLjM5My41NzIuOTYyLjk2MiAwIDAgMSAuNjY2LjI4NmEuOTcuOTcgMCAwIDEgLjMzOC0uMTRDMS4xMjIuMTIgMS4yMy4xMSAxLjMyOC4xMTlsMS41OTMuMTRjLjE2LjAxNC4zLjA0Ny40MjMuMWExLjE3IDEuMTcgMCAwIDEgLjU0NS40NDhjLjA2MS4wOTUuMTA5LjE5My4xNDQuMjk1YTEuNDA2IDEuNDA2IDAgMCAxIC4wNzcuNTgzbC0uMDI4LjMyMloiIGZpbGw9IndoaXRlIi8+Cjwvc3ZnPgo=">
+</a>
 
-**Requirements**:
-- NVIDIA GPU (Ampere or newer for `bf16` and Flash Attention) or AMD GPU
-- Python ≥3.10
-- PyTorch ≥2.4.1
+<table>
+<tr>
+<td>
 
-### Installation
+## Table of Contents
+- [Axolotl](#axolotl)
+  - [Table of Contents](#table-of-contents)
+  - [Quickstart ⚡](#quickstart-)
+    - [Edge Builds](#edge-builds-)
+    - [Axolotl CLI Usage](#axolotl-cli-usage)
+  - [Badge ❤🏷️](#badge-️)
+  - [Contributing 🤝](#contributing-)
+  - [Sponsors 🤝❤](#sponsors-)
+  - [Axolotl supports](#axolotl-supports)
+  - [Advanced Setup](#advanced-setup)
+    - [Environment](#environment)
+      - [Docker](#docker)
+      - [Conda/Pip venv](#condapip-venv)
+      - [Cloud GPU](#cloud-gpu)
+      - [Bare Metal Cloud GPU](#bare-metal-cloud-gpu)
+        - [LambdaLabs](#lambdalabs)
+        - [GCP](#gcp)
+      - [Windows](#windows)
+      - [Mac](#mac)
+      - [Google Colab](#google-colab)
+      - [Launching on public clouds via SkyPilot](#launching-on-public-clouds-via-skypilot)
+      - [Launching on public clouds via dstack](#launching-on-public-clouds-via-dstack)
+    - [Dataset](#dataset)
+    - [Config](#config)
+      - [All Config Options](#all-config-options)
+    - [Train](#train)
+      - [Preprocess dataset](#preprocess-dataset)
+      - [Multi-GPU](#multi-gpu)
+        - [DeepSpeed](#deepspeed)
+        - [FSDP](#fsdp)
+        - [FSDP + QLoRA](#fsdp--qlora)
+        - [Weights \& Biases Logging](#weights--biases-logging)
+        - [Special Tokens](#special-tokens)
+      - [Liger Kernel](#liger-kernel)
+    - [Inference Playground](#inference-playground)
+    - [Merge LORA to base](#merge-lora-to-base)
+  - [Common Errors 🧰](#common-errors-)
+    - [Tokenization Mismatch b/w Inference \& Training](#tokenization-mismatch-bw-inference--training)
+  - [Debugging Axolotl](#debugging-axolotl)
+  - [Need help? 🙋](#need-help-)
 
-```shell
+</td>
+<td>
+
+<div align="center">
+  <img src="image/axolotl_symbol_digital_white.svg" alt="axolotl" width="160">
+  <div>
+    <p>
+      <b>Axolotl provides a unified repository for fine-tuning <br />a variety of AI models with ease</b>
+    </p>
+    <p>
+      Go ahead and Axolotl questions!!
+    </p>
+    <img src="https://github.com/axolotl-ai-cloud/axolotl/actions/workflows/pre-commit.yml/badge.svg?branch=main" alt="pre-commit">
+    <img alt="PyTest Status" src="https://github.com/axolotl-ai-cloud/axolotl/actions/workflows/tests.yml/badge.svg?branch=main">
+  </div>
+</div>
+
+</td>
+</tr>
+</table>
+
+## Quickstart ⚡
+
+Get started with Axolotl in just a few steps! This quickstart guide will walk you through setting up and running a basic fine-tuning task.
+
+**Requirements**: *Nvidia* GPU (Ampere architecture or newer for `bf16` and Flash Attention) or *AMD* GPU, Python >=3.10 and PyTorch >=2.3.1.
+
+```bash
 pip3 install --no-build-isolation axolotl[flash-attn,deepspeed]
 
-# Download example axolotl configs, deepspeed configs
+# download examples and optionally deepspeed configs to the local path
 axolotl fetch examples
 axolotl fetch deepspeed_configs  # OPTIONAL
-```
 
-Other installation approaches are described [here](https://axolotl-ai-cloud.github.io/axolotl/docs/installation.html).
-
-### Your First Fine-tune
-
-```shell
-# Fetch axolotl examples
-axolotl fetch examples
-
-# Or, specify a custom path
-axolotl fetch examples --dest path/to/folder
-
-# Train a model using LoRA
+# finetune using lora
 axolotl train examples/llama-3/lora-1b.yml
 ```
 
-That's it! Check out our [Getting Started Guide](https://axolotl-ai-cloud.github.io/axolotl/docs/getting-started.html) for a more detailed walkthrough.
+### Edge Builds 🏎️
 
-## ✨ Key Features
+If you're looking for the latest features and updates between releases, you'll need to install
+from source.
 
-- **Multiple Model Support**: Train various models like LLaMA, Mistral, Mixtral, Pythia, and more
-- **Training Methods**: Full fine-tuning, LoRA, QLoRA, and more
-- **Easy Configuration**: Simple YAML files to control your training setup
-- **Performance Optimizations**: Flash Attention, xformers, multi-GPU training
-- **Flexible Dataset Handling**: Use various formats and custom datasets
-- **Cloud Ready**: Run on cloud platforms or local hardware
+```bash
+git clone https://github.com/axolotl-ai-cloud/axolotl.git
+cd axolotl
+pip3 install packaging ninja
+pip3 install --no-build-isolation -e '.[flash-attn,deepspeed]'
+```
 
-## 📚 Documentation
+### Axolotl CLI Usage
+We now support a new, more streamlined CLI using [click](https://click.palletsprojects.com/en/stable/).
 
-- [Installation Options](https://axolotl-ai-cloud.github.io/axolotl/docs/installation.html) - Detailed setup instructions for different environments
-- [Configuration Guide](https://axolotl-ai-cloud.github.io/axolotl/docs/config.html) - Full configuration options and examples
-- [Dataset Guide](https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/) - Supported formats and how to use them
-- [Multi-GPU Training](https://axolotl-ai-cloud.github.io/axolotl/docs/multi-gpu.html)
-- [Multi-Node Training](https://axolotl-ai-cloud.github.io/axolotl/docs/multi-node.html)
-- [Multipacking](https://axolotl-ai-cloud.github.io/axolotl/docs/multipack.html)
-- [FAQ](https://axolotl-ai-cloud.github.io/axolotl/docs/faq.html) - Frequently asked questions
+```bash
+# preprocess datasets - optional but recommended
+CUDA_VISIBLE_DEVICES="0" axolotl preprocess examples/llama-3/lora-1b.yml
 
-## 🤝 Getting Help
+# finetune lora
+axolotl train examples/llama-3/lora-1b.yml
 
-- Join our [Discord community](https://discord.gg/HhrNrHJPRb) for support
-- Check out our [Examples](https://github.com/axolotl-ai-cloud/axolotl/tree/main/examples/) directory
-- Read our [Debugging Guide](https://axolotl-ai-cloud.github.io/axolotl/docs/debugging.html)
-- Need dedicated support? Please contact [✉️wing@axolotl.ai](mailto:wing@axolotl.ai) for options
+# inference
+axolotl inference examples/llama-3/lora-1b.yml \
+    --lora-model-dir="./outputs/lora-out"
 
-## 🌟 Contributing
+# gradio
+axolotl inference examples/llama-3/lora-1b.yml \
+    --lora-model-dir="./outputs/lora-out" --gradio
 
-Contributions are welcome! Please see our [Contributing Guide](https://github.com/axolotl-ai-cloud/axolotl/blob/main/.github/CONTRIBUTING.md) for details.
+# remote yaml files - the yaml config can be hosted on a public URL
+# Note: the yaml config must directly link to the **raw** yaml
+axolotl train https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/examples/llama-3/lora-1b.yml
+```
 
-## Supported Models
+We've also added a new command for fetching `examples` and `deepspeed_configs` to your
+local machine. This will come in handy when installing `axolotl` from PyPI.
+
+```bash
+# Fetch example YAML files (stores in "examples/" folder)
+axolotl fetch examples
+
+# Fetch deepspeed config files (stores in "deepspeed_configs/" folder)
+axolotl fetch deepspeed_configs
+
+# Optionally, specify a destination folder
+axolotl fetch examples --dest path/to/folder
+```
+
+### Legacy Usage
+<details>
+
+<summary>Click to Expand</summary>
+
+While the Axolotl CLI is the preferred method for interacting with axolotl, we
+still support the legacy `-m axolotl.cli.*` usage.
+
+```bash
+# preprocess datasets - optional but recommended
+CUDA_VISIBLE_DEVICES="0" python -m axolotl.cli.preprocess examples/llama-3/lora-1b.yml
+
+# finetune lora
+accelerate launch -m axolotl.cli.train examples/llama-3/lora-1b.yml
+
+# inference
+accelerate launch -m axolotl.cli.inference examples/llama-3/lora-1b.yml \
+    --lora_model_dir="./outputs/lora-out"
+
+# gradio
+accelerate launch -m axolotl.cli.inference examples/llama-3/lora-1b.yml \
+    --lora_model_dir="./outputs/lora-out" --gradio
+
+# remote yaml files - the yaml config can be hosted on a public URL
+# Note: the yaml config must directly link to the **raw** yaml
+accelerate launch -m axolotl.cli.train https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/examples/llama-3/lora-1b.yml
+```
+
+</details>
+
+## Badge ❤🏷️
+
+Building something cool with Axolotl? Consider adding a badge to your model card.
+
+```markdown
+[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
+```
+
+[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
+
+## Sponsors 🤝❤
+
+If you love axolotl, consider sponsoring the project by reaching out directly to [wing@axolotl.ai](mailto:wing@axolotl.ai).
+
+---
+
+- [Modal](https://modal.com/) Modal lets you run data/AI jobs in the cloud, by just writing a few lines of Python. Customers use Modal to deploy Gen AI models at large scale, fine-tune LLM models, run protein folding simulations, and much more.
+
+---
+
+## Contributing 🤝
+
+Please read the [contributing guide](./.github/CONTRIBUTING.md)
+
+Bugs? Please check the [open issues](https://github.com/axolotl-ai-cloud/axolotl/issues/bug) else create a new Issue.
+
+PRs are **greatly welcome**!
+
+Please run the quickstart instructions followed by the below to setup env:
+```bash
+pip3 install -r requirements-dev.txt -r requirements-tests.txt
+pre-commit install
+
+# test
+pytest tests/
+
+# optional: run against all files
+pre-commit run --all-files
+```
+
+Thanks to all of our contributors to date. Help drive open source AI progress forward by contributing to Axolotl.
+
+<a href="https://github.com/axolotl-ai-cloud/axolotl/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=openaccess-ai-collective/axolotl" alt="contributor chart by https://contrib.rocks"/>
+</a>
+
+## Axolotl supports
 
 |             | fp16/fp32 | lora | qlora | gptq | gptq w/flash attn | flash attn | xformers attn |
 |-------------|:----------|:-----|-------|------|-------------------|------------|--------------|
@@ -136,16 +272,523 @@ Contributions are welcome! Please see our [Contributing Guide](https://github.co
 ❌: not supported
 ❓: untested
 
-## ❤️ Sponsors
+## Advanced Setup
 
-Thank you to our sponsors who help make Axolotl possible:
+### Environment
 
-- [Modal](https://www.modal.com?utm_source=github&utm_medium=github&utm_campaign=axolotl) - Modal lets you run
-jobs in the cloud, by just writing a few lines of Python. Customers use Modal to deploy Gen AI models at large scale,
-fine-tune large language models, run protein folding simulations, and much more.
+#### Docker
 
-Interested in sponsoring? Contact us at [wing@axolotl.ai](mailto:wing@axolotl.ai)
+  ```bash
+  docker run --gpus '"all"' --rm -it axolotlai/axolotl:main-latest
+  ```
 
-## 📜 License
+  Or run on the current files for development:
 
-This project is licensed under the Apache 2.0 License - see the [LICENSE](LICENSE) file for details.
+  ```sh
+  docker compose up -d
+  ```
+
+>[!Tip]
+> If you want to debug axolotl or prefer to use Docker as your development environment, see the [debugging guide's section on Docker](docs/debugging.qmd#debugging-with-docker).
+
+  <details>
+
+  <summary>Docker advanced</summary>
+
+  A more powerful Docker command to run would be this:
+
+  ```bash
+docker run --privileged --gpus '"all"' --shm-size 10g --rm -it --name axolotl --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 --mount type=bind,src="${PWD}",target=/workspace/axolotl -v ${HOME}/.cache/huggingface:/root/.cache/huggingface axolotlai/axolotl:main-latest
+  ```
+
+  It additionally:
+  * Prevents memory issues when running e.g. deepspeed (e.g. you could hit SIGBUS/signal 7 error) through `--ipc` and `--ulimit` args.
+  * Persists the downloaded HF data (models etc.) and your modifications to axolotl code through `--mount`/`-v` args.
+  * The `--name` argument simply makes it easier to refer to the container in vscode (`Dev Containers: Attach to Running Container...`) or in your terminal.
+  * The `--privileged` flag gives all capabilities to the container.
+  * The `--shm-size 10g` argument increases the shared memory size. Use this if you see `exitcode: -7` errors using deepspeed.
+
+  [More information on nvidia website](https://docs.nvidia.com/deeplearning/frameworks/user-guide/index.html#setincshmem)
+
+  </details>
+
+#### Conda/Pip venv
+  1. Install python >=**3.10**
+
+  2. Install pytorch stable https://pytorch.org/get-started/locally/
+
+  3. Install Axolotl along with python dependencies
+        ```bash
+        pip3 install packaging
+        pip3 install --no-build-isolation -e '.[flash-attn,deepspeed]'
+        ```
+  4. (Optional) Login to Huggingface to use gated models/datasets.
+        ```bash
+        huggingface-cli login
+        ```
+        Get the token at huggingface.co/settings/tokens
+
+#### Cloud GPU
+
+For cloud GPU providers that support docker images, use [`axolotlai/axolotl-cloud:main-latest`](https://hub.docker.com/r/axolotlai/axolotl-cloud/tags)
+
+- on Latitude.sh use this [direct link](https://latitude.sh/blueprint/989e0e79-3bf6-41ea-a46b-1f246e309d5c)
+- on JarvisLabs.ai use this [direct link](https://jarvislabs.ai/templates/axolotl)
+- on RunPod use this [direct link](https://runpod.io/gsc?template=v2ickqhz9s&ref=6i7fkpdz)
+
+#### Bare Metal Cloud GPU
+
+##### LambdaLabs
+
+  <details>
+
+  <summary>Click to Expand</summary>
+
+  1. Install python
+  ```bash
+  sudo apt update
+  sudo apt install -y python3.10
+
+  sudo update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1
+  sudo update-alternatives --config python # pick 3.10 if given option
+  python -V # should be 3.10
+
+  ```
+
+  2. Install pip
+  ```bash
+  wget https://bootstrap.pypa.io/get-pip.py
+  python get-pip.py
+  ```
+
+  3. Install Pytorch https://pytorch.org/get-started/locally/
+
+  4. Follow instructions on quickstart.
+
+  5. Run
+  ```bash
+  pip3 install protobuf==3.20.3
+  pip3 install -U --ignore-installed requests Pillow psutil scipy
+  ```
+
+  6. Set path
+  ```bash
+  export LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
+  ```
+  </details>
+
+##### GCP
+
+<details>
+
+<summary>Click to Expand</summary>
+
+Use a Deeplearning linux OS with cuda and pytorch installed. Then follow instructions on quickstart.
+
+Make sure to run the below to uninstall xla.
+```bash
+pip uninstall -y torch_xla[tpu]
+```
+
+</details>
+
+#### Windows
+Please use WSL or Docker!
+
+#### Mac
+
+Use the below instead of the install method in QuickStart.
+```
+pip3 install --no-build-isolation -e '.'
+```
+More info: [mac.md](/docs/mac.qmd)
+
+#### Google Colab
+
+Please use this example [notebook](examples/colab-notebooks/colab-axolotl-example.ipynb).
+
+#### Launching on public clouds via SkyPilot
+To launch on GPU instances (both on-demand and spot instances) on 7+ clouds (GCP, AWS, Azure, OCI, and more), you can use [SkyPilot](https://skypilot.readthedocs.io/en/latest/index.html):
+
+```bash
+pip install "skypilot-nightly[gcp,aws,azure,oci,lambda,kubernetes,ibm,scp]"  # choose your clouds
+sky check
+```
+
+Get the [example YAMLs](https://github.com/skypilot-org/skypilot/tree/master/llm/axolotl) of using Axolotl to finetune `mistralai/Mistral-7B-v0.1`:
+```
+git clone https://github.com/skypilot-org/skypilot.git
+cd skypilot/llm/axolotl
+```
+
+Use one command to launch:
+```bash
+# On-demand
+HF_TOKEN=xx sky launch axolotl.yaml --env HF_TOKEN
+
+# Managed spot (auto-recovery on preemption)
+HF_TOKEN=xx BUCKET=<unique-name> sky spot launch axolotl-spot.yaml --env HF_TOKEN --env BUCKET
+```
+
+#### Launching on public clouds via dstack
+To launch on GPU instance (both on-demand and spot instances) on public clouds (GCP, AWS, Azure, Lambda Labs, TensorDock, Vast.ai, and CUDO), you can use [dstack](https://dstack.ai/).
+
+Write a job description in YAML as below:
+
+```yaml
+# dstack.yaml
+type: task
+
+image: axolotlai/axolotl-cloud:main-latest
+
+env:
+  - HUGGING_FACE_HUB_TOKEN
+  - WANDB_API_KEY
+
+commands:
+  - accelerate launch -m axolotl.cli.train config.yaml
+
+ports:
+  - 6006
+
+resources:
+  gpu:
+    memory: 24GB..
+    count: 2
+```
+
+then, simply run the job with `dstack run` command. Append `--spot` option if you want spot instance. `dstack run` command will show you the instance with cheapest price across multi cloud services:
+
+```bash
+pip install dstack
+HUGGING_FACE_HUB_TOKEN=xxx WANDB_API_KEY=xxx dstack run . -f dstack.yaml # --spot
+```
+
+For further and fine-grained use cases, please refer to the official [dstack documents](https://dstack.ai/docs/) and the detailed description of [axolotl example](https://github.com/dstackai/dstack/tree/master/examples/fine-tuning/axolotl) on the official repository.
+
+### Dataset
+
+Axolotl supports a variety of dataset formats.  It is recommended to use a JSONL.  The schema of the JSONL depends upon the task and the prompt template you wish to use.  Instead of a JSONL, you can also use a HuggingFace dataset with columns for each JSONL field.
+
+See [the documentation](https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/) for more information on how to use different dataset formats.
+
+### Config
+
+See [examples](examples) for quick start. It is recommended to duplicate and modify to your needs. The most important options are:
+
+- model
+  ```yaml
+  base_model: ./llama-7b-hf # local or huggingface repo
+  ```
+  Note: The code will load the right architecture.
+
+- dataset
+  ```yaml
+  datasets:
+      # huggingface repo
+    - path: vicgalle/alpaca-gpt4
+      type: alpaca
+
+      # huggingface repo with specific configuration/subset
+    - path: EleutherAI/pile
+      name: enron_emails
+      type: completion # format from earlier
+      field: text # Optional[str] default: text, field to use for completion data
+
+      # huggingface repo with multiple named configurations/subsets
+    - path: bigcode/commitpackft
+      name:
+        - ruby
+        - python
+        - typescript
+      type: ... # unimplemented custom format
+
+      # chat_template https://axolotl-ai-cloud.github.io/axolotl/docs/dataset-formats/conversation.html#chat_template
+    - path: ...
+      type: chat_template
+      chat_template: chatml # defaults to tokenizer's chat_template
+
+      # local
+    - path: data.jsonl # or json
+      ds_type: json # see other options below
+      type: alpaca
+
+      # dataset with splits, but no train split
+    - path: knowrohit07/know_sql
+      type: context_qa.load_v2
+      train_on_split: validation
+
+      # loading from s3 or gcs
+      # s3 creds will be loaded from the system default and gcs only supports public access
+    - path: s3://path_to_ds # Accepts folder with arrow/parquet or file path like above. Supports s3, gcs.
+      ...
+
+      # Loading Data From a Public URL
+      # - The file format is `json` (which includes `jsonl`) by default. For different formats, adjust the `ds_type` option accordingly.
+    - path: https://some.url.com/yourdata.jsonl # The URL should be a direct link to the file you wish to load. URLs must use HTTPS protocol, not HTTP.
+      ds_type: json # this is the default, see other options below.
+  ```
+
+- loading
+  ```yaml
+  load_in_4bit: true
+  load_in_8bit: true
+
+  bf16: auto # require >=ampere, auto will detect if your GPU supports this and choose automatically.
+  fp16: # leave empty to use fp16 when bf16 is 'auto'. set to false if you want to fallback to fp32
+  tf32: true # require >=ampere
+
+  bfloat16: true # require >=ampere, use instead of bf16 when you don't want AMP (automatic mixed precision)
+  float16: true # use instead of fp16 when you don't want AMP
+  ```
+  Note: Repo does not do 4-bit quantization.
+
+- lora
+  ```yaml
+  adapter: lora # 'qlora' or leave blank for full finetune
+  lora_r: 8
+  lora_alpha: 16
+  lora_dropout: 0.05
+  lora_target_modules:
+    - q_proj
+    - v_proj
+  ```
+
+#### All Config Options
+
+See [these docs](docs/config.qmd) for all config options.
+
+### Train
+
+Run
+```bash
+accelerate launch -m axolotl.cli.train your_config.yml
+```
+
+> [!TIP]
+> You can also reference a config file that is hosted on a public URL, for example `accelerate launch -m axolotl.cli.train https://yourdomain.com/your_config.yml`
+
+#### Preprocess dataset
+
+You can optionally pre-tokenize dataset with the following before finetuning.
+This is recommended for large datasets.
+
+- Set `dataset_prepared_path:` to a local folder for saving and loading pre-tokenized dataset.
+- (Optional): Set `push_dataset_to_hub: hf_user/repo` to push it to Huggingface.
+- (Optional): Use `--debug` to see preprocessed examples.
+
+```bash
+python -m axolotl.cli.preprocess your_config.yml
+```
+
+#### Multi-GPU
+
+Below are the options available in axolotl for training with multiple GPUs. Note that DeepSpeed
+is the recommended multi-GPU option currently because FSDP may experience
+[loss instability](https://github.com/huggingface/transformers/issues/26498).
+
+##### DeepSpeed
+
+Deepspeed is an optimization suite for multi-gpu systems allowing you to train much larger models than you
+might typically be able to fit into your GPU's VRAM. More information about the various optimization types
+for deepspeed is available at https://huggingface.co/docs/accelerate/main/en/usage_guides/deepspeed#what-is-integrated
+
+We provide several default deepspeed JSON configurations for ZeRO stage 1, 2, and 3.
+
+```yaml
+deepspeed: deepspeed_configs/zero1.json
+```
+
+```shell
+accelerate launch -m axolotl.cli.train examples/llama-2/config.yml --deepspeed deepspeed_configs/zero1.json
+```
+
+##### FSDP
+
+- llama FSDP
+```yaml
+fsdp:
+  - full_shard
+  - auto_wrap
+fsdp_config:
+  fsdp_offload_params: true
+  fsdp_state_dict_type: FULL_STATE_DICT
+  fsdp_transformer_layer_cls_to_wrap: LlamaDecoderLayer
+```
+
+##### FSDP + QLoRA
+
+Axolotl supports training with FSDP and QLoRA, see [these docs](docs/fsdp_qlora.qmd) for more information.
+
+##### Weights & Biases Logging
+
+Make sure your `WANDB_API_KEY` environment variable is set (recommended) or you login to wandb with `wandb login`.
+
+- wandb options
+```yaml
+wandb_mode:
+wandb_project:
+wandb_entity:
+wandb_watch:
+wandb_name:
+wandb_log_model:
+```
+
+##### Comet Logging
+
+Make sure your `COMET_API_KEY` environment variable is set (recommended) or you login to wandb with `comet login`.
+
+- wandb options
+```yaml
+use_comet:
+comet_api_key:
+comet_workspace:
+comet_project_name:
+comet_experiment_key:
+comet_mode:
+comet_online:
+comet_experiment_config:
+```
+
+##### Special Tokens
+
+It is important to have special tokens like delimiters, end-of-sequence, beginning-of-sequence in your tokenizer's vocabulary.  This will help you avoid tokenization issues and help your model train better.  You can do this in axolotl like this:
+
+```yml
+special_tokens:
+  bos_token: "<s>"
+  eos_token: "</s>"
+  unk_token: "<unk>"
+tokens: # these are delimiters
+  - "<|im_start|>"
+  - "<|im_end|>"
+```
+
+When you include these tokens in your axolotl config, axolotl adds these tokens to the tokenizer's vocabulary.
+
+##### Liger Kernel
+
+Liger Kernel: Efficient Triton Kernels for LLM Training
+
+https://github.com/linkedin/Liger-Kernel
+
+Liger (LinkedIn GPU Efficient Runtime) Kernel is a collection of Triton kernels designed specifically for LLM training.
+It can effectively increase multi-GPU training throughput by 20% and reduces memory usage by 60%. The Liger Kernel
+composes well and is compatible with both FSDP and Deepspeed.
+
+```yaml
+plugins:
+  - axolotl.integrations.liger.LigerPlugin
+liger_rope: true
+liger_rms_norm: true
+liger_glu_activation: true
+liger_layer_norm: true
+liger_fused_linear_cross_entropy: true
+```
+
+### Inference Playground
+
+Axolotl allows you to load your model in an interactive terminal playground for quick experimentation.
+The config file is the same config file used for training.
+
+Pass the appropriate flag to the inference command, depending upon what kind of model was trained:
+
+- Pretrained LORA:
+  ```bash
+  python -m axolotl.cli.inference examples/your_config.yml --lora_model_dir="./lora-output-dir"
+  ```
+- Full weights finetune:
+  ```bash
+  python -m axolotl.cli.inference examples/your_config.yml --base_model="./completed-model"
+  ```
+- Full weights finetune w/ a prompt from a text file:
+  ```bash
+  cat /tmp/prompt.txt | python -m axolotl.cli.inference examples/your_config.yml \
+    --base_model="./completed-model" --prompter=None --load_in_8bit=True
+  ```
+-- With gradio hosting
+  ```bash
+  python -m axolotl.cli.inference examples/your_config.yml --gradio
+  ```
+
+Please use `--sample_packing False` if you have it on and receive the error similar to below:
+
+> RuntimeError: stack expects each tensor to be equal size, but got [1, 32, 1, 128] at entry 0 and [1, 32, 8, 128] at entry 1
+
+### Merge LORA to base
+
+The following command will merge your LORA adapater with your base model. You can optionally pass the argument `--lora_model_dir` to specify the directory where your LORA adapter was saved, otherwhise, this will be inferred from `output_dir` in your axolotl config file.  The merged model is saved in the sub-directory `{lora_model_dir}/merged`.
+
+```bash
+python3 -m axolotl.cli.merge_lora your_config.yml --lora_model_dir="./completed-model"
+```
+
+You may need to use the `gpu_memory_limit` and/or `lora_on_cpu` config options to avoid running out of memory. If you still run out of CUDA memory, you can try to merge in system RAM with
+
+```bash
+CUDA_VISIBLE_DEVICES="" python3 -m axolotl.cli.merge_lora ...
+```
+
+although this will be very slow, and using the config options above are recommended instead.
+
+## Common Errors 🧰
+
+See also the [FAQ's](./docs/faq.qmd) and [debugging guide](docs/debugging.qmd).
+
+> If you encounter a 'Cuda out of memory' error, it means your GPU ran out of memory during the training process. Here's how to resolve it:
+
+Please reduce any below
+  - `micro_batch_size`
+  - `eval_batch_size`
+  - `gradient_accumulation_steps`
+  - `sequence_len`
+
+If it does not help, try running without deepspeed and without accelerate (replace "accelerate launch" with "python") in the command.
+
+Using adamw_bnb_8bit might also save you some memory.
+
+> `failed (exitcode: -9)`
+
+Usually means your system has run out of system memory.
+Similarly, you should consider reducing the same settings as when you run out of VRAM.
+Additionally, look into upgrading your system RAM which should be simpler than GPU upgrades.
+
+> RuntimeError: expected scalar type Float but found Half
+
+Try set `fp16: true`
+
+> NotImplementedError: No operator found for `memory_efficient_attention_forward` ...
+
+Try to turn off xformers.
+
+> accelerate config missing
+
+It's safe to ignore it.
+
+> NCCL Timeouts during training
+
+See the [NCCL](docs/nccl.qmd) guide.
+
+
+### Tokenization Mismatch b/w Inference & Training
+
+For many formats, Axolotl constructs prompts by concatenating token ids _after_ tokenizing strings.  The reason for concatenating token ids rather than operating on strings is to maintain precise accounting for attention masks.
+
+If you decode a prompt constructed by axolotl, you might see spaces between tokens (or lack thereof) that you do not expect, especially around delimiters and special tokens.  When you are starting out with a new format, you should always do the following:
+
+1. Materialize some data using `python -m axolotl.cli.preprocess your_config.yml --debug`, and then decode the first few rows with your model's tokenizer.
+2. During inference, right before you pass a tensor of token ids to your model, decode these tokens back into a string.
+3. Make sure the inference string from #2 looks **exactly** like the data you fine tuned on from #1, including spaces and new lines.  If they aren't the same, adjust your inference server accordingly.
+4. As an additional troubleshooting step, you can look at the token ids between 1 and 2 to make sure they are identical.
+
+Having misalignment between your prompts during training and inference can cause models to perform very poorly, so it is worth checking this.  See [this blog post](https://hamel.dev/notes/llm/finetuning/05_tokenizer_gotchas.html) for a concrete example.
+
+## Debugging Axolotl
+
+See [this debugging guide](docs/debugging.qmd) for tips on debugging Axolotl, along with an example configuration for debugging with VSCode.
+
+## Need help? 🙋
+
+Join our [Discord server](https://discord.gg/HhrNrHJPRb) where our community members can help you.
+
+Need dedicated support? Please contact us at [✉️wing@axolotl.ai](ailto:wing@axolotl.ai) for dedicated support options.

_quarto.yml

@@ -28,21 +28,16 @@ website:
         - section: "How-To Guides"
           contents:
           # TODO Edit folder structure after we have more docs.
-            - docs/getting-started.qmd
-            - docs/installation.qmd
             - docs/debugging.qmd
-            - docs/inference.qmd
             - docs/multipack.qmd
             - docs/fsdp_qlora.qmd
             - docs/input_output.qmd
             - docs/rlhf.qmd
             - docs/nccl.qmd
             - docs/mac.qmd
-            - docs/multi-gpu.qmd
             - docs/multi-node.qmd
             - docs/unsloth.qmd
             - docs/amd_hpc.qmd
-            - docs/ray-integration.qmd
         - section: "Dataset Formats"
           contents: docs/dataset-formats/*
         - section: "Reference"
@@ -50,6 +45,7 @@ website:
             - docs/config.qmd
         - docs/faq.qmd
 
+
 format:
   html:
     theme: materia

cicd/Dockerfile.jinja

@@ -32,9 +32,9 @@ RUN if [ "$NIGHTLY_BUILD" = "true" ] ; then \
     fi
 
 RUN if [ "$AXOLOTL_EXTRAS" != "" ] ; then \
-        pip install --no-build-isolation -e .[deepspeed,flash-attn,optimizers,ray,vllm,$AXOLOTL_EXTRAS] $AXOLOTL_ARGS; \
+        pip install --no-build-isolation -e .[deepspeed,flash-attn,optimizers,$AXOLOTL_EXTRAS] $AXOLOTL_ARGS; \
     else \
-        pip install --no-build-isolation -e .[deepspeed,flash-attn,optimizers,ray,vllm] $AXOLOTL_ARGS; \
+        pip install --no-build-isolation -e .[deepspeed,flash-attn,optimizers] $AXOLOTL_ARGS; \
     fi
 
 RUN python scripts/unsloth_install.py | sh

cicd/cicd.sh

@@ -6,6 +6,5 @@ python -c "import torch; assert '$PYTORCH_VERSION' in torch.__version__"
 pytest -v --durations=10 -n8 --ignore=tests/e2e/ --ignore=tests/patched/ /workspace/axolotl/tests/
 # pytest -v --durations=10 -n8 --dist loadfile /workspace/axolotl/tests/patched/
 pytest -v --durations=10 /workspace/axolotl/tests/e2e/patched/
-pytest -v --durations=10 -n1 /workspace/axolotl/tests/e2e/solo/
 pytest -v --durations=10 /workspace/axolotl/tests/e2e/integrations/
-pytest -v --durations=10 --ignore=tests/e2e/solo/ --ignore=tests/e2e/patched/ --ignore=tests/e2e/multigpu/ --ignore=tests/e2e/integrations/ /workspace/axolotl/tests/e2e/
+pytest -v --durations=10 --ignore=tests/e2e/patched/ --ignore=tests/e2e/multigpu/ --ignore=tests/e2e/integrations/ /workspace/axolotl/tests/e2e/

cicd/multigpu.py

@@ -23,8 +23,8 @@ df_template = template_env.get_template("Dockerfile.jinja")
 df_args = {
     "AXOLOTL_EXTRAS": os.environ.get("AXOLOTL_EXTRAS", ""),
     "AXOLOTL_ARGS": os.environ.get("AXOLOTL_ARGS", ""),
-    "PYTORCH_VERSION": os.environ.get("PYTORCH_VERSION", "2.4.1"),
-    "BASE_TAG": os.environ.get("BASE_TAG", "main-base-py3.11-cu121-2.4.1"),
+    "PYTORCH_VERSION": os.environ.get("PYTORCH_VERSION", "2.3.1"),
+    "BASE_TAG": os.environ.get("BASE_TAG", "main-base-py3.11-cu121-2.3.1"),
     "CUDA": os.environ.get("CUDA", "121"),
     "GITHUB_REF": os.environ.get("GITHUB_REF", "refs/heads/main"),
     "GITHUB_SHA": os.environ.get("GITHUB_SHA", ""),

cicd/tests.py

@@ -23,8 +23,8 @@ df_template = template_env.get_template("Dockerfile.jinja")
 df_args = {
     "AXOLOTL_EXTRAS": os.environ.get("AXOLOTL_EXTRAS", ""),
     "AXOLOTL_ARGS": os.environ.get("AXOLOTL_ARGS", ""),
-    "PYTORCH_VERSION": os.environ.get("PYTORCH_VERSION", "2.4.1"),
-    "BASE_TAG": os.environ.get("BASE_TAG", "main-base-py3.11-cu121-2.4.1"),
+    "PYTORCH_VERSION": os.environ.get("PYTORCH_VERSION", "2.3.1"),
+    "BASE_TAG": os.environ.get("BASE_TAG", "main-base-py3.11-cu121-2.3.1"),
     "CUDA": os.environ.get("CUDA", "121"),
     "GITHUB_REF": os.environ.get("GITHUB_REF", "refs/heads/main"),
     "GITHUB_SHA": os.environ.get("GITHUB_SHA", ""),
@@ -38,12 +38,16 @@ temp_dir = tempfile.mkdtemp()
 with open(pathlib.Path(temp_dir) / "Dockerfile", "w", encoding="utf-8") as f:
     f.write(dockerfile_contents)
 
-cicd_image = Image.from_dockerfile(
-    pathlib.Path(temp_dir) / "Dockerfile",
-    context_mount=None,
-    force_build=True,
-    gpu="A10G",
-).env(df_args)
+cicd_image = (
+    Image.from_dockerfile(
+        pathlib.Path(temp_dir) / "Dockerfile",
+        context_mount=None,
+        force_build=True,
+        gpu="A10G",
+    )
+    .env(df_args)
+    .pip_install("fastapi==0.110.0", "pydantic==2.6.3")
+)
 
 app = App("Axolotl CI/CD", secrets=[])
 

docker/Dockerfile

@@ -20,9 +20,9 @@ WORKDIR /workspace/axolotl
 
 # If AXOLOTL_EXTRAS is set, append it in brackets
 RUN if [ "$AXOLOTL_EXTRAS" != "" ] ; then \
-        pip install --no-build-isolation -e .[deepspeed,flash-attn,optimizers,ray,vllm,$AXOLOTL_EXTRAS] $AXOLOTL_ARGS; \
+        pip install --no-build-isolation -e .[deepspeed,flash-attn,optimizers,$AXOLOTL_EXTRAS] $AXOLOTL_ARGS; \
     else \
-        pip install --no-build-isolation -e .[deepspeed,flash-attn,optimizers,ray,vllm] $AXOLOTL_ARGS; \
+        pip install --no-build-isolation -e .[deepspeed,flash-attn,optimizers] $AXOLOTL_ARGS; \
     fi
 
 RUN python scripts/unsloth_install.py | sh

docker/Dockerfile-cloud

@@ -20,8 +20,7 @@ RUN apt install --yes --no-install-recommends openssh-server tmux && \
     printf "\n[[ -z \"\$TMUX\"  ]] && { tmux attach-session -t ssh_tmux || tmux new-session -s ssh_tmux; exit; }\n" >> ~/.bashrc && \
     printf "[ ! -z \"\$TERM\" -a -r /etc/motd ] && cat /etc/motd\n" >> ~/.bashrc && \
     chmod +x /workspace/axolotl/scripts/cloud-entrypoint.sh && \
-    chmod +x /root/cloud-entrypoint.sh && \
-    echo 'set-option -g history-limit 5000' >> ~/.tmux.conf
+    chmod +x /root/cloud-entrypoint.sh
 
 ENTRYPOINT ["/root/cloud-entrypoint.sh"]
 CMD ["sleep", "infinity"]

docs/cli.qmd

@@ -1,256 +0,0 @@
-# Axolotl CLI Documentation
-
-The Axolotl CLI provides a streamlined interface for training and fine-tuning large language models. This guide covers
-the CLI commands, their usage, and common examples.
-
-### Table of Contents
-
-- Basic Commands
-- Command Reference
-  - fetch
-  - preprocess
-  - train
-  - inference
-  - merge-lora
-  - merge-sharded-fsdp-weights
-  - evaluate
-  - lm-eval
-- Legacy CLI Usage
-- Remote Compute with Modal Cloud
-  - Cloud Configuration
-  - Running on Modal Cloud
-  - Cloud Configuration Options
-
-
-### Basic Commands
-
-All Axolotl commands follow this general structure:
-
-```bash
-axolotl <command> [config.yml] [options]
-```
-
-The config file can be local or a URL to a raw YAML file.
-
-### Command Reference
-
-#### fetch
-
-Downloads example configurations and deepspeed configs to your local machine.
-
-```bash
-# Get example YAML files
-axolotl fetch examples
-
-# Get deepspeed config files
-axolotl fetch deepspeed_configs
-
-# Specify custom destination
-axolotl fetch examples --dest path/to/folder
-```
-
-#### preprocess
-
-Preprocesses and tokenizes your dataset before training. This is recommended for large datasets.
-
-```bash
-# Basic preprocessing
-axolotl preprocess config.yml
-
-# Preprocessing with one GPU
-CUDA_VISIBLE_DEVICES="0" axolotl preprocess config.yml
-
-# Debug mode to see processed examples
-axolotl preprocess config.yml --debug
-
-# Debug with limited examples
-axolotl preprocess config.yml --debug --debug-num-examples 5
-```
-
-Configuration options:
-
-```yaml
-dataset_prepared_path: Local folder for saving preprocessed data
-push_dataset_to_hub: HuggingFace repo to push preprocessed data (optional)
-```
-
-#### train
-
-Trains or fine-tunes a model using the configuration specified in your YAML file.
-
-```bash
-# Basic training
-axolotl train config.yml
-
-# Train and set/override specific options
-axolotl train config.yml \
-    --learning-rate 1e-4 \
-    --micro-batch-size 2 \
-    --num-epochs 3
-
-# Training without accelerate
-axolotl train config.yml --no-accelerate
-
-# Resume training from checkpoint
-axolotl train config.yml --resume-from-checkpoint path/to/checkpoint
-```
-
-#### inference
-
-Runs inference using your trained model in either CLI or Gradio interface mode.
-
-```bash
-# CLI inference with LoRA
-axolotl inference config.yml --lora-model-dir="./outputs/lora-out"
-
-# CLI inference with full model
-axolotl inference config.yml --base-model="./completed-model"
-
-# Gradio web interface
-axolotl inference config.yml --gradio \
-    --lora-model-dir="./outputs/lora-out"
-
-# Inference with input from file
-cat prompt.txt | axolotl inference config.yml \
-    --base-model="./completed-model"
-```
-
-#### merge-lora
-
-Merges trained LoRA adapters into the base model.
-
-```bash
-# Basic merge
-axolotl merge-lora config.yml
-
-# Specify LoRA directory (usually used with checkpoints)
-axolotl merge-lora config.yml --lora-model-dir="./lora-output/checkpoint-100"
-
-# Merge using CPU (if out of GPU memory)
-CUDA_VISIBLE_DEVICES="" axolotl merge-lora config.yml
-```
-
-Configuration options:
-
-```yaml
-gpu_memory_limit: Limit GPU memory usage
-lora_on_cpu: Load LoRA weights on CPU
-```
-
-#### merge-sharded-fsdp-weights
-
-Merges sharded FSDP model checkpoints into a single combined checkpoint.
-
-```bash
-# Basic merge
-axolotl merge-sharded-fsdp-weights config.yml
-```
-
-#### evaluate
-
-Evaluates a model's performance using metrics specified in the config.
-
-```bash
-# Basic evaluation
-axolotl evaluate config.yml
-```
-
-#### lm-eval
-
-Runs LM Evaluation Harness on your model.
-
-```bash
-# Basic evaluation
-axolotl lm-eval config.yml
-
-# Evaluate specific tasks
-axolotl lm-eval config.yml --tasks arc_challenge,hellaswag
-```
-
-Configuration options:
-
-```yaml
-lm_eval_tasks: List of tasks to evaluate
-lm_eval_batch_size: Batch size for evaluation
-output_dir: Directory to save evaluation results
-```
-
-### Legacy CLI Usage
-
-While the new Click-based CLI is preferred, Axolotl still supports the legacy module-based CLI:
-
-```bash
-# Preprocess
-python -m axolotl.cli.preprocess config.yml
-
-# Train
-accelerate launch -m axolotl.cli.train config.yml
-
-# Inference
-accelerate launch -m axolotl.cli.inference config.yml \
-    --lora_model_dir="./outputs/lora-out"
-
-# Gradio interface
-accelerate launch -m axolotl.cli.inference config.yml \
-    --lora_model_dir="./outputs/lora-out" --gradio
-```
-
-### Remote Compute with Modal Cloud
-
-Axolotl supports running training and inference workloads on Modal cloud infrastructure. This is configured using a
-cloud YAML file alongside your regular Axolotl config.
-
-#### Cloud Configuration
-
-Create a cloud config YAML with your Modal settings:
-
-```yaml
-# cloud_config.yml
-provider: modal
-gpu: a100  # Supported: l40s, a100-40gb, a100-80gb, a10g, h100, t4, l4
-gpu_count: 1    # Number of GPUs to use
-timeout: 86400  # Maximum runtime in seconds (24 hours)
-branch: main    # Git branch to use (optional)
-
-volumes:        # Persistent storage volumes
-  - name: axolotl-cache
-    mount: /workspace/cache
-
-env:            # Environment variables
-  - WANDB_API_KEY
-  - HF_TOKEN
-```
-
-#### Running on Modal Cloud
-
-Commands that support the --cloud flag:
-
-```bash
-# Preprocess on cloud
-axolotl preprocess config.yml --cloud cloud_config.yml
-
-# Train on cloud
-axolotl train config.yml --cloud cloud_config.yml
-
-# Train without accelerate on cloud
-axolotl train config.yml --cloud cloud_config.yml --no-accelerate
-
-# Run lm-eval on cloud
-axolotl lm-eval config.yml --cloud cloud_config.yml
-```
-
-#### Cloud Configuration Options
-
-```yaml
-provider: compute provider, currently only `modal` is supported
-gpu: GPU type to use
-gpu_count: Number of GPUs (default: 1)
-memory: RAM in GB (default: 128)
-timeout: Maximum runtime in seconds
-timeout_preprocess: Preprocessing timeout
-branch: Git branch to use
-docker_tag: Custom Docker image tag
-volumes: List of persistent storage volumes
-env: Environment variables to pass
-secrets: Secrets to inject
-```

docs/config.qmd

@@ -187,12 +187,6 @@ rl:
 # whether to perform weighting if doing DPO training. Boolean.
 dpo_use_weighting:
 
-# reward modelling: `True` or `False`
-reward_model:
-
-# process reward modelling: `True` or `False`
-process_reward_model:
-
 # The name of the chat template to use for training, following values are supported:
 # - tokenizer_default: Uses the chat template that is available in the tokenizer_config.json. If the chat template is not available in the tokenizer, it will raise an error. This is the default value.
 # - alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates are available in the axolotl codebase at src/axolotl/utils/chat_templates.py
@@ -250,8 +244,6 @@ total_num_tokens:
 sample_packing_group_size: 100000
 # The number of samples which can be packed into one sequence. Increase if using a large sequence_len with many short samples.
 sample_packing_bin_size: 200
-# whether to concatenate samples during pretraining
-pretraining_sample_concatenation:
 
 # Use batch flattening for speedups when not using sample_packing
 batch_flattening:
@@ -366,11 +358,10 @@ warmup_ratio: 0.05  # cannot use with warmup_steps
 learning_rate: 0.00003
 lr_quadratic_warmup:
 logging_steps:
-eval_steps: # Leave empty to eval at each epoch, integer for every N steps. float for fraction of total steps
+eval_steps: # Leave empty to eval at each epoch, integers for every N steps. decimal for fraction of total steps
 evals_per_epoch: # number of times per epoch to run evals, mutually exclusive with eval_steps
-eval_strategy: # Set to `"no"` to skip evaluation, `"epoch"` at end of each epoch, leave empty to infer from `eval_steps`.
-save_strategy: # Set to `"no"` to skip checkpoint saves, `"epoch"` at end of each epoch, `"best"` when better result is achieved, leave empty to infer from `save_steps`.
-save_steps: # Leave empty to save at each epoch, integer for every N steps. float for fraction of total steps
+save_strategy: # Set to `"no"` to skip checkpoint saves
+save_steps: # Leave empty to save at each epoch
 saves_per_epoch: # number of times per epoch to save a checkpoint, mutually exclusive with save_steps
 save_total_limit: # Checkpoints saved at a time
 # Maximum number of iterations to train for. It precedes num_epochs which means that

docs/dataset-formats/conversation.qmd

@@ -8,12 +8,14 @@ order: 3
 
 IMPORTANT: ShareGPT is deprecated!. Please see `chat_template` section below.
 
+
 ## pygmalion
 
 ```{.json filename="data.jsonl"}
 {"conversations": [{"role": "...", "value": "..."}]}
 ```
 
+
 ## chat_template
 
 Chat Template strategy uses a jinja2 template that converts a list of messages into a prompt. Support using tokenizer's template, a supported template, or custom jinja2.

docs/dataset-formats/stepwise_supervised.qmd

@@ -1,26 +0,0 @@
----
-title: Stepwise Supervised Format
-description: Format for datasets with stepwise completions and labels
-order: 3
----
-
-## Stepwise Supervised
-
-The stepwise supervised format is designed for chain-of-thought (COT) reasoning
-datasets where each example contains multiple completion steps and a preference label
-for each step.
-
-### Example
-
-Here's a simple example of a stepwise supervised dataset entry:
-
-```json
-{
-  "prompt": "Which number is larger, 9.8 or 9.11?",
-  "completions": [
-    "The fractional part of 9.8 is 0.8, while the fractional part of 9.11 is 0.11.",
-    "Since 0.11 is greater than 0.8, the number 9.11 is larger than 9.8."
-  ],
-  "labels": [true, false]
-}
-```

docs/getting-started.qmd

@@ -1,155 +0,0 @@
----
-title: "Getting Started with Axolotl"
-format:
-  html:
-    toc: true
-    toc-depth: 3
-    number-sections: true
-execute:
-  enabled: false
----
-
-This guide will walk you through your first model fine-tuning project with Axolotl.
-
-## Quick Example {#sec-quick-example}
-
-Let's start by fine-tuning a small language model using LoRA. This example uses a 1B parameter model to ensure it runs on most GPUs.
-Assuming `axolotl` is installed (if not, see our [Installation Guide](installation.qmd))
-
-1. Download example configs:
-```shell
-axolotl fetch examples
-```
-
-2. Run the training:
-```shell
-axolotl train examples/llama-3/lora-1b.yml
-```
-
-That's it! Let's understand what just happened.
-
-## Understanding the Process {#sec-understanding}
-
-### The Configuration File {#sec-config}
-
-The YAML configuration file controls everything about your training. Here's what (part of) our example config looks like:
-
-```yaml
-base_model: NousResearch/Llama-3.2-1B
-# hub_model_id: username/custom_model_name
-
-datasets:
-  - path: teknium/GPT4-LLM-Cleaned
-    type: alpaca
-dataset_prepared_path: last_run_prepared
-val_set_size: 0.1
-output_dir: ./outputs/lora-out
-
-adapter: lora
-lora_model_dir:
-```
-
-See our [Config options](config.qmd) for more details.
-
-### Training {#sec-training}
-
-When you run `axolotl train`, Axolotl:
-
-1. Downloads the base model
-2. (If specified) applies LoRA adapter layers
-3. Loads and processes the dataset
-4. Runs the training loop
-5. Saves the trained model and / or LoRA weights
-
-## Your First Custom Training {#sec-custom}
-
-Let's modify the example for your own data:
-
-1. Create a new config file `my_training.yml`:
-
-```yaml
-base_model: NousResearch/Nous-Hermes-llama-1b-v1
-adapter: lora
-
-# Training settings
-micro_batch_size: 2
-num_epochs: 3
-learning_rate: 0.0003
-
-# Your dataset
-datasets:
-  - path: my_data.jsonl        # Your local data file
-    type: alpaca               # Or other format
-```
-
-This specific config is for LoRA fine-tuning a model with instruction tuning data using
-the `alpaca` dataset format, which has the following format:
-
-```json
-{
-    "instruction": "Write a description of alpacas.",
-    "input": "",
-    "output": "Alpacas are domesticated South American camelids..."
-}
-```
-
-Please see our [Dataset Formats](dataset-formats) for more dataset formats and how to
-format them.
-
-2. Prepare your JSONL data in the specified format (in this case, the expected `alpaca
-format):
-
-```json
-{"instruction": "Classify this text", "input": "I love this!", "output": "positive"}
-{"instruction": "Classify this text", "input": "Not good at all", "output": "negative"}
-```
-
-Please consult the supported [Dataset Formats](dataset-formats/) for more details.
-
-3. Run the training:
-
-```shell
-axolotl train my_training.yml
-```
-
-## Common Tasks {#sec-common-tasks}
-
-### Testing Your Model {#sec-testing}
-
-After training, test your model:
-
-```shell
-axolotl inference my_training.yml --lora-model-dir="./outputs/lora-out"
-```
-
-### Preprocessing Data {#sec-preprocessing}
-
-For large datasets, preprocess first:
-
-```shell
-axolotl preprocess my_training.yml
-```
-
-### Using a UI {#sec-ui}
-
-Launch a Gradio interface:
-
-```shell
-axolotl inference my_training.yml --lora-model-dir="./outputs/lora-out" --gradio
-```
-
-## Next Steps {#sec-next-steps}
-
-Now that you have the basics, you might want to:
-
-- Try different model architectures
-- Experiment with hyperparameters
-- Use more advanced training methods
-- Scale up to larger models
-
-Check our other guides for details on these topics:
-
-- [Configuration Guide](config.qmd) - Full configuration options
-- [Dataset Formats](dataset-formats) - Working with different data formats
-- [Multi-GPU Training](multi-gpu.qmd)
-- [Multi-Node Training](multi-node.qmd)

docs/inference.qmd

@@ -1,148 +0,0 @@
----
-title: "Inference Guide"
-format:
-  html:
-    toc: true
-    toc-depth: 3
-    number-sections: true
-    code-tools: true
-execute:
-  enabled: false
----
-
-This guide covers how to use your trained models for inference, including model loading, interactive testing, and common troubleshooting steps.
-
-## Quick Start {#sec-quickstart}
-
-### Basic Inference {#sec-basic}
-
-::: {.panel-tabset}
-
-## LoRA Models
-
-```{.bash}
-axolotl inference your_config.yml --lora-model-dir="./lora-output-dir"
-```
-
-## Full Fine-tuned Models
-
-```{.bash}
-axolotl inference your_config.yml --base-model="./completed-model"
-```
-
-:::
-
-## Advanced Usage {#sec-advanced}
-
-### Gradio Interface {#sec-gradio}
-
-Launch an interactive web interface:
-
-```{.bash}
-axolotl inference your_config.yml --gradio
-```
-
-### File-based Prompts {#sec-file-prompts}
-
-Process prompts from a text file:
-
-```{.bash}
-cat /tmp/prompt.txt | axolotl inference your_config.yml \
-  --base-model="./completed-model" --prompter=None
-```
-
-### Memory Optimization {#sec-memory}
-
-For large models or limited memory:
-
-```{.bash}
-axolotl inference your_config.yml --load-in-8bit=True
-```
-
-## Merging LoRA Weights {#sec-merging}
-
-Merge LoRA adapters with the base model:
-
-```{.bash}
-axolotl merge-lora your_config.yml --lora-model-dir="./completed-model"
-```
-
-### Memory Management for Merging {#sec-memory-management}
-
-::: {.panel-tabset}
-
-## Configuration Options
-
-```{.yaml}
-gpu_memory_limit: 20GiB  # Adjust based on your GPU
-lora_on_cpu: true        # Process on CPU if needed
-```
-
-## Force CPU Merging
-
-```{.bash}
-CUDA_VISIBLE_DEVICES="" axolotl merge-lora ...
-```
-
-:::
-
-## Tokenization {#sec-tokenization}
-
-### Common Issues {#sec-tokenization-issues}
-
-::: {.callout-warning}
-Tokenization mismatches between training and inference are a common source of problems.
-:::
-
-To debug:
-
-1. Check training tokenization:
-```{.bash}
-axolotl preprocess your_config.yml --debug
-```
-
-2. Verify inference tokenization by decoding tokens before model input
-
-3. Compare token IDs between training and inference
-
-### Special Tokens {#sec-special-tokens}
-
-Configure special tokens in your YAML:
-
-```{.yaml}
-special_tokens:
-  bos_token: "<s>"
-  eos_token: "</s>"
-  unk_token: "<unk>"
-tokens:
-  - "<|im_start|>"
-  - "<|im_end|>"
-```
-
-## Troubleshooting {#sec-troubleshooting}
-
-### Common Problems {#sec-common-problems}
-
-::: {.panel-tabset}
-
-## Memory Issues
-
-- Use 8-bit loading
-- Reduce batch sizes
-- Try CPU offloading
-
-## Token Issues
-
-- Verify special tokens
-- Check tokenizer settings
-- Compare training and inference preprocessing
-
-## Performance Issues
-
-- Verify model loading
-- Check prompt formatting
-- Ensure temperature/sampling settings
-
-:::
-
-For more details, see our [debugging guide](debugging.qmd).

docs/installation.qmd

@@ -1,119 +0,0 @@
----
-title: "Installation Guide"
-format:
-  html:
-    toc: true
-    toc-depth: 3
-    number-sections: true
-    code-tools: true
-execute:
-  enabled: false
----
-
-This guide covers all the ways you can install and set up Axolotl for your environment.
-
-## Requirements {#sec-requirements}
-
-- NVIDIA GPU (Ampere architecture or newer for `bf16` and Flash Attention) or AMD GPU
-- Python ≥3.10
-- PyTorch ≥2.4.1
-
-## Installation Methods {#sec-installation-methods}
-
-### PyPI Installation (Recommended) {#sec-pypi}
-
-```{.bash}
-pip3 install --no-build-isolation axolotl[flash-attn,deepspeed]
-```
-
-We use `--no-build-isolation` in order to detect the installed PyTorch version (if
-installed) in order not to clobber it, and so that we set the correct version of
-dependencies that are specific to the PyTorch version or other installed
-co-dependencies.
-
-### Edge/Development Build {#sec-edge-build}
-
-For the latest features between releases:
-
-```{.bash}
-git clone https://github.com/axolotl-ai-cloud/axolotl.git
-cd axolotl
-pip3 install packaging ninja
-pip3 install --no-build-isolation -e '.[flash-attn,deepspeed]'
-```
-
-### Docker {#sec-docker}
-
-```{.bash}
-docker run --gpus '"all"' --rm -it axolotlai/axolotl:main-latest
-```
-
-For development with Docker:
-
-```{.bash}
-docker compose up -d
-```
-
-::: {.callout-tip}
-### Advanced Docker Configuration
-```{.bash}
-docker run --privileged --gpus '"all"' --shm-size 10g --rm -it \
-  --name axolotl --ipc=host \
-  --ulimit memlock=-1 --ulimit stack=67108864 \
-  --mount type=bind,src="${PWD}",target=/workspace/axolotl \
-  -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \
-  axolotlai/axolotl:main-latest
-```
-:::
-
-## Cloud Environments {#sec-cloud}
-
-### Cloud GPU Providers {#sec-cloud-gpu}
-
-For providers supporting Docker:
-
-- Use `axolotlai/axolotl-cloud:main-latest`
-- Available on:
-  - [Latitude.sh](https://latitude.sh/blueprint/989e0e79-3bf6-41ea-a46b-1f246e309d5c)
-  - [JarvisLabs.ai](https://jarvislabs.ai/templates/axolotl)
-  - [RunPod](https://runpod.io/gsc?template=v2ickqhz9s&ref=6i7fkpdz)
-
-### Google Colab {#sec-colab}
-
-Use our [example notebook](../examples/colab-notebooks/colab-axolotl-example.ipynb).
-
-## Platform-Specific Instructions {#sec-platform-specific}
-
-### macOS {#sec-macos}
-
-```{.bash}
-pip3 install --no-build-isolation -e '.'
-```
-
-See @sec-troubleshooting for Mac-specific issues.
-
-### Windows {#sec-windows}
-
-::: {.callout-important}
-We recommend using WSL2 (Windows Subsystem for Linux) or Docker.
-:::
-
-## Environment Managers {#sec-env-managers}
-
-### Conda/Pip venv {#sec-conda}
-
-1. Install Python ≥3.10
-2. Install PyTorch: https://pytorch.org/get-started/locally/
-3. Install Axolotl:
-   ```{.bash}
-   pip3 install packaging
-   pip3 install --no-build-isolation -e '.[flash-attn,deepspeed]'
-   ```
-4. (Optional) Login to Hugging Face:
-   ```{.bash}
-   huggingface-cli login
-   ```
-
-## Troubleshooting {#sec-troubleshooting}
-
-If you encounter installation issues, see our [FAQ](faq.qmd) and [Debugging Guide](debugging.qmd).

docs/lr_groups.qmd

@@ -1,29 +0,0 @@
----
-title: Learning Rate Groups
-description: "Setting different learning rates by module name"
----
-
-## Background
-
-Inspired by LoRA+, Axolotl allows practitioners to specify separate learning rates for each module or groups of
-modules in a model.
-
-## Example
-
-```yaml
-lr_groups:
-  - name: o_proj
-    modules:
-      - self_attn.o_proj.weight
-    lr: 1e-6
-  - name: q_proj
-    modules:
-      - model.layers.2.self_attn.q_proj.weight
-    lr: 1e-5
-
-learning_rate: 2e-5
-```
-
-In this example, we have a default learning rate of 2e-5 across the entire model, but we have a separate learning rate
-of 1e-6 for all the self attention `o_proj` modules across all layers, and a learning are of 1e-5 to the 3rd layer's
-self attention `q_proj` module.

docs/multi-gpu.qmd

@@ -1,118 +0,0 @@
----
-title: "Multi-GPU Training Guide"
-format:
-  html:
-    toc: true
-    toc-depth: 3
-    number-sections: true
-    code-tools: true
-execute:
-  enabled: false
----
-
-This guide covers advanced training configurations for multi-GPU setups using Axolotl.
-
-## Overview {#sec-overview}
-
-Axolotl supports several methods for multi-GPU training:
-
-- DeepSpeed (recommended)
-- FSDP (Fully Sharded Data Parallel)
-- FSDP + QLoRA
-
-## DeepSpeed {#sec-deepspeed}
-
-DeepSpeed is the recommended approach for multi-GPU training due to its stability and performance. It provides various optimization levels through ZeRO stages.
-
-### Configuration {#sec-deepspeed-config}
-
-Add to your YAML config:
-
-```{.yaml}
-deepspeed: deepspeed_configs/zero1.json
-```
-
-### Usage {#sec-deepspeed-usage}
-
-```{.bash}
-accelerate launch -m axolotl.cli.train examples/llama-2/config.yml --deepspeed deepspeed_configs/zero1.json
-```
-
-### ZeRO Stages {#sec-zero-stages}
-
-We provide default configurations for:
-
-- ZeRO Stage 1 (`zero1.json`)
-- ZeRO Stage 2 (`zero2.json`)
-- ZeRO Stage 3 (`zero3.json`)
-
-Choose based on your memory requirements and performance needs.
-
-## FSDP {#sec-fsdp}
-
-### Basic FSDP Configuration {#sec-fsdp-config}
-
-```{.yaml}
-fsdp:
-  - full_shard
-  - auto_wrap
-fsdp_config:
-  fsdp_offload_params: true
-  fsdp_state_dict_type: FULL_STATE_DICT
-  fsdp_transformer_layer_cls_to_wrap: LlamaDecoderLayer
-```
-
-### FSDP + QLoRA {#sec-fsdp-qlora}
-
-For combining FSDP with QLoRA, see our [dedicated guide](fsdp_qlora.qmd).
-
-## Performance Optimization {#sec-performance}
-
-### Liger Kernel Integration {#sec-liger}
-
-::: {.callout-note}
-Liger Kernel provides efficient Triton kernels for LLM training, offering:
-
-- 20% increase in multi-GPU training throughput
-- 60% reduction in memory usage
-- Compatibility with both FSDP and DeepSpeed
-:::
-
-Configuration:
-
-```{.yaml}
-plugins:
-  - axolotl.integrations.liger.LigerPlugin
-liger_rope: true
-liger_rms_norm: true
-liger_glu_activation: true
-liger_layer_norm: true
-liger_fused_linear_cross_entropy: true
-```
-
-## Troubleshooting {#sec-troubleshooting}
-
-### NCCL Issues {#sec-nccl}
-
-For NCCL-related problems, see our [NCCL troubleshooting guide](nccl.qmd).
-
-### Common Problems {#sec-common-problems}
-
-::: {.panel-tabset}
-
-## Memory Issues
-
-- Reduce `micro_batch_size`
-- Reduce `eval_batch_size`
-- Adjust `gradient_accumulation_steps`
-- Consider using a higher ZeRO stage
-
-## Training Instability
-
-- Start with DeepSpeed ZeRO-2
-- Monitor loss values
-- Check learning rates
-
-:::
-
-For more detailed troubleshooting, see our [debugging guide](debugging.qmd).

docs/ray-integration.qmd

@@ -1,93 +0,0 @@
----
-title: Ray Train integration
-description: How to use Axolotl with Ray Train
----
-
-Axolotl supports using Ray as an alternative to `accelerate` for orchestrating training. This is especially useful for multi-node training since you only have to setup code and dependencies in a single node and launch training as if you were using a single node.
-
-With the `--use-ray` CLI flag, Axolotl will use Ray Train's [`TorchTrainer`](https://docs.ray.io/en/latest/train/api/doc/ray.train.torch.TorchTrainer.html#ray.train.torch.TorchTrainer) to run training.
-
-## Ray cluster setup
-
-A prerequisite using the Ray Train integration is to setup a Ray cluster on your desired node(s). For a detailed guide on how you can get started with ray clusters, check the official Ray docs here: https://docs.ray.io/en/latest/cluster/getting-started.html
-
-Every Ray cluster has one _head_ node and a set of worker nodes. The head node is just like any other worker node, but it also runs certain special processes related to scheduling and orchestration. Ray-enabled scripts are run on the head node and depending on the resources (number of CPUs, GPUs, etc) they request, will be scheduled to run certain tasks on the worker nodes. For more on key concepts behind a Ray cluster, you can refer this [doc](https://docs.ray.io/en/latest/cluster/key-concepts.html#cluster-key-concepts).
-
-## Sanity check
-
-To run a sanity check on whether your ray cluster is setup properly, execute the following on the head node:
-
-```bash
-ray status
-```
-
-The output should have a summary of your Ray cluster - list of all the nodes in your cluster, the number of CPUs and GPUs in your cluster, etc. For example, if you have a cluster with 1 CPU-only head node and 2 4xL40S worker nodes, the output can look like this:
-
-
-```
-Node status
----------------------------------------------------------------
-Active:
- 1 head
-Idle:
- 2 4xL40S:48CPU-384GB
-Pending:
- (no pending nodes)
-Recent failures:
- (no failures)
-
-Resources
----------------------------------------------------------------
-Usage:
- 0.0/96.0 CPU
- 0.0/8.0 GPU
- 0B/800.00GiB memory
- 0B/229.57GiB object_store_memory
-
-Demands:
- (no resource demands)
-```
-
-You should also be able to see the same on the [Ray dashboard](https://docs.ray.io/en/latest/ray-observability/getting-started.html).
-
-
-## Configuring training with Ray Train
-
-You can find an example configuration at `configs/llama-3/lora-1b-ray.yaml`.
-
-The key parameters to note here are:
-
-```yaml
-...
-use_ray: true
-ray_num_workers: 4
-# optional
-resources_per_worker:
-    GPU: 1
-...
-```
-
-- `use_ray`: This is the flag that enables the Ray Train integration. You can either use the corresponding `--use-ray` flag in the CLI or set `use_ray` in the config file.
-- `ray_num_workers`: This is the number of workers/GPUs to use for training.
-- `resources_per_worker`: This is the Ray [resource request](https://docs.ray.io/en/latest/ray-core/scheduling/resources.html) for each worker. This can be used to request a specific GPU type or a custom resource for each worker. For example, if your ray cluster has GPUs of different types, and you only want to use NVIDIA L40S GPUs, you can do
-
-```yaml
-resources_per_worker:
-    accelerator_type:L40S: 0.001
-```
-
-## Launching training
-
-You can simply run the following command on the head node:
-
-```bash
-axolotl train examples/llama-3/lora-1b-ray.yml --use-ray
-```
-
-This will launch training on the head node and workers will be scheduled automatically by Ray Train to run on the appropriate head or worker nodes.
-
-You can also monitor training progress on the Ray dashboard.
-
-Coming back to the example on a Ray cluster with 1 head node and 2 4xL40S worker nodes, let's say you want to make use of all 8 GPUs. You would be able to just set `ray_num_workers: 8` and run the previous command. The Cluster tab will show the following:
-
-![Ray dashboard](./images/ray-cluster-dashboard.png)

docs/reward_modelling.qmd

@@ -1,47 +0,0 @@
----
-title: "Reward Modelling"
-description: "Reward models are used to guide models towards behaviors which is preferred by humans, by training over large datasets annotated with human preferences. "
----
-
-### Overview
-
-Reward modelling is a technique used to train models to predict the reward or value of a given input. This is particularly useful in reinforcement learning scenarios where the model needs to evaluate the quality of its actions or predictions.
-We support the reward modelling techniques supported by `trl`.
-
-### (Outcome) Reward Models
-
-Outcome reward models are trained using data which contains preference annotations for an entire interaction between the user and model (e.g. rather than per-turn or per-step).
-
-```yaml
-base_model: google/gemma-2-2b
-model_type: AutoModelForSequenceClassification
-num_labels: 1
-tokenizer_type: AutoTokenizer
-
-reward_model: true
-chat_template: gemma
-datasets:
-  - path: argilla/distilabel-intel-orca-dpo-pairs
-    type: bradley_terry.chat_template
-
-val_set_size: 0.1
-eval_steps: 100
-```
-
-### Process Reward Models (PRM)
-
-Process reward models are trained using data which contains preference annotations for each step in a series of interactions. Typically, PRMs are trained to provide reward signals over each step of a reasoning trace and are used for downstream reinforcement learning.
-```yaml
-base_model: Qwen/Qwen2.5-3B
-model_type: AutoModelForTokenClassification
-num_labels: 2
-
-process_reward_model: true
-datasets:
-  - path: trl-lib/math_shepherd
-    type: stepwise_supervised
-    split: train
-
-val_set_size: 0.1
-eval_steps: 100
-```

examples/cerebras/btlm-ft.yml

@@ -46,7 +46,7 @@ output_dir: ./outputs/btlm-out
 gradient_accumulation_steps: 1
 micro_batch_size: 1
 num_epochs: 1
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 adam_beta2: 0.95
 adam_eps: 0.000000001
 max_grad_norm: 1.0

examples/cloud/modal.yaml

@@ -1,28 +0,0 @@
-project_name:
-volumes:
-  - name: axolotl-data
-    mount: /workspace/data
-  - name: axolotl-artifacts
-    mount: /workspace/artifacts
-
-# environment variables from local to set as secrets
-secrets:
-  - HF_TOKEN
-  - WANDB_API_KEY
-
-# Which branch of axolotl to use remotely
-branch:
-
-# additional custom commands when building the image
-dockerfile_commands:
-
-gpu: h100
-gpu_count: 1
-
-# Train specific configurations
-memory: 128
-timeout: 86400
-
-# Preprocess specific configurations
-memory_preprocess: 32
-timeout_preprocess: 14400

examples/deepseek-v2/fft-fsdp-16b.yaml

@@ -27,7 +27,7 @@ wandb_log_model:
 gradient_accumulation_steps: 8
 micro_batch_size: 1
 num_epochs: 1
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 2e-5
 

examples/deepseek-v2/qlora-fsdp-2_5.yaml

@@ -47,7 +47,7 @@ peft_use_rslora: true
 gradient_accumulation_steps: 1
 micro_batch_size: 8
 num_epochs: 1
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 2e-5
 

examples/gemma2/reward-model.yaml

@@ -1,7 +1,6 @@
 base_model: google/gemma-2-2b
 # optionally might have model_type or tokenizer_type
 model_type: AutoModelForSequenceClassification
-num_labels: 1
 tokenizer_type: AutoTokenizer
 # Automatically upload checkpoint and final model to HF
 # hub_model_id: username/custom_model_name

examples/jamba/qlora_fsdp_large.yaml

@@ -34,7 +34,7 @@ lora_target_linear: false
 gradient_accumulation_steps: 4
 micro_batch_size: 1
 num_epochs: 2
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 0.00001
 

examples/llama-2/gptq-lora.yml

@@ -42,7 +42,7 @@ output_dir: ./outputs/model-out
 gradient_accumulation_steps: 1
 micro_batch_size: 1
 num_epochs: 4
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 adam_beta2: 0.95
 adam_eps: 0.00001
 max_grad_norm: 1.0

examples/llama-2/qlora-fsdp.yml

@@ -39,7 +39,7 @@ wandb_log_model:
 gradient_accumulation_steps: 4
 micro_batch_size: 4
 num_epochs: 4
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 0.00001
 

examples/llama-3/fft-8b-liger-fsdp.yaml

@@ -37,7 +37,7 @@ wandb_log_model:
 gradient_accumulation_steps: 4
 micro_batch_size: 2
 num_epochs: 1
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 2e-5
 

examples/llama-3/lora-1b-ray.yml

@@ -1,79 +0,0 @@
-base_model: NousResearch/Llama-3.2-1B
-# Automatically upload checkpoint and final model to HF
-# hub_model_id: username/custom_model_name
-
-load_in_8bit: false
-load_in_4bit: false
-strict: false
-
-datasets:
-  - path: teknium/GPT4-LLM-Cleaned
-    type: alpaca
-dataset_prepared_path: last_run_prepared
-val_set_size: 0.1
-output_dir: ./outputs/lora-out
-
-adapter: lora
-lora_model_dir:
-
-sequence_len: 2048
-sample_packing: true
-eval_sample_packing: true
-pad_to_sequence_len: true
-
-lora_r: 16
-lora_alpha: 32
-lora_dropout: 0.05
-lora_fan_in_fan_out:
-lora_target_modules:
-  - gate_proj
-  - down_proj
-  - up_proj
-  - q_proj
-  - v_proj
-  - k_proj
-  - o_proj
-
-wandb_project:
-wandb_entity:
-wandb_watch:
-wandb_name:
-wandb_log_model:
-
-gradient_accumulation_steps: 2
-micro_batch_size: 2
-num_epochs: 1
-optimizer: adamw_8bit
-lr_scheduler: cosine
-learning_rate: 0.0002
-
-train_on_inputs: false
-group_by_length: false
-bf16: auto
-fp16:
-tf32: false
-
-gradient_checkpointing: true
-early_stopping_patience:
-resume_from_checkpoint:
-local_rank:
-logging_steps: 1
-xformers_attention:
-flash_attention: true
-
-loss_watchdog_threshold: 5.0
-loss_watchdog_patience: 3
-
-warmup_steps: 10
-evals_per_epoch: 4
-saves_per_epoch: 1
-debug:
-deepspeed: deepspeed_configs/zero3.json
-weight_decay: 0.0
-fsdp:
-fsdp_config:
-special_tokens:
-  pad_token: "<|end_of_text|>"
-
-use_ray: true
-ray_num_workers: 4

examples/llama-3/qlora-fsdp-405b.yaml

@@ -30,7 +30,7 @@ lora_target_linear: true
 gradient_accumulation_steps: 4
 micro_batch_size: 1
 num_epochs: 2
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 0.00001
 

examples/llama-3/qlora-fsdp-70b.yaml

@@ -39,7 +39,7 @@ wandb_log_model:
 gradient_accumulation_steps: 4
 micro_batch_size: 1
 num_epochs: 4
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 0.00001
 

examples/mistral/lora-mps.yml

@@ -47,7 +47,7 @@ wandb_log_model:
 gradient_accumulation_steps: 8
 micro_batch_size: 1
 num_epochs: 2
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 0.0002
 

examples/mistral/mixtral-8x22b-qlora-fsdp.yml

@@ -41,7 +41,7 @@ wandb_log_model:
 gradient_accumulation_steps: 4
 micro_batch_size: 2
 num_epochs: 1
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 0.0002
 

examples/mistral/mixtral-qlora-fsdp.yml

@@ -43,7 +43,7 @@ wandb_log_model:
 gradient_accumulation_steps: 4
 micro_batch_size: 2
 num_epochs: 1
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 0.0002
 

examples/phi/phi-ft.yml

@@ -38,7 +38,7 @@ wandb_log_model:
 gradient_accumulation_steps: 1
 micro_batch_size: 2
 num_epochs: 4
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 adam_beta2: 0.95
 adam_epsilon: 0.00001
 max_grad_norm: 1.0

examples/phi/phi-qlora.yml

@@ -38,7 +38,7 @@ wandb_log_model:
 gradient_accumulation_steps: 1
 micro_batch_size: 2
 num_epochs: 4
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 adam_beta2: 0.95
 adam_epsilon: 0.00001
 max_grad_norm: 1.0

examples/phi/phi2-ft.yml

@@ -38,7 +38,7 @@ wandb_log_model:
 gradient_accumulation_steps: 1
 micro_batch_size: 2
 num_epochs: 4
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 adam_beta2: 0.95
 adam_epsilon: 0.00001
 max_grad_norm: 1.0

examples/phi/phi3-ft-fsdp.yml

@@ -39,7 +39,7 @@ wandb_log_model:
 gradient_accumulation_steps: 2
 micro_batch_size: 12
 num_epochs: 2
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 adam_beta2: 0.95
 adam_epsilon: 0.00001
 max_grad_norm: 1.0

examples/phi/phi3-ft.yml

@@ -35,7 +35,7 @@ lora_fan_in_fan_out:
 gradient_accumulation_steps: 1
 micro_batch_size: 2
 num_epochs: 1
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 adam_beta2: 0.95
 adam_epsilon: 0.00001
 max_grad_norm: 1.0

examples/qwen2/prm.yaml

@@ -1,72 +0,0 @@
-base_model: Qwen/Qwen2.5-3B
-# optionally might have model_type or tokenizer_type
-model_type: AutoModelForTokenClassification
-num_labels: 2
-tokenizer_type: AutoTokenizer
-# Automatically upload checkpoint and final model to HF
-# hub_model_id: username/custom_model_name
-
-load_in_8bit: false
-load_in_4bit: false
-strict: false
-
-process_reward_model: true
-chat_template:
-datasets:
-  - path: trl-lib/math_shepherd
-    type: stepwise_supervised
-    step_separator: "\n"
-    max_completion_length:
-    train_on_last_step_only: false
-
-val_set_size: 0.2
-output_dir: ./outputs/out
-remove_unused_columns: false
-
-sequence_len: 2048
-sample_packing: false
-eval_sample_packing: false
-pad_to_sequence_len: true
-
-wandb_project:
-wandb_entity:
-wandb_watch:
-wandb_name:
-wandb_log_model:
-
-
-gradient_accumulation_steps: 1
-micro_batch_size: 8
-eval_batch_size: 8
-num_epochs: 1
-optimizer: adamw_torch
-lr_scheduler: cosine
-learning_rate: 0.0002
-
-train_on_inputs: false
-group_by_length: false
-bf16: true
-fp16:
-tf32:
-gradient_checkpointing: true
-gradient_checkpointing_kwargs:
-  use_reentrant: false
-early_stopping_patience:
-resume_from_checkpoint:
-local_rank:
-logging_steps: 1
-xformers_attention:
-flash_attention: true
-
-warmup_ratio: 0.1
-evals_per_epoch:
-eval_table_size:
-eval_max_new_tokens: 128
-eval_steps: 100
-saves_per_epoch: 1
-debug:
-deepspeed:
-weight_decay: 0.0
-fsdp:
-fsdp_config:
-special_tokens:

examples/qwen2/qlora-fsdp.yaml

@@ -37,7 +37,7 @@ wandb_log_model:
 gradient_accumulation_steps: 4
 micro_batch_size: 1
 num_epochs: 4
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 0.0002
 

examples/qwen2/reward-model.yaml

@@ -1,67 +0,0 @@
-base_model:  Qwen/Qwen2.5-0.5B
-# optionally might have model_type or tokenizer_type
-model_type: AutoModelForSequenceClassification
-num_labels: 1
-tokenizer_type: AutoTokenizer
-# Automatically upload checkpoint and final model to HF
-# hub_model_id: username/custom_model_name
-
-load_in_8bit: false
-load_in_4bit: false
-strict: false
-
-reward_model: true
-chat_template: qwen_25
-datasets:
-  - path: argilla/distilabel-intel-orca-dpo-pairs
-    type: bradley_terry.chat_template
-val_set_size: 0.0
-output_dir: ./outputs/out
-remove_unused_columns: false
-
-sequence_len: 2048
-sample_packing: false
-eval_sample_packing: false
-pad_to_sequence_len: true
-
-wandb_project:
-wandb_entity:
-wandb_watch:
-wandb_name:
-wandb_log_model:
-
-
-gradient_accumulation_steps: 4
-micro_batch_size: 2
-num_epochs: 4
-optimizer: adamw_bnb_8bit
-lr_scheduler: cosine
-learning_rate: 0.0002
-
-train_on_inputs: false
-group_by_length: false
-bf16: true
-fp16:
-tf32: true
-
-gradient_checkpointing: true
-gradient_checkpointing_kwargs:
-  use_reentrant: false
-early_stopping_patience:
-resume_from_checkpoint:
-local_rank:
-logging_steps: 1
-xformers_attention:
-flash_attention: true
-
-warmup_ratio: 0.1
-evals_per_epoch:
-eval_table_size:
-eval_max_new_tokens: 128
-saves_per_epoch: 1
-debug:
-deepspeed:
-weight_decay: 0.0
-fsdp:
-fsdp_config:
-special_tokens:

examples/tiny-llama/lora-mps.yml

@@ -38,7 +38,7 @@ wandb_log_model:
 gradient_accumulation_steps: 4
 micro_batch_size: 2
 num_epochs: 4
-optimizer: adamw_torch_fused
+optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 0.0002
 

requirements.txt

@@ -1,7 +1,7 @@
 --extra-index-url https://huggingface.github.io/autogptq-index/whl/cu118/
 
 # START section of dependencies that don't install on Darwin/MacOS
-bitsandbytes==0.45.1
+bitsandbytes==0.45.0
 triton>=3.0.0
 mamba-ssm==1.2.0.post1
 flash-attn==2.7.0.post2
@@ -13,20 +13,19 @@ liger-kernel==0.5.2
 packaging==23.2
 
 peft==0.14.0
-transformers==4.48.2
+transformers==4.47.1
 tokenizers>=0.21.0
-accelerate==1.3.0
+accelerate==1.2.1
 datasets==3.2.0
 deepspeed==0.16.1
-trl==0.14.0
+trl==0.13.0
 
 optimum==1.16.2
 hf_transfer
 sentencepiece
 gradio==3.50.2
 
-modal==0.70.5
-pydantic==2.10.6
+pydantic==2.6.3
 addict
 fire
 PyYAML>=6.0

scripts/chat_datasets.py

@@ -30,7 +30,7 @@ def parse_dataset(dataset=None, split="train"):
         )
     ds_cfg["field_messages"] = field_messages
 
-    message_fields = features[field_messages][0].keys()
+    message_fields = features["conversations"][0].keys()
     message_field_role = None
     for key in ["from", "role"]:
         if key in message_fields:

scripts/motd

@@ -1,15 +1,10 @@
 
-     #@@ #@@      @@# @@#
-    @@  @@          @@  @@           =@@#                               @@                 #@    =@@#.
-    @@    #@@@@@@@@@    @@           #@#@=                              @@                 #@     .=@@
-      #@@@@@@@@@@@@@@@@@            =@# @#     ##=     ##    =####=+    @@      =#####+  =#@@###.   @@
-    @@@@@@@@@@/  +@@/  +@@          #@  =@=     #@=   @@   =@#+  +#@#   @@    =@#+  +#@#   #@.      @@
-    @@@@@@@@@@  ##@@  ##@@         =@#   @#      =@# @#    @@      @@   @@    @@      #@   #@       @@
-     @@@@@@@@@@@@@@@@@@@@          #@=+++#@=      =@@#     @@      @@   @@    @@      #@   #@       @@
-                                  =@#=====@@     =@# @#    @@      @@   @@    @@      #@   #@       @@
-    @@@@@@@@@@@@@@@@  @@@@        #@      #@=   #@=  +@@   #@#    =@#   @@.   =@#    =@#   #@.      @@
-                                 =@#       @#  #@=     #@   =#@@@@#=    +#@@=  +#@@@@#=    .##@@+   @@
-    @@@@  @@@@@@@@@@@@@@@@
+                                 dP            dP   dP
+                                 88            88   88
+      .d8888b. dP.  .dP .d8888b. 88 .d8888b. d8888P 88
+      88'  `88  `8bd8'  88'  `88 88 88'  `88   88   88
+      88.  .88  .d88b.  88.  .88 88 88.  .88   88   88
+      `88888P8 dP'  `dP `88888P' dP `88888P'   dP   dP
 
 Welcome to the axolotl cloud image! If the you've mounted a disk to /workspace and the axolotl directory ie empty, run the following commands:
 

setup.py

@@ -32,6 +32,8 @@ def parse_requirements():
                 _install_requires.append(line)
     try:
         xformers_version = [req for req in _install_requires if "xformers" in req][0]
+        triton_version = [req for req in _install_requires if "triton" in req][0]
+        torchao_version = [req for req in _install_requires if "torchao" in req][0]
         autoawq_version = [req for req in _install_requires if "autoawq" in req][0]
         if "Darwin" in platform.system():
             # skip packages not compatible with OSX
@@ -85,8 +87,24 @@ def parse_requirements():
                 else:
                     _install_requires.pop(_install_requires.index(xformers_version))
                     _install_requires.append("xformers==0.0.28.post1")
+            elif (major, minor) >= (2, 3):
+                _install_requires.pop(_install_requires.index(torchao_version))
+                _install_requires.pop(_install_requires.index(triton_version))
+                _install_requires.append("triton>=2.3.1")
+                if patch == 0:
+                    _install_requires.pop(_install_requires.index(xformers_version))
+                    _install_requires.append("xformers>=0.0.26.post1")
+                else:
+                    _install_requires.pop(_install_requires.index(xformers_version))
+                    _install_requires.append("xformers>=0.0.27")
+            elif (major, minor) >= (2, 2):
+                _install_requires.pop(_install_requires.index(torchao_version))
+                _install_requires.pop(_install_requires.index(xformers_version))
+                _install_requires.append("xformers>=0.0.25.post1")
             else:
-                raise ValueError("axolotl requires torch>=2.4")
+                _install_requires.pop(_install_requires.index(torchao_version))
+                _install_requires.pop(_install_requires.index(xformers_version))
+                _install_requires.append("xformers>=0.0.23.post1")
 
     except PackageNotFoundError:
         pass
@@ -150,11 +168,5 @@ setup(
             "lomo-optim==0.1.1",
             "torch-optimi==0.2.1",
         ],
-        "ray": [
-            "ray[train]",
-        ],
-        "vllm": [
-            "vllm>=0.7.1",
-        ],
     },
 )

src/axolotl/cli/args.py

@@ -31,8 +31,6 @@ class TrainerCliArgs:
     merge_lora: bool = field(default=False)
     prompter: Optional[str] = field(default=None)
     shard: bool = field(default=False)
-    main_process_port: Optional[int] = field(default=None)
-    num_processes: Optional[int] = field(default=None)
 
 
 @dataclass

src/axolotl/cli/cloud/__init__.py

@@ -1,61 +0,0 @@
-"""
-launch axolotl in supported cloud platforms
-"""
-from pathlib import Path
-from typing import Union
-
-import yaml
-
-from axolotl.cli.art import print_axolotl_text_art
-from axolotl.cli.cloud.modal_ import ModalCloud
-from axolotl.utils.dict import DictDefault
-
-
-def load_cloud_cfg(cloud_config: Union[Path, str]) -> DictDefault:
-    """Load and validate cloud configuration."""
-    # Load cloud configuration.
-    with open(cloud_config, encoding="utf-8") as file:
-        cloud_cfg: DictDefault = DictDefault(yaml.safe_load(file))
-    return cloud_cfg
-
-
-def do_cli_preprocess(
-    cloud_config: Union[Path, str],
-    config: Union[Path, str],
-) -> None:
-    print_axolotl_text_art()
-    cloud_cfg = load_cloud_cfg(cloud_config)
-    cloud = ModalCloud(cloud_cfg)
-    with open(config, "r", encoding="utf-8") as file:
-        config_yaml = file.read()
-    cloud.preprocess(config_yaml)
-
-
-def do_cli_train(
-    cloud_config: Union[Path, str],
-    config: Union[Path, str],
-    accelerate: bool = True,
-    cwd=None,
-    **kwargs,
-) -> None:
-    print_axolotl_text_art()
-    cloud_cfg = load_cloud_cfg(cloud_config)
-    cloud = ModalCloud(cloud_cfg)
-    with open(config, "r", encoding="utf-8") as file:
-        config_yaml = file.read()
-    local_dirs = {}
-    if cwd and not Path(cwd).joinpath("src", "axolotl").exists():
-        local_dirs = {"/workspace/mounts": cwd}
-    cloud.train(config_yaml, accelerate=accelerate, local_dirs=local_dirs, **kwargs)
-
-
-def do_cli_lm_eval(
-    cloud_config: Union[Path, str],
-    config: Union[Path, str],
-) -> None:
-    print_axolotl_text_art()
-    cloud_cfg = load_cloud_cfg(cloud_config)
-    cloud = ModalCloud(cloud_cfg)
-    with open(config, "r", encoding="utf-8") as file:
-        config_yaml = file.read()
-    cloud.lm_eval(config_yaml)

src/axolotl/cli/cloud/base.py

@@ -1,18 +0,0 @@
-"""
-base class for cloud platforms from cli
-"""
-from abc import ABC, abstractmethod
-
-
-class Cloud(ABC):
-    """
-    Abstract base class for cloud platforms.
-    """
-
-    @abstractmethod
-    def preprocess(self, config_yaml: str, *args, **kwargs) -> None:
-        pass
-
-    @abstractmethod
-    def train(self, config_yaml: str, accelerate: bool = True) -> str:
-        pass

src/axolotl/cli/cloud/modal_.py

@@ -1,306 +0,0 @@
-"""
-Modal Cloud support from CLI
-"""
-import copy
-import json
-import os
-import subprocess  # nosec B404
-from pathlib import Path
-from random import randint
-from typing import Optional
-
-import modal
-
-from axolotl.cli.cloud.base import Cloud
-
-
-def run_cmd(cmd: str, run_folder: str, volumes=None):
-    """Run a command inside a folder, with Modal Volume reloading before and commit on success."""
-    # Ensure volumes contain latest files.
-    if volumes:
-        for _, vol in volumes.items():
-            vol.reload()
-
-    # modal workaround so it doesn't use the automounted axolotl
-    new_env = copy.deepcopy(os.environ)
-
-    if "PYTHONPATH" in new_env:
-        paths = ["/workspace/mounts"]
-        for sub_python_path_str in new_env["PYTHONPATH"].split(":"):
-            sub_python_path = Path(sub_python_path_str)
-            if not sub_python_path.joinpath("src", "axolotl").exists():
-                # we don't want to use the automounted axolotl or unexpected behavior happens
-                paths.append(str(sub_python_path))
-        if paths:
-            new_env["PYTHONPATH"] = ":".join(paths)
-        else:
-            del new_env["PYTHONPATH"]
-
-    # Propagate errors from subprocess.
-    if exit_code := subprocess.call(  # nosec B603
-        cmd.split(), cwd=run_folder, env=new_env
-    ):
-        exit(exit_code)  # pylint: disable=consider-using-sys-exit
-
-    # Commit writes to volume.
-    if volumes:
-        for _, vol in volumes.items():
-            vol.commit()
-
-
-class ModalCloud(Cloud):
-    """
-    Modal Cloud implementation.
-    """
-
-    def __init__(self, config, app=None):
-        self.config = config
-        if not app:
-            app = modal.App()
-        self.app = app
-
-        self.volumes = {}
-        if config.volumes:
-            for volume_config in config.volumes:
-                _, mount, vol = self.create_volume(volume_config)
-                self.volumes[mount] = (vol, volume_config)
-
-    def get_env(self):
-        res = {
-            "HF_DATASETS_CACHE": "/workspace/data/huggingface-cache/datasets",
-            "HF_HUB_CACHE": "/workspace/data/huggingface-cache/hub",
-        }
-
-        for key in self.config.get("env", []):
-            if isinstance(key, str):
-                if val := os.environ.get(key, ""):
-                    res[key] = val
-            elif isinstance(key, dict):
-                (key_, val) = list(key.items())[0]
-                res[key_] = val
-        return res
-
-    def get_image(self):
-        docker_tag = "main-py3.11-cu124-2.5.1"
-        if self.config.docker_tag:
-            docker_tag = self.config.docker_tag
-        docker_image = f"axolotlai/axolotl:{docker_tag}"
-
-        # grab the sha256 hash from docker hub for this image+tag
-        # this ensures that we always get the latest image for this tag, even if it's already cached
-        try:
-            manifest = subprocess.check_output(  # nosec B602
-                f"docker manifest inspect {docker_image}",
-                shell=True,
-            ).decode("utf-8")
-            sha256_hash = json.loads(manifest)["manifests"][0]["digest"]
-        except subprocess.CalledProcessError:
-            sha256_hash = None
-
-        # create the image
-        if sha256_hash:
-            image = modal.Image.from_registry(f"axolotlai/axolotl@{sha256_hash}")
-        else:
-            image = modal.Image.from_registry(docker_image)
-
-        dockerfile_commands = []
-        if self.config.dockerfile_commands:
-            dockerfile_commands.extend(self.config.dockerfile_commands)
-
-        # branch
-        if self.config.branch:
-            dockerfile_commands.extend(
-                [
-                    # Random id for cache busting of branch commits
-                    f"RUN echo '{str(randint(0, 1000000))}'",  # nosec B311
-                    f"RUN cd /workspace/axolotl && git fetch && git checkout {self.config.branch}",
-                ]
-            )
-
-        if dockerfile_commands:
-            image = image.dockerfile_commands(dockerfile_commands)
-
-        if env := self.get_env():
-            image = image.env(env)
-
-        image = image.pip_install("fastapi==0.110.0", "pydantic==2.6.3")
-
-        return image
-
-    def get_secrets(self):
-        res = []
-        if self.config.secrets:
-            for key in self.config.get("secrets", []):
-                # pylint: disable=duplicate-code
-                if isinstance(key, str):
-                    if val := os.environ.get(key, ""):
-                        res.append(modal.Secret.from_dict({key: val}))
-                elif isinstance(key, dict):
-                    (key_, val) = list(key.items())[0]
-                    res.append(modal.Secret.from_dict({key_: val}))
-        return res
-
-    def create_volume(self, volume_config):
-        name = volume_config.name
-        mount = volume_config.mount
-        return name, mount, modal.Volume.from_name(name, create_if_missing=True)
-
-    def get_ephemeral_disk_size(self):
-        return 1000 * 525  # 1 TiB
-
-    def get_preprocess_timeout(self):
-        if self.config.timeout_preprocess:
-            return int(self.config.timeout_preprocess)
-        return 60 * 60 * 3  # 3 hours
-
-    def get_preprocess_memory(self):
-        memory = 128  # default to 128GiB
-        if self.config.memory:
-            memory = int(self.config.memory)
-        if self.config.memory_preprocess:
-            memory = int(self.config.memory_preprocess)
-        return 1024 * memory
-
-    def get_preprocess_env(self):
-        return self.app.function(
-            image=self.get_image(),
-            volumes={k: v[0] for k, v in self.volumes.items()},
-            cpu=8.0,
-            ephemeral_disk=self.get_ephemeral_disk_size(),
-            memory=self.get_preprocess_memory(),
-            timeout=self.get_preprocess_timeout(),
-            secrets=self.get_secrets(),
-        )
-
-    def preprocess(self, config_yaml: str, *args, **kwargs):
-        modal_fn = self.get_preprocess_env()(_preprocess)
-        with modal.enable_output():
-            with self.app.run(detach=True):
-                modal_fn.remote(
-                    config_yaml,
-                    volumes={k: v[0] for k, v in self.volumes.items()},
-                    *args,
-                    **kwargs,
-                )
-
-    def get_train_timeout(self):
-        if self.config.timeout:
-            return int(self.config.timeout)
-        return 60 * 60 * 24  # 24 hours
-
-    def get_train_gpu(self):  # pylint: disable=too-many-return-statements
-        count = self.config.gpu_count or 1
-        family = self.config.gpu.lower() or "l40s"
-
-        if family == "l40s":
-            return modal.gpu.L40S(count=count)
-        if family in ["a100", "a100-40gb"]:
-            return modal.gpu.A100(count=count, size="40GB")
-        if family == "a100-80gb":
-            return modal.gpu.A100(count=count, size="80GB")
-        if family in ["a10", "a10g"]:
-            return modal.gpu.A10G(count=count)
-        if family == "h100":
-            return modal.gpu.H100(count=count)
-        if family == "t4":
-            return modal.gpu.T4(count=count)
-        if family == "l4":
-            return modal.gpu.L4(count=count)
-        raise ValueError(f"Unsupported GPU family: {family}")
-
-    def get_train_memory(self):
-        memory = 128  # default to 128GiB
-        if self.config.memory:
-            memory = int(self.config.memory)
-        return 1024 * memory
-
-    def get_train_env(self, local_dirs=None):
-        image = self.get_image()
-        for mount, local_dir in (local_dirs or {}).items():
-            image = image.add_local_dir(local_dir, mount)
-        return self.app.function(
-            image=image,
-            volumes={k: v[0] for k, v in self.volumes.items()},
-            cpu=16.0,
-            gpu=self.get_train_gpu(),
-            memory=self.get_train_memory(),
-            timeout=self.get_train_timeout(),
-            secrets=self.get_secrets(),
-        )
-
-    def train(
-        self,
-        config_yaml: str,
-        accelerate: bool = True,
-        local_dirs: Optional[dict[str, str]] = None,
-        **kwargs,
-    ):
-        modal_fn = self.get_train_env(local_dirs)(_train)
-        with modal.enable_output():
-            with self.app.run(detach=True):
-                modal_fn.remote(
-                    config_yaml,
-                    accelerate=accelerate,
-                    volumes={k: v[0] for k, v in self.volumes.items()},
-                    **kwargs,
-                )
-
-    def lm_eval(self, config_yaml: str):
-        modal_fn = self.get_train_env()(_lm_eval)
-        with modal.enable_output():
-            with self.app.run(detach=True):
-                if self.config.get("spawn", False):
-                    modal_fn_exec = modal_fn.spawn
-                else:
-                    modal_fn_exec = modal_fn.remote
-                modal_fn_exec(
-                    config_yaml,
-                    volumes={k: v[0] for k, v in self.volumes.items()},
-                )
-
-
-def _preprocess(config_yaml: str, volumes=None):
-    Path("/workspace/artifacts/axolotl").mkdir(parents=True, exist_ok=True)
-    with open(
-        "/workspace/artifacts/axolotl/config.yaml", "w", encoding="utf-8"
-    ) as f_out:
-        f_out.write(config_yaml)
-    run_folder = "/workspace/artifacts/axolotl"
-    run_cmd(
-        "axolotl preprocess /workspace/artifacts/axolotl/config.yaml --dataset-processes=8",
-        run_folder,
-        volumes,
-    )
-
-
-def _train(config_yaml: str, accelerate: bool = True, volumes=None, **kwargs):
-    with open(
-        "/workspace/artifacts/axolotl/config.yaml", "w", encoding="utf-8"
-    ) as f_out:
-        f_out.write(config_yaml)
-    run_folder = "/workspace/artifacts/axolotl"
-    if accelerate:
-        accelerate_args = "--accelerate"
-    else:
-        accelerate_args = "--no-accelerate"
-    num_processes_args = ""
-    if num_processes := kwargs.pop("num_processes", None):
-        num_processes_args = f"--num-processes {num_processes}"
-    run_cmd(
-        f"axolotl train {accelerate_args} {num_processes_args} /workspace/artifacts/axolotl/config.yaml",
-        run_folder,
-        volumes,
-    )
-
-
-def _lm_eval(config_yaml: str, volumes=None):
-    with open(
-        "/workspace/artifacts/axolotl/config.yaml", "w", encoding="utf-8"
-    ) as f_out:
-        f_out.write(config_yaml)
-    run_folder = "/workspace/artifacts/axolotl"
-    run_cmd(
-        "axolotl lm-eval /workspace/artifacts/axolotl/config.yaml",
-        run_folder,
-        volumes,
-    )

src/axolotl/cli/main.py

@@ -1,19 +1,10 @@
 """Click CLI definitions for various axolotl commands."""
 # pylint: disable=redefined-outer-name
 
-import logging
-import os
-import random
 import subprocess  # nosec B404
-import tempfile
-from copy import deepcopy
-from itertools import product
-from pathlib import Path
 from typing import Optional
 
 import click
-import yaml
-from dotenv import load_dotenv
 
 import axolotl
 from axolotl.cli.args import EvaluateCliArgs, PreprocessCliArgs, TrainerCliArgs
@@ -24,81 +15,10 @@ from axolotl.cli.utils import (
     fetch_from_github,
     filter_none_kwargs,
 )
-from axolotl.integrations.lm_eval.cli import lm_eval
 from axolotl.utils import set_pytorch_cuda_alloc_conf
 from axolotl.utils.config.models.input.v0_4_1 import AxolotlInputConfig
 
 
-def generate_sweep_configs(base_config, sweeps_config):
-    """
-    Recursively generates all possible configurations by applying sweeps to the base config.
-
-    Args:
-        base_config (dict): The original configuration dictionary
-        sweeps_config (dict): Dictionary where keys are parameters and values are either:
-            - lists of values to sweep independently
-            - or for paired values, a list of dicts under the '_' key
-
-    Returns:
-        list: List of all possible configuration dictionaries
-
-    Example:
-        sweeps_config = {
-            'learning_rate': [0.1, 0.01],
-            '_': [
-                {'load_in_8bit': True, 'adapter': 'lora'},
-                {'load_in_4bit': True, 'adapter': 'qlora'}
-            ]
-        }
-    """
-    # Separate paired values from regular sweeps
-    paired_values = sweeps_config.get("_", [])
-    regular_sweeps = {k: v for k, v in sweeps_config.items() if k != "_"}
-
-    # Process regular sweeps
-    param_names = list(regular_sweeps.keys())
-    param_values = list(regular_sweeps.values())
-
-    # Generate combinations for regular sweeps
-    regular_combinations = list(product(*param_values)) if param_values else [()]
-
-    # Combine regular sweeps with paired values
-    all_combinations = []
-    for reg_combo in regular_combinations:
-        if paired_values:
-            for paired_set in paired_values:
-                new_config = {}
-                # new_config = deepcopy(base_config)
-                # Combine regular parameters with paired parameters
-                full_combo = {**dict(zip(param_names, reg_combo)), **paired_set}
-                for param_name, param_value in full_combo.items():
-                    new_config[param_name] = param_value
-                print(new_config)
-                all_combinations.append(new_config)
-        else:
-            # If no paired values, just use regular combinations
-            # new_config = deepcopy(base_config)
-            new_config = {}
-            for param_name, param_value in zip(param_names, reg_combo):
-                new_config[param_name] = param_value
-            print(new_config)
-            all_combinations.append(new_config)
-
-    # randomize the order of trials
-    random.seed(42)
-    random.shuffle(all_combinations)
-
-    # Generate a new config for each combination
-    result_configs = []
-    for combination in all_combinations:
-        new_config = deepcopy(base_config)
-        for param_name, param_value in combination.items():
-            new_config[param_name] = param_value
-        result_configs.append(new_config)
-
-    return result_configs
-
-
 @click.group()
 @click.version_option(version=axolotl.__version__, prog_name="axolotl")
 def cli():
@@ -107,28 +27,23 @@ def cli():
 
 @cli.command()
 @click.argument("config", type=click.Path(exists=True, path_type=str))
-@click.option("--cloud", default=None, type=click.Path(exists=True, path_type=str))
 @add_options_from_dataclass(PreprocessCliArgs)
 @add_options_from_config(AxolotlInputConfig)
 @filter_none_kwargs
-def preprocess(config: str, cloud: Optional[str] = None, **kwargs) -> None:
+def preprocess(config: str, **kwargs) -> None:
     """
     Preprocess datasets before training.
 
     Args:
         config: Path to `axolotl` config YAML file.
-        cloud: Path to a cloud accelerator configuration file.
         kwargs: Additional keyword arguments which correspond to CLI args or `axolotl`
             config options.
     """
-    if cloud:
-        from axolotl.cli.cloud import do_cli_preprocess
+    kwargs = {k: v for k, v in kwargs.items() if v is not None}
 
-        do_cli_preprocess(cloud_config=cloud, config=config)
-    else:
-        from axolotl.cli.preprocess import do_cli
+    from axolotl.cli.preprocess import do_cli
 
-        do_cli(config=config, **kwargs)
+    do_cli(config=config, **kwargs)
 
 
 @cli.command()
@@ -138,104 +53,32 @@ def preprocess(config: str, cloud: Optional[str] = None, **kwargs) -> None:
     default=True,
     help="Use accelerate launch for multi-GPU training",
 )
-@click.option("--cloud", default=None, type=click.Path(exists=True, path_type=str))
-@click.option(
-    "--sweep",
-    type=click.Path(exists=True, path_type=str),
-    help="YAML config for sweeping hyperparameters",
-)
 @add_options_from_dataclass(TrainerCliArgs)
 @add_options_from_config(AxolotlInputConfig)
 @filter_none_kwargs
-def train(
-    config: str,
-    accelerate: bool,
-    cloud: Optional[str] = None,
-    sweep: Optional[str] = None,
-    **kwargs,
-) -> None:
+def train(config: str, accelerate: bool, **kwargs) -> None:
     """
     Train or fine-tune a model.
 
     Args:
         config: Path to `axolotl` config YAML file.
         accelerate: Whether to use `accelerate` launcher.
-        cloud: Path to a cloud accelerator configuration file
-        sweep: Path to YAML config for sweeping hyperparameters.
         kwargs: Additional keyword arguments which correspond to CLI args or `axolotl`
             config options.
     """
     # Enable expandable segments for cuda allocation to improve VRAM usage
     set_pytorch_cuda_alloc_conf()
-    from axolotl.cli.cloud import do_cli_train
-
-    if "use_ray" in kwargs and kwargs["use_ray"]:
-        accelerate = False
-    if sweep:
-        # load the sweep configuration yaml file
-        with open(sweep, "r", encoding="utf-8") as fin:
-            sweep_config: dict[str, list] = yaml.safe_load(fin)
-        with open(config, "r", encoding="utf-8") as fin:
-            base_config: dict[str, list] = yaml.safe_load(fin)
-
-        # generate all possible configurations
-        permutations = generate_sweep_configs(base_config, sweep_config)
-
-        def iter_configs():
-            for perm in permutations:
-                # open temp directory for temporary configurations
-                with tempfile.TemporaryDirectory() as temp_dir:
-                    with open(
-                        Path(temp_dir) / "config.yaml", "w", encoding="utf-8"
-                    ) as fout:
-                        yaml.dump(perm, fout)
-                    yield str(Path(temp_dir) / "config.yaml")
 
+    if accelerate:
+        base_cmd = ["accelerate", "launch", "-m", "axolotl.cli.train"]
+        if config:
+            base_cmd.append(config)
+        cmd = build_command(base_cmd, kwargs)
+        subprocess.run(cmd, check=True)  # nosec B603
     else:
+        from axolotl.cli.train import do_cli
 
-        def iter_configs():
-            yield config
-
-    for cfg_file in iter_configs():
-        # handle errors from subprocess so we can continue rest of sweeps
-        try:
-            if accelerate:
-                if cloud:
-                    cwd = os.getcwd()
-                    do_cli_train(
-                        cloud_config=cloud, config=config, accelerate=True, cwd=cwd, **kwargs
-                    )
-                else:
-                    accelerate_args = []
-                    if "main_process_port" in kwargs:
-                        main_process_port = kwargs.pop("main_process_port", None)
-                        accelerate_args.append("--main_process_port")
-                        accelerate_args.append(str(main_process_port))
-                    if "num_processes" in kwargs:
-                        num_processes = kwargs.pop("num_processes", None)
-                        accelerate_args.append("--num_processes")
-                        accelerate_args.append(str(num_processes))
-
-                    base_cmd = ["accelerate", "launch"]
-                    base_cmd.extend(accelerate_args)
-                    base_cmd.extend(["-m", "axolotl.cli.train"])
-                    if cfg_file:
-                        base_cmd.append(cfg_file)
-                    cmd = build_command(base_cmd, kwargs)
-                    subprocess.run(cmd, check=True)  # nosec B603
-            else:
-                if cloud:
-                    do_cli_train(
-                        cloud_config=cloud, config=config, accelerate=False, **kwargs
-                    )
-                else:
-                    from axolotl.cli.train import do_cli
-
-                    do_cli(config=cfg_file, **kwargs)
-        except subprocess.CalledProcessError as exc:
-            logging.error(f"Failed to train/fine-tune config '{cfg_file}': {exc}")
-            if not sweep:
-                raise exc
+        do_cli(config=config, **kwargs)
 
 
 @cli.command()
@@ -354,6 +197,7 @@ def merge_lora(config: str, **kwargs) -> None:
 
     Args:
         config: Path to `axolotl` config YAML file.
+        accelerate: Whether to use `accelerate` launcher.
         kwargs: Additional keyword arguments which correspond to CLI args or `axolotl`
             config options.
     """
@@ -380,13 +224,9 @@ def fetch(directory: str, dest: Optional[str]) -> None:
     fetch_from_github(f"{directory}/", dest)
 
 
-cli.add_command(lm_eval)
-
-
 def main():
     cli()
 
 
 if __name__ == "__main__":
-    load_dotenv()
     main()

src/axolotl/cli/train.py

@@ -5,7 +5,6 @@ from pathlib import Path
 from typing import Union
 
 import fire
-from accelerate import Accelerator
 from dotenv import load_dotenv
 from transformers.hf_argparser import HfArgumentParser
 
@@ -16,7 +15,6 @@ from axolotl.cli.config import load_cfg
 from axolotl.common.datasets import load_datasets, load_preference_datasets
 from axolotl.integrations.base import PluginManager
 from axolotl.train import train
-from axolotl.utils.config import normalize_config, resolve_dtype
 from axolotl.utils.dict import DictDefault
 
 LOG = logging.getLogger(__name__)
@@ -65,47 +63,7 @@ def do_cli(config: Union[Path, str] = Path("examples/"), **kwargs) -> None:
         return_remaining_strings=True
     )
 
-    if parsed_cfg.use_ray:
-        from ray.train import RunConfig, ScalingConfig
-        from ray.train.torch import TorchTrainer
-
-        train_loop_config = {"cfg": parsed_cfg.to_dict(), "cli_args": parsed_cli_args}
-        trainer = TorchTrainer(
-            ray_train_func,
-            train_loop_config=train_loop_config,
-            scaling_config=ScalingConfig(
-                num_workers=parsed_cfg.ray_num_workers,
-                resources_per_worker=parsed_cfg.resources_per_worker.to_dict(),
-                use_gpu=True,
-            ),
-            run_config=RunConfig(
-                name=parsed_cfg.ray_run_name,
-                storage_path=Path(parsed_cfg.output_dir).absolute().as_posix(),
-            ),
-        )
-        return trainer.fit()
-    return do_train(parsed_cfg, parsed_cli_args)
-
-
-def ray_train_func(kwargs: dict):
-    # cast `cfg` back to DictDefault (ray tune deepcopy has issues with DictDefault so needed it to be dict)
-    # also renormalize the config now that TorchTrainer has spawned distributed workers
-    cfg = DictDefault(kwargs["cfg"])
-    normalize_config(cfg)
-
-    # now that we are on the worker node, we can check `is_torch_bf16_gpu_available` to resolve dtype
-    resolve_dtype(cfg)
-
-    # ray serializing objects gets rid of frozen attribute - HF expects dict not DefaultDict
-    if cfg.deepspeed:
-        cfg.deepspeed = cfg.deepspeed.to_dict()
-
-    # initialize accelerator before model instantiation
-    Accelerator(gradient_accumulation_steps=cfg.gradient_accumulation_steps)
-
-    kwargs["cfg"] = cfg
-
-    do_train(**kwargs)
+    do_train(parsed_cfg, parsed_cli_args)
 
 
 if __name__ == "__main__":

src/axolotl/common/datasets.py

@@ -122,11 +122,9 @@ def load_preference_datasets(
         `total_num_steps`.
     """
     train_dataset, eval_dataset = load_prepare_preference_datasets(cfg)
-    total_num_steps: Optional[int] = int(
+    total_num_steps = int(
         math.ceil(len(train_dataset) * cfg.num_epochs / cfg.batch_size)
     )
-    if cfg.rl == "grpo":
-        total_num_steps = None
 
     if cli_args.debug or cfg.debug:
         LOG.info("check_dataset_labels...")

src/axolotl/core/trainer_builder.py

@@ -39,22 +39,19 @@ from trl.trainer.utils import RewardDataCollatorWithPadding
 
 from axolotl.core.trainers.base import (
     AxolotlCPOTrainer,
+    AxolotlDPOTrainer,
     AxolotlKTOTrainer,
     AxolotlMambaTrainer,
     AxolotlORPOTrainer,
-    AxolotlPRMTrainer,
     AxolotlRewardTrainer,
     AxolotlTrainer,
     ReLoRATrainer,
 )
-from axolotl.core.trainers.dpo import DPOStrategy
-from axolotl.core.trainers.dpo.args import AxolotlDPOConfig
-from axolotl.core.trainers.grpo import GRPOStrategy
 from axolotl.core.training_args import (
     AxolotlCPOConfig,
+    AxolotlDPOConfig,
     AxolotlKTOConfig,
     AxolotlORPOConfig,
-    AxolotlPRMConfig,
     AxolotlRewardConfig,
     AxolotlTrainingArguments,
 )
@@ -227,8 +224,7 @@ class TrainerBuilderBase(abc.ABC):
 
 class HFCausalTrainerBuilder(TrainerBuilderBase):
     """
-    Build the HuggingFace training args/trainer for causal models
-    and reward modelling using TRL.
+    Build the HuggingFace training args/trainer for Causal models
     """
 
     def get_callbacks(self):
@@ -308,8 +304,6 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
             return AxolotlMambaTrainer
         if self.cfg.reward_model:
             return AxolotlRewardTrainer
-        if self.cfg.process_reward_model:
-            return AxolotlPRMTrainer
         return AxolotlTrainer
 
     def build(self, total_num_steps):
@@ -566,7 +560,6 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
         ] = self.cfg.loraplus_lr_embedding
         training_arguments_kwargs["embedding_lr"] = self.cfg.embedding_lr
         training_arguments_kwargs["embedding_lr_scale"] = self.cfg.embedding_lr_scale
-        training_arguments_kwargs["lr_groups"] = self.cfg.lr_groups
 
         if self.cfg.lr_scheduler in ["one_cycle", "log_sweep"]:
             training_arguments_kwargs["lr_scheduler_type"] = "cosine"
@@ -653,7 +646,7 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
         trainer_kwargs = {}
 
         if self.cfg.reward_model:
-            training_arguments_kwargs["max_length"] = self.cfg.sequence_len
+            trainer_kwargs["max_length"] = self.cfg.sequence_len
 
         # pylint: disable=duplicate-code
         if self.cfg.optimizer in [
@@ -710,17 +703,12 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
             training_arguments_kwargs[
                 "kd_zscore_base_temp"
             ] = self.cfg.kd_zscore_base_temp
-        if self.cfg.kd_top_k_before_softmax is not None:
-            training_arguments_kwargs[
-                "kd_top_k_before_softmax"
-            ] = self.cfg.kd_top_k_before_softmax
 
-        if self.cfg.reward_model:
-            training_args_cls = AxolotlRewardConfig
-        elif self.cfg.process_reward_model:
-            training_args_cls = AxolotlPRMConfig
-        else:
-            training_args_cls = AxolotlTrainingArguments
+        training_args_cls = (
+            AxolotlTrainingArguments
+            if not self.cfg.reward_model
+            else AxolotlRewardConfig
+        )
         training_args = training_args_cls(  # pylint: disable=unexpected-keyword-arg
             **training_arguments_kwargs,
         )
@@ -754,9 +742,9 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
         if eval_data_collator := self.build_collator(
             training_args, is_eval=True, **data_collator_kwargs
         ):
-            if not (self.cfg.reward_model or self.cfg.process_reward_model):
+            if not self.cfg.reward_model:
                 trainer_kwargs["eval_data_collator"] = eval_data_collator
-        if not (self.cfg.reward_model or self.cfg.process_reward_model):
+        if not self.cfg.reward_model:
             trainer_kwargs["bench_data_collator"] = transformers.DataCollatorForSeq2Seq(
                 self.tokenizer,
                 return_tensors="pt",
@@ -767,10 +755,8 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
             trainer_kwargs["processing_class"] = self.tokenizer
         else:
             trainer_kwargs["tokenizer"] = self.tokenizer
-        if (
-            not (trainer_cls in [AxolotlRewardTrainer, AxolotlPRMTrainer])
-            and self.cfg.datasets is not None
-        ):
+
+        if (trainer_cls is not AxolotlRewardTrainer) and self.cfg.datasets is not None:
             trainer_kwargs["dataset_tags"] = [
                 d["path"] for d in self.cfg.datasets if not Path(d["path"]).is_dir()
             ]
@@ -798,10 +784,6 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
         self, training_args: AxolotlTrainingArguments, is_eval=False, **kwargs
     ):
         if training_args.pretraining:
-            if self.cfg.pretraining_sample_concatenation is False:
-                return DataCollatorForSeq2Seq(self.tokenizer, **kwargs)
-            if self.cfg.micro_batch_size > 1:
-                return DataCollatorForSeq2Seq(self.tokenizer, **kwargs)
             return None
 
         if self.cfg.model_config_type == "mamba":
@@ -870,7 +852,7 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
 
 class HFRLTrainerBuilder(TrainerBuilderBase):
     """
-    Trainer factory class for TRL-based RLHF trainers (e.g. DPO)
+    Trainer factory class for DPO Trainer
     """
 
     def get_callbacks(self):
@@ -966,11 +948,10 @@ class HFRLTrainerBuilder(TrainerBuilderBase):
             # default to saving each epoch if not defined
             training_args_kwargs["save_strategy"] = "epoch"
 
-        if self.cfg.dataset_processes:
-            training_args_kwargs["dataset_num_proc"] = self.cfg.dataset_processes
+        training_args_kwargs["dataset_num_proc"] = self.cfg.dataset_processes
 
-        if (self.cfg.trl and self.cfg.trl.beta) or self.cfg.rl_beta:
-            training_args_kwargs["beta"] = self.cfg.trl.beta or self.cfg.rl_beta
+        if self.cfg.rl_beta:
+            training_args_kwargs["beta"] = self.cfg.rl_beta
         if self.cfg.orpo_alpha:
             # trl does some odd mapping of alpha to beta to reuse the beta parameter ???
             training_args_kwargs["beta"] = self.cfg.orpo_alpha
@@ -979,7 +960,6 @@ class HFRLTrainerBuilder(TrainerBuilderBase):
             training_args_kwargs["rpo_alpha"] = self.cfg.rpo_alpha
 
         training_args_cls = None
-        blocklist_args_kwargs = []
         if self.cfg.rl == "simpo":
             training_args_cls = AxolotlCPOConfig
             training_args_kwargs["loss_type"] = "simpo"
@@ -1004,15 +984,11 @@ class HFRLTrainerBuilder(TrainerBuilderBase):
                 self.cfg.kto_undesirable_weight or 1.0
             )
 
+            training_args_kwargs["dataset_num_proc"] = self.cfg.dataset_processes
             training_args_kwargs["max_length"] = self.cfg.sequence_len
             if self.cfg.max_prompt_len:
                 training_args_kwargs["max_prompt_length"] = self.cfg.max_prompt_len
 
-        elif self.cfg.rl == "grpo":
-            training_args_cls = GRPOStrategy.get_training_args_class()
-            training_args_kwargs.update(GRPOStrategy.set_training_args_kwargs(self.cfg))
-            blocklist_args_kwargs = GRPOStrategy.get_blocklist_args_kwargs()
-
         else:
             training_args_cls = AxolotlDPOConfig
             if self.cfg.rl == "ipo":
@@ -1023,20 +999,11 @@ class HFRLTrainerBuilder(TrainerBuilderBase):
             training_args_kwargs["generate_during_eval"] = self.cfg.use_wandb
             if self.cfg.dpo_use_weighting is not None:
                 training_args_kwargs["use_weighting"] = self.cfg.dpo_use_weighting
-            if self.cfg.dpo_use_logits_to_keep is not None:
-                training_args_kwargs[
-                    "use_logits_to_keep"
-                ] = self.cfg.dpo_use_logits_to_keep
 
-        for blocklist_key in blocklist_args_kwargs:
-            if blocklist_key in training_args_kwargs:
-                del training_args_kwargs[blocklist_key]
-
-        max_steps = self.cfg.max_steps or total_num_steps or -1
         training_args = training_args_cls(  # pylint: disable=unexpected-keyword-arg
-            self.cfg.output_dir,
+            output_dir=self.cfg.output_dir,
             per_device_train_batch_size=self.cfg.micro_batch_size,
-            max_steps=max_steps,
+            max_steps=self.cfg.max_steps or total_num_steps,
             gradient_accumulation_steps=self.cfg.gradient_accumulation_steps,
             learning_rate=self.cfg.learning_rate,
             warmup_steps=self.cfg.warmup_steps,
@@ -1063,12 +1030,8 @@ class HFRLTrainerBuilder(TrainerBuilderBase):
             dpo_trainer_kwargs[
                 "precompute_ref_log_probs"
             ] = self.cfg.precompute_ref_log_probs
-        if self.cfg.rl == "grpo":
-            trainer_cls = GRPOStrategy.get_trainer_class()
-            trainer_cls_args = [self.model]
-            dpo_trainer_kwargs.update(GRPOStrategy.set_trainer_kwargs(self.cfg))
-        elif self.cfg.rl in ["dpo", "ipo"]:
-            trainer_cls = DPOStrategy.get_trainer_class()
+        if self.cfg.rl in ["dpo", "ipo"]:
+            trainer_cls = AxolotlDPOTrainer
             trainer_cls_args = [self.model, self.model_ref]
         elif self.cfg.rl == "orpo":
             trainer_cls = AxolotlORPOTrainer
@@ -1088,9 +1051,7 @@ class HFRLTrainerBuilder(TrainerBuilderBase):
         else:
             dpo_trainer_kwargs["tokenizer"] = self.tokenizer
 
-        if self.cfg.datasets is not None and (
-            trainer_cls is DPOStrategy.get_trainer_class()
-        ):
+        if self.cfg.datasets is not None and (trainer_cls is AxolotlDPOTrainer):
             dpo_trainer_kwargs["dataset_tags"] = [
                 d["path"] for d in self.cfg.datasets if not Path(d["path"]).is_dir()
             ]

src/axolotl/core/trainers/base.py

@@ -5,21 +5,23 @@ module for customized trainers
 from __future__ import annotations
 
 # pylint: disable=too-many-lines
+import gc
 import logging
 import os
 from collections import defaultdict
 from functools import wraps
-from typing import Dict, Literal, Optional
+from typing import Any, Dict, Literal, Optional, Union
 
 import torch
 from datasets import Dataset
 from peft.optimizers import create_loraplus_optimizer
+from torch import nn
 from torch.optim.lr_scheduler import OneCycleLR
 from torch.utils.data import BatchSampler, DataLoader, RandomSampler, SequentialSampler
 from transformers import Trainer
 from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR, seed_worker
 from transformers.utils import is_sagemaker_mp_enabled
-from trl import CPOTrainer, KTOTrainer, ORPOTrainer, PRMTrainer, RewardTrainer
+from trl import CPOTrainer, DPOTrainer, KTOTrainer, ORPOTrainer, RewardTrainer
 from trl.trainer.utils import pad_to_length
 
 from axolotl.monkeypatch.relora import ReLoRAScheduler
@@ -192,95 +194,11 @@ class AxolotlTrainer(SchedulerMixin, Trainer):
             )
         return super()._wrap_model(model, training=training, dataloader=dataloader)
 
-    def create_optimizer_grouped_parameters(self, opt_model, optimizer_kwargs):
-        decay_parameters = self.get_decay_parameter_names(opt_model)
-        params = {
-            "to_weight_decay": {},  # LayerNorm and bias
-            "embeddings": {},  # lm_head, embed_tokens,
-            "no_weight_decay": {},
-        }
-        lr_groups_lookup = {}
-        lr_groups_learning_rates = {}
-        if self.args.lr_groups:
-            for lr_group in self.args.lr_groups:
-                group_name = lr_group["name"]
-                group_modules = lr_group["modules"]
-                for module in group_modules:
-                    lr_groups_lookup[module] = group_name
-                lr_groups_learning_rates[group_name] = lr_group["lr"]
-                params[f"to_weight_decay_{group_name}"] = {}
-
-        for name, param in opt_model.named_parameters():
-            if not param.requires_grad:
-                continue
-            if name.endswith("modules_to_save.default.weight") or any(
-                embed_name in name for embed_name in ["embed_tokens", "lm_head"]
-            ):
-                params["embeddings"][name] = param
-            elif name in decay_parameters:
-                lr_group_modules = [
-                    group_modules
-                    for group_modules in lr_groups_lookup
-                    if group_modules in name
-                ]
-                if lr_groups_lookup and any(lr_group_modules):
-                    lr_group_module = lr_group_modules[0]
-                    group_name = lr_groups_lookup[lr_group_module]
-                    params[f"to_weight_decay_{group_name}"][name] = param
-                else:
-                    params["to_weight_decay"][name] = param
-            else:
-                params["no_weight_decay"][name] = param
-        optimizer_grouped_parameters = []
-        if params["to_weight_decay"]:
-            optimizer_grouped_parameters.append(
-                {
-                    "params": list(params["to_weight_decay"].values()),
-                    "weight_decay": self.args.weight_decay,
-                    "lr": optimizer_kwargs["lr"],
-                }
-            )
-        if params["embeddings"]:
-            lr = optimizer_kwargs["lr"]  # pylint: disable=invalid-name
-            if self.args.embedding_lr_scale:
-                lr *= self.args.embedding_lr_scale  # pylint: disable=invalid-name
-            elif self.args.embedding_lr:
-                lr = self.args.embedding_lr  # pylint: disable=invalid-name
-            optimizer_grouped_parameters.append(
-                {
-                    "params": list(params["embeddings"].values()),
-                    "weight_decay": 0.0,
-                    "lr": lr,
-                }
-            )
-        if params["no_weight_decay"]:
-            optimizer_grouped_parameters.append(
-                {
-                    "params": list(params["no_weight_decay"].values()),
-                    "weight_decay": 0.0,
-                    "lr": optimizer_kwargs["lr"],
-                }
-            )
-        for group_name, group_lr in lr_groups_learning_rates.items():
-            if params[f"to_weight_decay_{group_name}"]:
-                optimizer_grouped_parameters.append(
-                    {
-                        "params": list(
-                            params[f"to_weight_decay_{group_name}"].values()
-                        ),
-                        "weight_decay": self.args.weight_decay,
-                        "lr": group_lr,
-                    }
-                )
-
-        return optimizer_grouped_parameters
-
     def create_optimizer(self):
         if (
             self.args.loraplus_lr_ratio is None
             and self.args.embedding_lr_scale is None
             and self.args.embedding_lr is None
-            and self.args.lr_groups is None
             and self.args.alternate_optimizer
             not in [
                 "optimi_adamw",
@@ -294,13 +212,59 @@ class AxolotlTrainer(SchedulerMixin, Trainer):
 
         opt_model = self.model_wrapped if is_sagemaker_mp_enabled() else self.model
         if self.optimizer is None:  # pylint: disable=access-member-before-definition
+            decay_parameters = self.get_decay_parameter_names(opt_model)
+            params = {
+                "to_weight_decay": {},  # LayerNorm and bias
+                "embeddings": {},  # lm_head, embed_tokens,
+                "no_weight_decay": {},
+            }
+
             optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(
                 self.args,
                 opt_model,
             )
-            optimizer_grouped_parameters = self.create_optimizer_grouped_parameters(
-                opt_model, optimizer_kwargs
-            )
+
+            for name, param in opt_model.named_parameters():
+                if not param.requires_grad:
+                    continue
+                if name.endswith("modules_to_save.default.weight") or any(
+                    embed_name in name for embed_name in ["embed_tokens", "lm_head"]
+                ):
+                    params["embeddings"][name] = param
+                elif name in decay_parameters:
+                    params["to_weight_decay"][name] = param
+                else:
+                    params["no_weight_decay"][name] = param
+            optimizer_grouped_parameters = []
+            if params["to_weight_decay"]:
+                optimizer_grouped_parameters.append(
+                    {
+                        "params": list(params["to_weight_decay"].values()),
+                        "weight_decay": self.args.weight_decay,
+                        "lr": optimizer_kwargs["lr"],
+                    }
+                )
+            if params["embeddings"]:
+                lr = optimizer_kwargs["lr"]  # pylint: disable=invalid-name
+                if self.args.embedding_lr_scale:
+                    lr *= self.args.embedding_lr_scale  # pylint: disable=invalid-name
+                elif self.args.embedding_lr:
+                    lr = self.args.embedding_lr  # pylint: disable=invalid-name
+                optimizer_grouped_parameters.append(
+                    {
+                        "params": list(params["embeddings"].values()),
+                        "weight_decay": 0.0,
+                        "lr": lr,
+                    }
+                )
+            if params["no_weight_decay"]:
+                optimizer_grouped_parameters.append(
+                    {
+                        "params": list(params["no_weight_decay"].values()),
+                        "weight_decay": 0.0,
+                        "lr": optimizer_kwargs["lr"],
+                    }
+                )
 
             if self.args.loraplus_lr_ratio is not None:
                 loraplus_lr_ratio = getattr(self.args, "loraplus_lr_ratio", None)
@@ -317,7 +281,6 @@ class AxolotlTrainer(SchedulerMixin, Trainer):
             elif (
                 self.args.embedding_lr_scale is not None
                 or self.args.embedding_lr is not None
-                or self.args.lr_groups is not None
             ):
                 self.optimizer = (  # pylint: disable=attribute-defined-outside-init
                     optimizer_cls(optimizer_grouped_parameters, **optimizer_kwargs)
@@ -838,6 +801,106 @@ class ReLoRATrainer(AxolotlTrainer):
         return self.lr_scheduler
 
 
+class AxolotlDPOTrainer(SchedulerMixin, DPOTrainer):
+    """
+    Extend the base DPOTrainer for axolotl helpers
+    """
+
+    tag_names = ["axolotl", "dpo"]
+
+    def __init__(self, *args, dataset_tags=None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.dataset_tags = dataset_tags
+        self.optimizer = None
+
+    def create_optimizer(self):
+        if self.args.loraplus_lr_ratio is None:
+            return super().create_optimizer()
+
+        opt_model = self.model_wrapped if is_sagemaker_mp_enabled() else self.model
+        if self.optimizer is None:  # pylint: disable=access-member-before-definition
+            optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(
+                self.args,
+                opt_model,
+            )
+
+            loraplus_lr_ratio = getattr(self.args, "loraplus_lr_ratio", None)
+            if loraplus_lr_ratio:
+                print("Using lora+")
+            loraplus_lr_embedding = getattr(self.args, "loraplus_lr_embedding", None)
+            self.optimizer = create_loraplus_optimizer(  # pylint: disable=attribute-defined-outside-init
+                opt_model,
+                optimizer_cls,
+                loraplus_lr_ratio=loraplus_lr_ratio,
+                loraplus_lr_embedding=loraplus_lr_embedding,
+                **optimizer_kwargs,
+            )
+
+        if is_sagemaker_mp_enabled():
+            self.optimizer = smp.DistributedOptimizer(  # pylint: disable=attribute-defined-outside-init
+                self.optimizer
+            )
+
+        return self.optimizer
+
+    @wraps(DPOTrainer.push_to_hub)
+    def push_to_hub(self, *args, **kwargs) -> str:
+        """
+        Overwrite the `push_to_hub` method in order to force-add the tags when pushing the
+        model on the Hub. Please refer to `~transformers.Trainer.push_to_hub` for more details.
+        """
+        kwargs = _sanitize_kwargs_for_ds_tagging(
+            dataset_tags=self.dataset_tags, kwargs=kwargs
+        )
+        kwargs = _sanitize_kwargs_for_tagging(tag_names=self.tag_names, kwargs=kwargs)
+
+        return super().push_to_hub(*args, **kwargs)
+
+    @staticmethod
+    def tokenize_row(
+        features,
+        processing_class,
+        max_prompt_length,
+        max_completion_length,
+        add_special_tokens,
+    ) -> Dict:
+        res = DPOTrainer.tokenize_row(
+            features,
+            processing_class,
+            max_prompt_length,
+            max_completion_length,
+            add_special_tokens,
+        )
+        # fix when the tokenizer doesn't have a bos_token_id, e.g. Qwen
+        if processing_class.bos_token is None and res["prompt_input_ids"][0] is None:
+            for key in res.keys():
+                res[key] = res[key][1:]
+
+        if processing_class.bos_token and processing_class.bos_token_id is not None:
+            # dpo trainer may incorrectly prepend the bos_token_id to the dpo outputs
+            if res["chosen_input_ids"][0] == processing_class.bos_token_id:
+                res["chosen_input_ids"] = res["chosen_input_ids"][1:]
+                res["chosen_labels"] = res["chosen_labels"][1:]
+                res["chosen_attention_mask"] = res["chosen_attention_mask"][1:]
+            if res["rejected_input_ids"][0] == processing_class.bos_token_id:
+                res["rejected_input_ids"] = res["rejected_input_ids"][1:]
+                res["rejected_labels"] = res["rejected_labels"][1:]
+                res["rejected_attention_mask"] = res["rejected_attention_mask"][1:]
+
+        return res
+
+    def training_step(
+        self,
+        model: nn.Module,
+        inputs: Dict[str, Union[torch.Tensor, Any]],
+        num_items_in_batch=None,
+    ) -> torch.Tensor:
+        loss: torch.Tensor = super().training_step(model, inputs, num_items_in_batch)
+        gc.collect()
+        torch.cuda.empty_cache()
+        return loss
+
+
 class AxolotlORPOTrainer(SchedulerMixin, ORPOTrainer):
     """
     Extend the base ORPOTrainer for axolotl helpers
@@ -868,11 +931,3 @@ class AxolotlRewardTrainer(SchedulerMixin, RewardTrainer):
     """
 
     tag_names = ["axolotl", "reward"]
-
-
-class AxolotlPRMTrainer(SchedulerMixin, PRMTrainer):
-    """
-    Extend the base trl.PRMTrainer for axolotl helpers
-    """
-
-    tag_names = ["axolotl", "prm"]

src/axolotl/core/trainers/dpo/__init__.py

@@ -1,33 +0,0 @@
-"""
-DPO Specific Strategy for training
-"""
-from axolotl.core.trainers.dpo.trainer import AxolotlDPOTrainer
-
-
-class DPOStrategy:
-    """
-    Strategy for DPO training
-    """
-
-    @classmethod
-    def get_trainer_class(cls):
-        return AxolotlDPOTrainer
-
-    @classmethod
-    def get_training_args_class(cls):
-        from axolotl.core.trainers.dpo.args import AxolotlDPOConfig
-
-        return AxolotlDPOConfig
-
-    @classmethod
-    def set_training_args_kwargs(cls, cfg):
-        training_args_kwargs = {}
-        if cfg.rl == "ipo":
-            training_args_kwargs["loss_type"] = "ipo"
-        training_args_kwargs["max_length"] = cfg.sequence_len
-        training_args_kwargs["max_completion_length"] = None
-        training_args_kwargs["max_prompt_length"] = cfg.sequence_len
-        training_args_kwargs["generate_during_eval"] = cfg.use_wandb
-        if cfg.dpo_use_weighting is not None:
-            training_args_kwargs["use_weighting"] = cfg.dpo_use_weighting
-        return training_args_kwargs

src/axolotl/core/trainers/dpo/args.py

@@ -1,15 +0,0 @@
-"""
-Axolotl specific DPO args
-"""
-from dataclasses import dataclass
-
-from trl import DPOConfig
-
-from axolotl.core.training_args import AxolotlTrainingMixins
-
-
-@dataclass
-class AxolotlDPOConfig(AxolotlTrainingMixins, DPOConfig):
-    """
-    DPO config for DPO training
-    """

src/axolotl/core/trainers/dpo/trainer.py

@@ -1,125 +0,0 @@
-"""
-DPO trainer for axolotl
-"""
-import gc
-from functools import wraps
-from typing import Any, Dict, Union
-
-import torch
-from peft.optimizers import create_loraplus_optimizer
-from torch import nn
-from transformers import Trainer
-from transformers.utils import is_sagemaker_mp_enabled
-from trl import DPOTrainer
-
-from axolotl.core.trainers.base import (
-    SchedulerMixin,
-    _sanitize_kwargs_for_ds_tagging,
-    _sanitize_kwargs_for_tagging,
-)
-
-if is_sagemaker_mp_enabled():
-    import smdistributed.modelparallel.torch as smp
-
-
-class AxolotlDPOTrainer(SchedulerMixin, DPOTrainer):
-    """
-    Extend the base DPOTrainer for axolotl helpers
-    """
-
-    tag_names = ["axolotl", "dpo"]
-
-    def __init__(self, *args, dataset_tags=None, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.dataset_tags = dataset_tags
-        self.optimizer = None
-        self.model_accepts_loss_kwargs = False
-
-    def create_optimizer(self):
-        # pylint: disable=duplicate-code
-        if self.args.loraplus_lr_ratio is None:
-            return super().create_optimizer()
-
-        opt_model = self.model_wrapped if is_sagemaker_mp_enabled() else self.model
-        if self.optimizer is None:  # pylint: disable=access-member-before-definition
-            optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(
-                self.args,
-                opt_model,
-            )
-
-            loraplus_lr_ratio = getattr(self.args, "loraplus_lr_ratio", None)
-            if loraplus_lr_ratio:
-                print("Using lora+")
-            loraplus_lr_embedding = getattr(self.args, "loraplus_lr_embedding", None)
-            # pylint: disable=duplicate-code
-            self.optimizer = create_loraplus_optimizer(  # pylint: disable=attribute-defined-outside-init
-                opt_model,
-                optimizer_cls,
-                loraplus_lr_ratio=loraplus_lr_ratio,
-                loraplus_lr_embedding=loraplus_lr_embedding,
-                **optimizer_kwargs,
-            )
-
-        if is_sagemaker_mp_enabled():
-            self.optimizer = smp.DistributedOptimizer(  # pylint: disable=attribute-defined-outside-init
-                self.optimizer
-            )
-
-        return self.optimizer
-
-    @wraps(DPOTrainer.push_to_hub)
-    def push_to_hub(self, *args, **kwargs) -> str:
-        """
-        Overwrite the `push_to_hub` method in order to force-add the tags when pushing the
-        model on the Hub. Please refer to `~transformers.Trainer.push_to_hub` for more details.
-        """
-        kwargs = _sanitize_kwargs_for_ds_tagging(
-            dataset_tags=self.dataset_tags, kwargs=kwargs
-        )
-        kwargs = _sanitize_kwargs_for_tagging(tag_names=self.tag_names, kwargs=kwargs)
-
-        return super().push_to_hub(*args, **kwargs)
-
-    @staticmethod
-    def tokenize_row(
-        features,
-        processing_class,
-        max_prompt_length,
-        max_completion_length,
-        add_special_tokens,
-    ) -> Dict:
-        res = DPOTrainer.tokenize_row(
-            features,
-            processing_class,
-            max_prompt_length,
-            max_completion_length,
-            add_special_tokens,
-        )
-        # fix when the tokenizer doesn't have a bos_token_id, e.g. Qwen
-        if processing_class.bos_token is None and res["prompt_input_ids"][0] is None:
-            for key in res.keys():
-                res[key] = res[key][1:]
-
-        if processing_class.bos_token and processing_class.bos_token_id is not None:
-            # dpo trainer may incorrectly prepend the bos_token_id to the dpo outputs
-            if res["chosen_input_ids"][0] == processing_class.bos_token_id:
-                res["chosen_input_ids"] = res["chosen_input_ids"][1:]
-                res["chosen_labels"] = res["chosen_labels"][1:]
-                res["chosen_attention_mask"] = res["chosen_attention_mask"][1:]
-            if res["rejected_input_ids"][0] == processing_class.bos_token_id:
-                res["rejected_input_ids"] = res["rejected_input_ids"][1:]
-                res["rejected_labels"] = res["rejected_labels"][1:]
-                res["rejected_attention_mask"] = res["rejected_attention_mask"][1:]
-
-        return res
-
-    def training_step(
-        self,
-        model: nn.Module,
-        inputs: Dict[str, Union[torch.Tensor, Any]],
-        num_items_in_batch=None,
-    ) -> torch.Tensor:
-        loss: torch.Tensor = super().training_step(model, inputs, num_items_in_batch)
-        gc.collect()
-        torch.cuda.empty_cache()
-        return loss

src/axolotl/core/trainers/grpo/__init__.py

@@ -1,113 +0,0 @@
-"""
-GRPO Specific Strategy for training
-"""
-
-import importlib
-import inspect
-import logging
-
-from trl.trainer.grpo_trainer import RewardFunc
-
-from axolotl.core.trainers.grpo.trainer import AxolotlGRPOTrainer
-
-LOG = logging.getLogger("axolotl")
-
-
-class GRPOStrategy:
-    """
-    Strategy for GRPO training
-    """
-
-    @classmethod
-    def get_trainer_class(cls):
-        return AxolotlGRPOTrainer
-
-    @classmethod
-    def get_training_args_class(cls):
-        from axolotl.core.trainers.grpo.args import AxolotlGRPOConfig
-
-        return AxolotlGRPOConfig
-
-    @classmethod
-    def set_training_args_kwargs(cls, cfg):
-        grpo_args_kwargs = {}
-        if cfg.trl and cfg.trl.use_vllm:
-            grpo_args_kwargs["use_vllm"] = cfg.trl.use_vllm
-            if cfg.trl and cfg.trl.vllm_device:
-                grpo_args_kwargs["vllm_device"] = cfg.trl.vllm_device
-            else:
-                grpo_args_kwargs["vllm_device"] = "auto"
-            if cfg.trl and cfg.trl.vllm_gpu_memory_utilization:
-                grpo_args_kwargs[
-                    "vllm_gpu_memory_utilization"
-                ] = cfg.trl.vllm_gpu_memory_utilization
-            if cfg.trl and cfg.trl.vllm_max_model_len:
-                grpo_args_kwargs["vllm_max_model_len"] = cfg.trl.vllm_max_model_len
-        if cfg.trl and cfg.trl.num_generations:
-            grpo_args_kwargs["num_generations"] = cfg.trl.num_generations
-        if cfg.trl and cfg.trl.sync_ref_model:
-            grpo_args_kwargs["sync_ref_model"] = cfg.trl.sync_ref_model
-            if cfg.trl and cfg.trl.ref_model_mixup_alpha:
-                grpo_args_kwargs[
-                    "ref_model_mixup_alpha"
-                ] = cfg.trl.ref_model_mixup_alpha
-            if cfg.trl and cfg.trl.ref_model_sync_steps:
-                grpo_args_kwargs["ref_model_sync_steps"] = cfg.trl.ref_model_sync_steps
-        grpo_args_kwargs["max_completion_length"] = cfg.trl.max_completion_length
-        return grpo_args_kwargs
-
-    @classmethod
-    def set_trainer_kwargs(cls, cfg):
-        trainer_kwargs = {}
-        if cfg.trl and cfg.trl.reward_funcs:
-            reward_funcs = []
-            for reward_func_fqn in cfg.trl.reward_funcs:
-                reward_funcs.append(cls.get_reward_func(reward_func_fqn))
-            trainer_kwargs["reward_funcs"] = reward_funcs
-        if cfg.trl and cfg.trl.reward_processing_classes:
-            trainer_kwargs[
-                "reward_processing_classes"
-            ] = cfg.trl.reward_processing_classes
-        return trainer_kwargs
-
-    @classmethod
-    def get_collator(cls, *args, **kwargs):  # pylint: disable=unused-argument
-        # No data collation is needed in GRPO, handled by trl's trainer __init__
-        return None
-
-    @classmethod
-    def get_blocklist_args_kwargs(cls):
-        return ["dataset_num_proc"]
-
-    @classmethod
-    def get_reward_func(cls, reward_func_fqn: str) -> RewardFunc:
-        """
-        Returns the reward function from the given fully qualified name, or the path to the reward function model.
-
-        Args:
-            reward_func_fqn (str): Fully qualified name of the reward function (e.g. r1_grpo.gsm8k_transform),
-                or a HF hub path to the reward model.
-        Raises:
-            ValueError: If the reward function does not accept at least two arguments.
-
-        Returns:
-            RewardFunc: A callable that accepts prompts and completions and returns rewards,
-                or a path to a reward model.
-
-        """
-        try:
-            # use importlib to dynamically load the reward function from the module
-            reward_func_module_name = reward_func_fqn.split(".")[-1]
-            reward_func_module = importlib.import_module(reward_func_fqn.split(".")[-2])
-            reward_func = getattr(reward_func_module, reward_func_module_name)
-            if not len(inspect.signature(reward_func).parameters) >= 2:
-                raise ValueError(
-                    "Reward function must accept at least two arguments: prompts: list and completions: list"
-                )
-            return reward_func
-        except ModuleNotFoundError:
-            # the user has passed a string (ideally indicating the path of a reward model)
-            LOG.info(
-                f"Reward function {reward_func} is a pre-trained model path - if this is unexpected, please check the reward function path."
-            )
-            return reward_func

src/axolotl/core/trainers/grpo/args.py

@@ -1,15 +0,0 @@
-"""
-Axolotl Specific Training Args
-"""
-from dataclasses import dataclass
-
-from trl import GRPOConfig
-
-from axolotl.core.training_args import AxolotlTrainingMixins
-
-
-@dataclass
-class AxolotlGRPOConfig(AxolotlTrainingMixins, GRPOConfig):
-    """
-    Axolotl GRPO Config for GRPO training
-    """

src/axolotl/core/trainers/grpo/trainer.py

@@ -1,14 +0,0 @@
-"""
-Axolotl GRPO trainer
-"""
-from trl import GRPOTrainer
-
-from axolotl.core.trainers.base import SchedulerMixin
-
-
-class AxolotlGRPOTrainer(SchedulerMixin, GRPOTrainer):
-    """
-    Extend the base GRPOTrainer for axolotl helpers
-    """
-
-    _tag_names = ["trl", "grpo", "axolotl"]

src/axolotl/core/training_args.py

@@ -5,7 +5,7 @@ from dataclasses import dataclass, field
 from typing import Optional
 
 from transformers import TrainingArguments
-from trl import CPOConfig, KTOConfig, ORPOConfig, PRMConfig, RewardConfig
+from trl import CPOConfig, DPOConfig, KTOConfig, ORPOConfig, RewardConfig
 
 
 @dataclass
@@ -125,10 +125,6 @@ class AxolotlTrainingMixins:
         default=None,
         metadata={"help": "Scale the learning rate for the embedding layers."},
     )
-    lr_groups: Optional[list[dict]] = field(
-        default=None,
-        metadata={"help": "Specify learning rate groups for with different LRs."},
-    )
     embedding_lr: Optional[float] = field(
         default=None,
         metadata={"help": "absolute learning rate for the embedding layers."},
@@ -199,13 +195,6 @@ class AxolotlTrainingMixins:
         },
     )
 
-    kd_top_k_before_softmax: Optional[bool] = field(
-        default=None,
-        metadata={
-            "help": "Whether to apply top_k_before_softmax to the logits when using KD"
-        },
-    )
-
 
 @dataclass
 class AxolotlTrainingArguments(AxolotlTrainingMixins, TrainingArguments):
@@ -217,6 +206,13 @@ class AxolotlTrainingArguments(AxolotlTrainingMixins, TrainingArguments):
     """
 
 
+@dataclass
+class AxolotlDPOConfig(AxolotlTrainingMixins, DPOConfig):
+    """
+    DPO config for DPO training
+    """
+
+
 @dataclass
 class AxolotlORPOConfig(AxolotlTrainingMixins, ORPOConfig):
     """
@@ -248,10 +244,3 @@ class AxolotlRewardConfig(AxolotlTrainingMixins, RewardConfig):
     """
     Reward config for Reward training
     """
-
-
-@dataclass
-class AxolotlPRMConfig(AxolotlTrainingMixins, PRMConfig):
-    """
-    PRM config for PRM training
-    """

src/axolotl/datasets.py

@@ -52,17 +52,12 @@ class TokenizedPromptDataset(Dataset):
         if self.prompt_tokenizer.supports_batched:
             map_kwargs["batched"] = True
             map_kwargs["batch_size"] = 1_000
-
-        if (
-            hasattr(self.prompt_tokenizer, "filter_rows")
-            and self.prompt_tokenizer.filter_rows
-        ):
+        if self.prompt_tokenizer.filter_rows:
             dataset = dataset.filter(
                 self.prompt_tokenizer.filter_rows,
                 num_proc=num_proc,
                 desc="Strategy Filtering Rows",
             )
-
         return dataset.map(
             self.prompt_tokenizer.tokenize_prompt,
             num_proc=num_proc,

src/axolotl/integrations/kd/args.py

@@ -32,6 +32,3 @@ class KDArgs(BaseModel):
     kd_alpha: Optional[float] = None  # loss coefficient for KD loss
     kd_temperature: Optional[float] = None  # temperature for sampling during KD
     kd_zscore_base_temp: Optional[float] = None  # base temperature for zscore scaling
-    kd_top_k_before_softmax: Optional[
-        bool
-    ] = None  # whether to sample top k before softmax during KD

src/axolotl/integrations/kd/topk_logprob/forward_kl.py

@@ -58,7 +58,6 @@ def loss(
     target_mask: torch.Tensor,
     num_items_in_batch: int = -1,  # Use -1 to indicate "None"
     kd_temperature: float = 1.0,
-    top_k_before_softmax: int = 0,
 ) -> torch.Tensor:
     """
     A KD loss function that is TorchScript-friendly.
@@ -75,8 +74,6 @@ def loss(
         num_items_in_batch (int, optional): The number of items in the batch.
         kd_temperature (float, optional): The temperature for KD.
             Default: 1.0
-        top_k_before_softmax (int, optional): Flag of whether to apply softmax before gathering student top-k logits
-            Default: 0
     """
 
     target_logprobs = target_logprobs.float()
@@ -86,46 +83,26 @@ def loss(
     # student_logits shape:   [B, student_seq_len, vocab_size]
     teacher_seq_len = target_token_ids.shape[1]
 
-    if top_k_before_softmax:
-        # Slice student logits to match teacher-provided sequence length
-        student_logits_for_kd = student_logits[
-            :, :teacher_seq_len, :
-        ]  # [B, teacher_seq_len, vocab_size]
+    # Slice student logits to match teacher-provided sequence length
+    student_logits_for_kd = student_logits[
+        :, :teacher_seq_len, :
+    ]  # [B, teacher_seq_len, vocab_size]
 
-        # Gather student logits for teacher's top-K tokens
-        student_logits_topk = torch.gather(
-            student_logits_for_kd, dim=-1, index=target_token_ids
-        )  # [B, teacher_seq_len, K]
+    # Gather student logits for teacher's top-K tokens
+    student_logits_topk = torch.gather(
+        student_logits_for_kd, dim=-1, index=target_token_ids
+    )  # [B, teacher_seq_len, K]
 
-        student_logits_topk = student_logits_topk.float()
+    student_logits_topk = student_logits_topk.float()
 
-        # Apply KD temperature to student’s logits
-        if kd_temperature != 1.0:
-            student_logits_topk = student_logits_topk / kd_temperature
+    # Apply KD temperature to student’s logits
+    if kd_temperature != 1.0:
+        student_logits_topk = student_logits_topk / kd_temperature
 
-        # Convert student top-k logits to logprobs
-        student_logprobs_topk = student_logits_topk - torch.logsumexp(
-            student_logits_topk, dim=-1, keepdim=True
-        )  # [B, teacher_seq_len, K]
-    else:
-        # Slice student logits to match teacher-provided sequence length
-        student_logits_for_kd = (
-            student_logits[:, :teacher_seq_len, :] / kd_temperature
-        )  # [B, teacher_seq_len, vocab_size]
-
-        # keep in full precision for numerical stability of loss
-        student_logits_for_kd = student_logits_for_kd.float()
-
-        # Gather student logits for teacher's top-K tokens
-        student_logits_topk = torch.gather(
-            student_logits_for_kd, dim=-1, index=target_token_ids
-        )  # [B, teacher_seq_len, K]
-
-        # Compute logsumexp across full vocabulary
-        student_lse = torch.logsumexp(student_logits_for_kd, dim=-1, keepdim=True)
-
-        #  Convert just the top-k logits to logprobs
-        student_logprobs_topk = student_logits_topk - student_lse
+    # Convert student top-k logits to logprobs
+    student_logprobs_topk = student_logits_topk - torch.logsumexp(
+        student_logits_topk, dim=-1, keepdim=True
+    )  # [B, teacher_seq_len, K]
 
     # Convert teacher_mask to boolean for indexing
     # In TorchScript, .bool() is sometimes unsupported, so we do:

src/axolotl/integrations/kd/trainer.py

@@ -67,7 +67,7 @@ class AxolotlKDTrainer(AxolotlTrainer):
         outputs = model(**inputs)
 
         # FIXME: account for tokenizer.padding_side
-        student_logits = outputs["logits"][:, : seq_len - 1, :].contiguous()
+        student_logits = outputs["logits"][:, :seq_len, :].contiguous()
 
         shift_logits = student_logits.contiguous()
         target_logprobs_for_loss = target_logprobs[..., 1:, :].contiguous()
@@ -92,7 +92,6 @@ class AxolotlKDTrainer(AxolotlTrainer):
                 target_mask_for_loss,
                 num_items_in_batch=num_items_in_batch,
                 kd_temperature=self.args.kd_temperature,
-                top_k_before_softmax=1 if self.args.kd_top_k_before_softmax else 0,
             )
 
         if self.args.kd_ce_alpha > 0:

src/axolotl/integrations/lm_eval/__init__.py

@@ -2,9 +2,9 @@
 Module for the Plugin for LM Eval Harness
 """
 import subprocess  # nosec
+from datetime import datetime
 
 from axolotl.integrations.base import BasePlugin
-from axolotl.integrations.lm_eval.cli import build_lm_eval_command
 
 from .args import LMEvalArgs  # pylint: disable=unused-import. # noqa: F401
 
@@ -18,20 +18,25 @@ class LMEvalPlugin(BasePlugin):
         return "axolotl.integrations.lm_eval.LMEvalArgs"
 
     def post_train_unload(self, cfg):
-        if cfg.lm_eval_post_train:
-            # pylint: disable=duplicate-code
-            for lm_eval_args in build_lm_eval_command(
-                cfg.lm_eval_tasks,
-                bfloat16=cfg.bfloat16 or cfg.bf16,
-                flash_attention=cfg.flash_attention,
-                output_dir=cfg.output_dir,
-                batch_size=cfg.lm_eval_batch_size,
-                wandb_project=cfg.wandb_project,
-                wandb_entity=cfg.wandb_entity,
-                wandb_name=cfg.wandb_name,
-                model=cfg.lm_eval_model or cfg.hub_model_id,
-            ):
-                subprocess.run(  # nosec
-                    lm_eval_args,
-                    check=True,
-                )
+        tasks = ",".join(cfg.lm_eval_tasks)
+        fa2 = ",attn_implementation=flash_attention_2" if cfg.flash_attention else ""
+        dtype = ",dtype=bfloat16" if cfg.bf16 else ",dtype=float16"
+        output_path = cfg.output_dir
+        output_path += "" if cfg.output_dir.endswith("/") else "/"
+        output_path += "lm_eval_results/" + datetime.now().strftime("%Y%m%d_%H%M%S")
+        subprocess.run(  # nosec
+            [
+                "lm_eval",
+                "--model",
+                "hf",
+                "--model_args",
+                f"pretrained={cfg.output_dir}{fa2}{dtype}",
+                "--tasks",
+                tasks,
+                "--batch_size",
+                str(cfg.lm_eval_batch_size),
+                "--output_path",
+                output_path,
+            ],
+            check=True,
+        )

src/axolotl/integrations/lm_eval/args.py

@@ -13,5 +13,3 @@ class LMEvalArgs(BaseModel):
 
     lm_eval_tasks: List[str] = []
     lm_eval_batch_size: Optional[int] = 8
-    lm_eval_post_train: Optional[bool] = True
-    lm_eval_model: Optional[str] = None

src/axolotl/integrations/lm_eval/cli.py

@@ -1,119 +0,0 @@
-"""
-axolotl CLI for running lm_eval tasks
-"""
-import subprocess  # nosec
-from collections import defaultdict
-from datetime import datetime
-from typing import Optional
-
-import click
-import yaml
-
-from axolotl.utils.dict import DictDefault
-
-
-def build_lm_eval_command(
-    tasks: list[str],
-    bfloat16=True,
-    flash_attention=False,
-    output_dir="./",
-    batch_size=8,
-    wandb_project=None,
-    wandb_entity=None,
-    wandb_name=None,
-    model=None,
-    revision=None,
-    apply_chat_template=None,
-    fewshot_as_multiturn=None,
-):
-    tasks_by_num_fewshot: dict[str, list] = defaultdict(list)
-    if isinstance(tasks, str):
-        tasks = [tasks]
-    for task in tasks:
-        num_fewshot = "-1"
-        task_parts = task.split(":")
-        task_name = task_parts[0]
-        if len(task_parts) == 2:
-            task_name, num_fewshot = task_parts
-        tasks_by_num_fewshot[str(num_fewshot)].append(task_name)
-
-    for num_fewshot, tasks_list in tasks_by_num_fewshot.items():
-        tasks_str = ",".join(tasks_list)
-        num_fewshot_val = num_fewshot if num_fewshot != "-1" else None
-        pretrained = "pretrained="
-        pretrained += model if model else output_dir
-        fa2 = ",attn_implementation=flash_attention_2" if flash_attention else ""
-        dtype = ",dtype=bfloat16" if bfloat16 else ",dtype=float16"
-        revision = f",revision={revision}" if revision else ""
-        output_path = output_dir
-        output_path += "" if output_dir.endswith("/") else "/"
-        output_path += "lm_eval_results/" + datetime.now().strftime("%Y%m%d_%H%M%S")
-        lm_eval_args = [
-            "lm_eval",
-            "--model",
-            "hf",
-            "--model_args",
-            f"{pretrained}{fa2}{dtype}{revision}",
-            "--tasks",
-            tasks_str,
-            "--batch_size",
-            str(batch_size),
-            "--output_path",
-            output_path,
-        ]
-        wandb_args = []
-        if wandb_project:
-            wandb_args.append(f"project={wandb_project}")
-        if wandb_entity:
-            wandb_args.append(f"entity={wandb_entity}")
-        if wandb_name:
-            wandb_args.append(f"name={wandb_name}")
-        if wandb_args:
-            lm_eval_args.append("--wandb_args")
-            lm_eval_args.append(",".join(wandb_args))
-        if apply_chat_template:
-            lm_eval_args.append("--apply_chat_template")
-        if num_fewshot_val:
-            lm_eval_args.append("--num_fewshot")
-            lm_eval_args.append(str(num_fewshot_val))
-            if apply_chat_template and fewshot_as_multiturn:
-                lm_eval_args.append("--fewshot_as_multiturn")
-
-        yield lm_eval_args
-
-
-@click.command()
-@click.argument("config", type=click.Path(exists=True, path_type=str))
-@click.option("--cloud", default=None, type=click.Path(exists=True, path_type=str))
-def lm_eval(config: str, cloud: Optional[str] = None):
-    """
-    use lm eval to evaluate a trained language model
-    """
-
-    if cloud:
-        from axolotl.cli.cloud import do_cli_lm_eval
-
-        do_cli_lm_eval(cloud_config=cloud, config=config)
-    else:
-        with open(config, encoding="utf-8") as file:
-            cfg: DictDefault = DictDefault(yaml.safe_load(file))
-
-        # pylint: disable=duplicate-code
-        for lm_eval_args in build_lm_eval_command(
-            cfg.lm_eval_tasks,
-            bfloat16=cfg.bfloat16 or cfg.bf16,
-            flash_attention=cfg.flash_attention,
-            output_dir=cfg.output_dir,
-            batch_size=cfg.lm_eval_batch_size,
-            wandb_project=cfg.wandb_project,
-            wandb_entity=cfg.wandb_entity,
-            wandb_name=cfg.wandb_name,
-            model=cfg.lm_eval_model or cfg.hub_model_id,
-            revision=cfg.revision,
-            apply_chat_template=cfg.apply_chat_template,
-            fewshot_as_multiturn=cfg.fewshot_as_multiturn,
-        ):
-            subprocess.run(  # nosec
-                lm_eval_args,
-                check=True,
-            )

src/axolotl/monkeypatch/trainer_grad_accum.py

@@ -0,0 +1,308 @@
+"""
+fix for FSDP gradient accumulation
+see https://github.com/huggingface/transformers/pull/35128
+"""
+import inspect
+import logging
+
+from transformers import LlamaForCausalLM, Trainer
+from transformers.modeling_flash_attention_utils import _flash_attention_forward
+
+from axolotl.monkeypatch.utils import detab_code
+
+LOG = logging.getLogger("axolotl.monkeypatch.trainer_grad_accum")
+
+ORIGINAL_CONTEXT_CODE = """
+    with self.compute_loss_context_manager():
+        loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
+"""
+
+PATCHED_CONTEXT_CODE = """
+    with self.compute_loss_context_manager():
+        if self.model_accepts_loss_kwargs:
+            loss = self.compute_loss(model, inputs, num_items_in_batch=num_items_in_batch)
+        else:
+            loss = self.compute_loss(model, inputs)
+"""
+
+ORIGINAL_LLAMA_FCLM_CODE = """
+    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+    output_hidden_states = (
+        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+    )
+    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+    # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+    outputs = self.model(
+        input_ids=input_ids,
+        attention_mask=attention_mask,
+        position_ids=position_ids,
+        past_key_values=past_key_values,
+        inputs_embeds=inputs_embeds,
+        use_cache=use_cache,
+        output_attentions=output_attentions,
+        output_hidden_states=output_hidden_states,
+        return_dict=return_dict,
+        cache_position=cache_position,
+        **kwargs,
+    )
+
+    hidden_states = outputs[0]
+    # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+    logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :])
+
+    loss = None
+    if labels is not None:
+        loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **kwargs)
+"""
+
+PATCHED_LLAMA_FCLM_CODE = """
+    output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+    output_hidden_states = (
+        output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+    )
+    return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+    # remove num_items_in_batch otherwise self.model attempts to pass it to flash_attention
+    num_items_in_batch = kwargs.pop("num_items_in_batch", None)
+
+    # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+    outputs = self.model(
+        input_ids=input_ids,
+        attention_mask=attention_mask,
+        position_ids=position_ids,
+        past_key_values=past_key_values,
+        inputs_embeds=inputs_embeds,
+        use_cache=use_cache,
+        output_attentions=output_attentions,
+        output_hidden_states=output_hidden_states,
+        return_dict=return_dict,
+        cache_position=cache_position,
+        **kwargs,
+    )
+    hidden_states = outputs[0]
+    # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+    logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :])
+
+    loss = None
+    if labels is not None:
+        loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, num_items_in_batch=num_items_in_batch, **kwargs)
+"""
+
+
+def get_training_step_code() -> str:
+    training_step = inspect.getsource(
+        Trainer.training_step  # pylint: disable=protected-access
+    )
+    return training_step
+
+
+def check_training_step_is_patchable() -> bool:
+    training_step = get_training_step_code()
+    training_step, _ = detab_code(training_step)
+    return ORIGINAL_CONTEXT_CODE in training_step
+
+
+def patch_training_step_for_ga():
+    """
+    monkeypatch for fixing the training loop for gradient accumulation
+    """
+
+    try:
+        training_step = get_training_step_code()
+    except OSError:
+        return
+    Trainer._original_training_step = training_step  # pylint: disable=protected-access
+    training_step, _ = detab_code(training_step)
+    if ORIGINAL_CONTEXT_CODE not in training_step:
+        return
+    # assert (
+    #     ORIGINAL_CONTEXT_CODE in training_step
+    # ), "Original training_step code not found"
+
+    training_step = training_step.replace(ORIGINAL_CONTEXT_CODE, PATCHED_CONTEXT_CODE)
+    training_step = training_step.replace(
+        "def training_step(",
+        "def _fixed_training_step(",
+        1,
+    )
+
+    # load imports necessary
+    import transformers.trainer
+
+    items_to_import = []
+    for item in dir(transformers.trainer):
+        if item in training_step:
+            items_to_import.append(item)
+
+    exec(  # pylint: disable=exec-used  # nosec B102
+        "from transformers.trainer import ("
+        + ", ".join(x for x in items_to_import)
+        + ")",
+        globals(),
+    )
+    exec(training_step, globals())  # pylint: disable=exec-used  # nosec B102
+    LOG.info("patching training_step")
+    Trainer.training_step = (  # pylint: disable=protected-access
+        _fixed_training_step  # pylint: disable=undefined-variable  # noqa: F821
+    )
+
+
+def get_model_forward_code() -> str:
+    forward = inspect.getsource(
+        LlamaForCausalLM.forward  # pylint: disable=protected-access
+    )
+    return forward
+
+
+def check_forward_is_patchable() -> bool:
+    forward = get_model_forward_code()
+    forward, _ = detab_code(forward)
+    return ORIGINAL_LLAMA_FCLM_CODE in forward
+
+
+def patch_forward_for_ga():
+    """
+    monkeypatch for fixing the training loop for gradient accumulation
+    """
+
+    try:
+        forward = get_model_forward_code()
+    except OSError:
+        return
+    LlamaForCausalLM._original_forward = forward  # pylint: disable=protected-access
+    forward, _ = detab_code(forward)
+    if ORIGINAL_LLAMA_FCLM_CODE not in forward:
+        return
+    # assert ORIGINAL_LLAMA_FCLM_CODE in forward, "Original forward code not found"
+
+    forward = forward.replace(ORIGINAL_LLAMA_FCLM_CODE, PATCHED_LLAMA_FCLM_CODE)
+    forward = forward.replace(
+        "def forward(",
+        "def _fixed_forward(",
+        1,
+    )
+
+    # load imports necessary
+    import transformers.models.llama.modeling_llama
+
+    items_to_import = []
+    for item in dir(transformers.models.llama.modeling_llama):
+        if item in forward:
+            items_to_import.append(item)
+
+    exec(  # pylint: disable=exec-used  # nosec B102
+        "from transformers.models.llama.modeling_llama import ("
+        + ", ".join(x for x in items_to_import)
+        + ")",
+        globals(),
+    )
+    exec(forward, globals())  # pylint: disable=exec-used  # nosec B102
+    LOG.info("patching forward")
+    LlamaForCausalLM.forward = (  # pylint: disable=protected-access
+        _fixed_forward  # pylint: disable=undefined-variable  # noqa: F821
+    )
+
+
+ORIGINAL_TRAINER_CODE = """
+                context = (
+                    functools.partial(self.accelerator.no_sync, model=model)
+                    if i != len(batch_samples) - 1
+                    else contextlib.nullcontext
+                )
+                with context():
+                    tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
+"""
+
+PATCHED_TRAINER_CODE = """
+                disable_deepspeed_no_sync = (
+                        self.accelerator.distributed_type == DistributedType.DEEPSPEED
+                        # and self.accelerator.deepspeed_engine_wrapped.engine.zero_optimization_partition_gradients()
+                )
+                context = (
+                    functools.partial(self.accelerator.no_sync, model=model)
+                    if i != len(batch_samples) - 1 and not disable_deepspeed_no_sync
+                    else contextlib.nullcontext
+                )
+                with context():
+                    tr_loss_step = self.training_step(model, inputs, num_items_in_batch)
+"""
+
+
+def get_training_loop_code() -> str:
+    training_loop = inspect.getsource(
+        Trainer._inner_training_loop  # pylint: disable=protected-access
+    )
+    return training_loop
+
+
+def check_training_loop_is_patchable() -> bool:
+    training_loop = get_training_loop_code()
+    training_loop, _ = detab_code(training_loop)
+    return ORIGINAL_TRAINER_CODE in training_loop
+
+
+def patch_training_loop_for_deepspeed_0_16_x():
+    """
+    monkeypatch for fixing the training loop for deepspeed GA
+
+    see https://github.com/huggingface/transformers/pull/35157
+    """
+
+    try:
+        training_loop = get_training_loop_code()
+    except OSError:
+        return
+    Trainer._original_inner_training_loop = (  # pylint: disable=protected-access
+        training_loop
+    )
+    training_loop, _ = detab_code(training_loop)
+    if ORIGINAL_TRAINER_CODE not in training_loop:
+        return
+
+    training_loop = training_loop.replace(ORIGINAL_TRAINER_CODE, PATCHED_TRAINER_CODE)
+    training_loop = training_loop.replace(
+        "def _inner_training_loop(",
+        "def _fixed_inner_training_loop(",
+        1,
+    )
+
+    # load imports necessary
+    import transformers.trainer
+
+    items_to_import = []
+    for item in dir(transformers.trainer):
+        if item in training_loop:
+            items_to_import.append(item)
+
+    exec(  # pylint: disable=exec-used  # nosec B102
+        "from transformers.trainer import ("
+        + ", ".join(x for x in items_to_import)
+        + ")",
+        globals(),
+    )
+    exec(training_loop, globals())  # pylint: disable=exec-used  # nosec B102
+    LOG.info("patching _inner_training_loop for fsdp optimizer save")
+    Trainer._inner_training_loop = (  # pylint: disable=protected-access
+        _fixed_inner_training_loop  # pylint: disable=undefined-variable  # noqa: F821
+    )
+
+
+def patch_flash_attention_forward():
+    """
+    monkeypatch for fixing the forward pass for flash attention to ignore num_items_in_batch
+    """
+
+    import transformers.modeling_flash_attention_utils
+
+    def proxy_flash_attention_forward(*args, **kwargs):
+        kwargs.pop("num_items_in_batch", None)
+
+        return _flash_attention_forward(*args, **kwargs)
+
+    transformers.modeling_flash_attention_utils._flash_attention_forward = (  # pylint: disable=protected-access
+        proxy_flash_attention_forward
+    )
+    transformers.models.llama.modeling_llama._flash_attention_forward = (  # pylint: disable=protected-access
+        proxy_flash_attention_forward
+    )

src/axolotl/monkeypatch/transformers_fa_utils.py

@@ -1,67 +0,0 @@
-"""
-see https://github.com/huggingface/transformers/pull/35834
-"""
-
-import logging
-from functools import partial
-from typing import Optional
-
-import torch
-
-logger = logging.getLogger(__name__)
-
-
-def fixed_fa_peft_integration_check(
-    query: torch.Tensor,
-    key: torch.Tensor,
-    value: torch.Tensor,
-    target_dtype: Optional[torch.dtype] = None,
-    preferred_dtype: Optional[torch.dtype] = None,
-):
-    """
-    PEFT usually casts the layer norms in float32 for training stability reasons
-    therefore the input hidden states gets silently casted in float32. Hence, we need
-    cast them back in float16 / bfloat16 just to be sure everything works as expected.
-    This might slowdown training & inference so it is recommended to not cast the LayerNorms!
-
-    Args:
-        query (`torch.Tensor`):
-            Input query states to be passed to Flash Attention API
-        key (`torch.Tensor`):
-            Input key states to be passed to Flash Attention API
-        value (`torch.Tensor`):
-            Input value states to be passed to Flash Attention API
-        target_dtype (`torch.dtype`, *optional*):
-            The dtype to convert the attention tensors to. Conversion can be ignored by
-            not providing the target dtype.
-        preferred_dtype (`torch.dtype`, *optional*):
-            The preferred dtype to convert the attention tensors to regardless of the
-            target dtype.
-    """
-    if target_dtype is None and preferred_dtype is None:
-        return query, key, value
-
-    if preferred_dtype and target_dtype != preferred_dtype:
-        target_dtype = preferred_dtype
-
-    # check if any of query, key, or value are in float32. If so, cast them back to target dtype.
-    if any(module.dtype == torch.float32 for module in [query, key, value]):
-        logger.warning_once(
-            f"The input hidden states seems to be silently casted in float32, this might be related to"
-            f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
-            f" {target_dtype}."
-        )
-
-        query = query.to(target_dtype)
-        key = key.to(target_dtype)
-        value = value.to(target_dtype)
-
-    return query, key, value
-
-
-def patch_fa_peft_integration():
-    import transformers.modeling_flash_attention_utils
-
-    transformers.modeling_flash_attention_utils.fa_peft_integration_check = partial(
-        fixed_fa_peft_integration_check, preferred_dtype=None
-    )

src/axolotl/prompt_strategies/base.py

@@ -13,19 +13,8 @@ def load(strategy, cfg, module_base=None, **kwargs):
         if len(strategy.split(".")) == 1:
             strategy = strategy + ".default"
         load_fn = strategy.split(".")[-1]
-        if len(strategy.split(".")) > 1:
-            try:
-                importlib.import_module(
-                    strategy.split(".")[-2],
-                    ".".join(strategy.split(".")[:-2]),
-                )
-                module_base = ".".join(strategy.split(".")[:-2])
-                strategy = strategy.split(".")[-2]
-            except ModuleNotFoundError:
-                strategy = "." + ".".join(strategy.split(".")[:-1])
-        else:
-            strategy = "." + ".".join(strategy.split(".")[:-1])
-        mod = importlib.import_module(strategy, module_base)
+        strategy = ".".join(strategy.split(".")[:-1])
+        mod = importlib.import_module(f".{strategy}", module_base)
         func = getattr(mod, load_fn)
         return func(cfg, **kwargs)
     except Exception:  # pylint: disable=broad-exception-caught

src/axolotl/prompt_strategies/bradley_terry/chat_template.py

@@ -47,7 +47,7 @@ class BTChatTemplateStrategy(ChatTemplateStrategy):
 
         if len(chosen_tokenized["input_ids"]) > max_length:
             LOG.warning(
-                f"To-be-trimmed chosen sequence exceeds max sequence length: {len(chosen_tokenized['input_ids'])}",
+                f"Chosen sequence exceeds max sequence length: {len(chosen_tokenized['input_ids'])}",
             )
 
             chosen_tokenized["input_ids"] = chosen_tokenized["input_ids"][:max_length]
@@ -70,7 +70,7 @@ class BTChatTemplateStrategy(ChatTemplateStrategy):
 
         if len(rejected_tokenized["input_ids"]) > max_length:
             LOG.warning(
-                f"To-be-trimmed rejected sequence exceeds max sequence length: {len(rejected_tokenized['input_ids'])}",
+                f"Rejected sequence exceeds max sequence length: {len(rejected_tokenized['input_ids'])}",
             )
 
             rejected_tokenized["input_ids"] = rejected_tokenized["input_ids"][

src/axolotl/prompt_strategies/dpo/passthrough.py

@@ -1,14 +0,0 @@
-"""
-DPO prompt strategies passthrough/zero-processing strategy
-"""
-
-
-def default(
-    cfg, dataset_idx=0, **kwargs
-):  # pylint: disable=possibly-unused-variable,unused-argument
-    def transform_fn(
-        sample, tokenizer=None
-    ):  # pylint: disable=possibly-unused-variable,unused-argument
-        return sample
-
-    return transform_fn

src/axolotl/prompt_strategies/stepwise_supervised.py

@@ -1,116 +0,0 @@
-"""
-Module for stepwise datasets, typically including a prompt and reasoning traces,
-and (optionally) per-step, or per-prompt-trace labels for reward modelling.
-"""
-
-from itertools import chain
-from typing import Dict, List, Optional, Union
-
-from transformers import BatchEncoding, PreTrainedTokenizer
-
-from axolotl.prompt_tokenizers import IGNORE_INDEX
-from axolotl.utils.dict import DictDefault
-
-
-class StepwiseSupervisedPromptTokenizingStrategy:
-    """
-    Tokenizing strategy for supervised stepwise datasets, typically used for COT-reasoning.
-    These datasets should include the following columns:
-    - prompt: the prompt text
-    - completions: a list of `n` completion steps
-    - labels: a list of `n` labels indicating the "correctness" of each step
-    """
-
-    def __init__(
-        self,
-        tokenizer,
-        sequence_len: int = 2048,
-        step_separator: str = "\n",
-        max_completion_length: Optional[int] = None,
-        train_on_last_step_only: bool = False,
-    ):
-        self.tokenizer = tokenizer
-        self.sequence_len = sequence_len
-        self.step_separator = step_separator
-        self.max_completion_length = max_completion_length
-        self.train_on_last_step_only = train_on_last_step_only
-
-    def tokenize_prompt(
-        self, prompt: Dict[str, Union[str, List[str]]]
-    ) -> BatchEncoding:
-        # Inspired by TRL's PRMTRainer
-        # https://github.com/huggingface/trl/blob/ed7de87dc766478c024b68f12530d1b0e7c3ff23/trl/trainer/prm_trainer.py#L206
-        prompt_ids = self.tokenizer(prompt["prompt"], add_special_tokens=False)[
-            "input_ids"
-        ]
-
-        completions_ids = [
-            self.tokenizer(completion, add_special_tokens=False)["input_ids"]
-            for completion in prompt["completions"]
-        ]
-
-        # Handle labels
-        if self.train_on_last_step_only:
-            labels = [IGNORE_INDEX] * (len(prompt["labels"]) - 1) + [
-                int(prompt["labels"][-1])
-            ]
-        else:
-            labels = [int(label) for label in prompt["labels"]]
-
-        # Add step separators
-        separator_ids = self.tokenizer.encode(
-            self.step_separator, add_special_tokens=False
-        )
-        completions_ids = [completion + separator_ids for completion in completions_ids]
-
-        # Create step-wise labels
-        labels = [
-            [IGNORE_INDEX] * (len(completion) - 1) + [label]  # type: ignore
-            for completion, label in zip(completions_ids, labels)
-        ]
-
-        # Join all steps
-        completion_ids = list(chain(*completions_ids))
-        labels = list(chain(*labels))  # type: ignore
-
-        # Handle max lengths
-        if self.max_completion_length:
-            completion_ids = completion_ids[: self.max_completion_length]
-            labels = labels[: self.max_completion_length]
-
-        # Add BOS token if model has one
-        if self.tokenizer.bos_token_id is not None:
-            prompt_ids = [self.tokenizer.bos_token_id] + prompt_ids
-
-        # Combine prompt and completion
-        input_ids = prompt_ids + completion_ids
-
-        full_labels = [IGNORE_INDEX] * len(prompt_ids) + labels
-        # Apply max sequence length
-        if self.sequence_len:
-            input_ids = input_ids[: self.sequence_len]
-            full_labels = full_labels[: self.sequence_len]
-
-        return {
-            "input_ids": input_ids,
-            "labels": full_labels,
-            "attention_mask": [1] * len(input_ids),
-        }
-
-    @property
-    def supports_batched(self):
-        return False
-
-
-def load(
-    tokenizer: PreTrainedTokenizer,
-    cfg: DictDefault,
-    ds_cfg: DictDefault,
-) -> StepwiseSupervisedPromptTokenizingStrategy:
-    return StepwiseSupervisedPromptTokenizingStrategy(
-        tokenizer,
-        cfg.sequence_len,
-        step_separator=ds_cfg.get("step_separator", "\n"),
-        max_completion_length=ds_cfg.max_completion_length,
-        train_on_last_step_only=ds_cfg.get("train_on_last_step_only", False),
-    )

src/axolotl/train.py

@@ -141,9 +141,7 @@ def train(
         model.config.save_pretrained(str(Path(cfg.output_dir)))
 
     # In case we want to stop early with ctrl+c, this is a nice to have to save the pretrained model
-    if (
-        cfg.local_rank == 0 and not cfg.use_ray
-    ):  # ray workers don't have access to this signal
+    if cfg.local_rank == 0:
 
         def terminate_handler(_, __, model_weakref):
             if model_weakref() is not None:
@@ -261,7 +259,7 @@ def train(
                 .decode("utf-8")
             }
             if cfg.datasets is not None:
-                if cfg.rl is not None or cfg.reward_model or cfg.process_reward_model:
+                if cfg.rl is not None or cfg.reward_model:
                     dataset_tags = [
                         d["path"] for d in cfg.datasets if not Path(d["path"]).is_dir()
                     ]

src/axolotl/utils/callbacks/__init__.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import gc
 import logging
+import math
 import os
 import traceback
 from shutil import copyfile
@@ -829,6 +830,13 @@ class SaveModelCallback(TrainerCallback):
         # Save
         if state.global_step >= state.max_steps:
             control.should_save = True
+        elif (
+            args.save_strategy == IntervalStrategy.STEPS
+            and state.save_steps < 1.0
+            and state.global_step % math.ceil(state.save_steps * state.max_steps) == 0
+        ):
+            # workaround to save model on fractional save_steps
+            control.should_save = True
 
     def on_train_end(  # pylint: disable=unused-argument
         self, args, state, control, **kwargs
@@ -846,12 +854,6 @@ class GCCallback(TrainerCallback):
     def on_step_end(
         self, args, state, control, **kwargs  # pylint: disable=unused-argument
     ):
-        if self.gc_steps > 0 and state.global_step % self.gc_steps == 0:
+        if state.global_step % self.gc_steps == 0:
             torch.cuda.empty_cache()
             gc.collect()
-
-    def on_epoch_end(
-        self, args, state, control, **kwargs  # pylint: disable=unused-argument
-    ):
-        torch.cuda.empty_cache()
-        gc.collect()

src/axolotl/utils/config/__init__.py

@@ -1,6 +1,4 @@
 """Module for working with config dicts"""
-
-import json
 import logging
 import os
 from typing import Optional
@@ -58,10 +56,33 @@ def choose_device(cfg):
         cfg.device_map = None
 
 
-def resolve_dtype(cfg):
-    if (
-        cfg.bf16 == "auto" and not cfg.use_ray
-    ):  # if we use ray we want to defer this check to the worker node
+def normalize_config(cfg):
+    # setup some derived config / hyperparams
+    cfg.gradient_accumulation_steps = cfg.gradient_accumulation_steps or (
+        cfg.batch_size // cfg.micro_batch_size
+    )
+    cfg.batch_size = (
+        cfg.batch_size or cfg.micro_batch_size * cfg.gradient_accumulation_steps
+    )
+    if cfg.eval_batch_size is None:
+        cfg.eval_batch_size = cfg.micro_batch_size
+    cfg.world_size = int(os.environ.get("WORLD_SIZE", 1))
+    cfg.local_rank = int(os.environ.get("LOCAL_RANK", 0))
+    cfg.eval_table_size = cfg.eval_table_size or 0
+    cfg.eval_max_new_tokens = cfg.eval_max_new_tokens or 128
+    cfg.eval_causal_lm_metrics = cfg.eval_causal_lm_metrics or [
+        "sacrebleu",
+        "comet",
+        "ter",
+        "chrf",
+    ]
+    choose_device(cfg)
+    cfg.ddp = cfg.ddp if cfg.ddp is not None else cfg.world_size != 1
+    if cfg.ddp:
+        cfg.device_map = {"": int(os.environ.get("LOCAL_RANK", 0))}
+        cfg.batch_size = cfg.batch_size * cfg.world_size
+
+    if cfg.bf16 == "auto":
         if is_torch_bf16_gpu_available():
             LOG.debug("bf16 support detected, enabling for this configuration.")
             cfg.bf16 = True
@@ -89,59 +110,14 @@ def resolve_dtype(cfg):
     else:
         cfg.torch_dtype = torch.float32
 
-
-def normalize_config(cfg):
-    # setup some derived config / hyperparams
-    cfg.gradient_accumulation_steps = cfg.gradient_accumulation_steps or (
-        cfg.batch_size // cfg.micro_batch_size
-    )
-    cfg.batch_size = (
-        cfg.batch_size or cfg.micro_batch_size * cfg.gradient_accumulation_steps
-    )
-    if cfg.eval_batch_size is None:
-        cfg.eval_batch_size = cfg.micro_batch_size
-    cfg.world_size = int(os.environ.get("WORLD_SIZE", 1))
-    cfg.local_rank = int(os.environ.get("LOCAL_RANK", 0))
-    cfg.eval_table_size = cfg.eval_table_size or 0
-    cfg.eval_max_new_tokens = cfg.eval_max_new_tokens or 128
-    cfg.eval_causal_lm_metrics = cfg.eval_causal_lm_metrics or [
-        "sacrebleu",
-        "comet",
-        "ter",
-        "chrf",
-    ]
-    choose_device(cfg)
-    cfg.ddp = cfg.ddp if cfg.ddp is not None else cfg.world_size != 1
-    if cfg.ddp:
-        cfg.device_map = {"": int(os.environ.get("LOCAL_RANK", 0))}
-        cfg.batch_size = cfg.batch_size * cfg.world_size
-
-    if not cfg.use_ray:
-        # delay resolving dtype until on worker node when launching with ray
-        resolve_dtype(cfg)
-
-    if cfg.deepspeed:
-        if isinstance(cfg.deepspeed, str) and os.path.exists(cfg.deepspeed):
-            ds_config_path = cfg.deepspeed
-            with open(ds_config_path, encoding="utf-8") as f:
-                cfg.deepspeed = json.load(f)
-
     if cfg.saves_per_epoch:
         save_steps = 1.0 / (cfg.saves_per_epoch * cfg.num_epochs)
         if save_steps < 1.0:  # prevent saves on every step
             cfg.save_steps = save_steps
-        elif save_steps > 1:
-            LOG.warning(
-                f"Invalid value for save_steps ({save_steps}) from saves_per_epoch and/or num_epochs. Saving at training end only."
-            )
     if (cfg.val_set_size or cfg.test_datasets) and cfg.evals_per_epoch:
         eval_steps = 1.0 / (cfg.evals_per_epoch * cfg.num_epochs)
         if eval_steps < 1.0:  # prevent evals on every step
             cfg.eval_steps = eval_steps
-        elif eval_steps > 1:
-            LOG.warning(
-                f"Invalid value for eval_steps ({eval_steps}) from evals_per_epoch and/or num_epochs. Skipping evaluations."
-            )
 
     cfg.dataset_processes = cfg.dataset_processes or os.cpu_count()
 

src/axolotl/utils/config/models/input/v0_4_1/__init__.py

@@ -24,8 +24,6 @@ from transformers.utils.import_utils import is_torch_npu_available
 
 from axolotl.utils.config.models.internals import EnvCapabilities, GPUCapabilities
 
-from .trl import TrlConfig
-
 LOG = logging.getLogger("axolotl.utils.config.models.input")
 
 SUPPORTED_METRICS = {"sacrebleu", "comet", "ter", "chrf", "perplexity"}
@@ -35,7 +33,6 @@ class RLType(str, Enum):
     """RL trainer type configuration subset"""
 
     dpo = "dpo"  # pylint: disable=invalid-name
-    grpo = "grpo"  # pylint: disable=invalid-name
     ipo = "ipo"  # pylint: disable=invalid-name
     orpo = "orpo"  # pylint: disable=invalid-name
     kto = "kto"  # pylint: disable=invalid-name
@@ -150,14 +147,6 @@ class UserDefinedPrompterType(BaseModel):
     field: Optional[str] = None
 
 
-class LrGroup(BaseModel):
-    """Custom learning rate group configuration"""
-
-    name: str
-    modules: List[str]
-    lr: float
-
-
 class SFTDataset(BaseModel):
     """SFT configuration subset"""
 
@@ -242,18 +231,6 @@ class DPODataset(BaseModel):
     revision: Optional[str] = None
 
 
-class StepwiseSupervisedDataset(BaseModel):
-    """Stepwise supervised dataset configuration subset"""
-
-    path: Optional[str] = None
-    split: Optional[str] = None
-    data_files: Optional[List[str]] = None
-    revision: Optional[str] = None
-    step_separator: Optional[str] = None
-    max_completion_length: Optional[int] = None
-    train_on_last_step_only: Optional[bool] = None
-
-
 class UserDefinedKTOType(BaseModel):
     """User defined typing for KTO"""
 
@@ -501,13 +478,12 @@ class HyperparametersConfig(BaseModel):
     cosine_min_lr_ratio: Optional[float] = None
     cosine_constant_lr_ratio: Optional[float] = None
     lr_div_factor: Optional[float] = None
-    lr_groups: Optional[List[LrGroup]] = None
 
     adam_epsilon: Optional[float] = None
     adam_beta1: Optional[float] = None
     adam_beta2: Optional[float] = None
     max_grad_norm: Optional[float] = None
-    num_epochs: float = Field(default=1.0)
+    num_epochs: int = Field(default=1)
 
     @field_validator("batch_size")
     @classmethod
@@ -534,7 +510,7 @@ class ModelOutputConfig(BaseModel):
     output_dir: str = Field(default="./model-out")
     hub_model_id: Optional[str] = None
     hub_strategy: Optional[str] = None
-    save_safetensors: Optional[bool] = True
+    save_safetensors: Optional[bool] = None
 
 
 class MLFlowConfig(BaseModel):
@@ -613,30 +589,6 @@ class GradioConfig(BaseModel):
     gradio_temperature: Optional[float] = None
 
 
-class RayConfig(BaseModel):
-    """Ray launcher configuration subset"""
-
-    use_ray: bool = Field(default=False)
-    ray_run_name: Optional[str] = Field(
-        default=None,
-        metadata={
-            "help": "The training results will be saved at `saves/ray_run_name`."
-        },
-    )
-    ray_num_workers: int = Field(
-        default=1,
-        metadata={
-            "help": "The number of workers for Ray training. Default is 1 worker."
-        },
-    )
-    resources_per_worker: dict = Field(
-        default_factory=lambda: {"GPU": 1},
-        metadata={
-            "help": "The resources per worker for Ray training. Default is to use 1 GPU per worker."
-        },
-    )
-
-
 # pylint: disable=too-many-public-methods,too-many-ancestors
 class AxolotlInputConfig(
     ModelInputConfig,
@@ -649,7 +601,6 @@ class AxolotlInputConfig(
     CometConfig,
     LISAConfig,
     GradioConfig,
-    RayConfig,
     RemappedParameters,
     DeprecatedParameters,
     BaseModel,
@@ -666,23 +617,15 @@ class AxolotlInputConfig(
     auto_resume_from_checkpoints: Optional[bool] = None
     resize_token_embeddings_to_32x: Optional[bool] = None
     mean_resizing_embeddings: Optional[bool] = False
-    # optionally shrink the embeddings when the tokenizer vocab size is smaller
-    shrink_embeddings: Optional[bool] = None
 
     rl: Optional[RLType] = None
-    trl: Optional[TrlConfig] = Field(
-        default_factory=lambda: TrlConfig(),  # pylint: disable=unnecessary-lambda
-    )
     reward_model: Optional[bool] = None
-    process_reward_model: Optional[bool] = None
-    num_labels: Optional[int] = None
     dpo_use_weighting: Optional[
         bool
     ] = None  # whether to use weighting in DPO trainer. If none, default is false in the trainer.
-    dpo_use_logits_to_keep: Optional[bool] = None
 
-    datasets: Optional[conlist(Union[SFTDataset, DPODataset, KTODataset, StepwiseSupervisedDataset], min_length=1)] = None  # type: ignore
-    test_datasets: Optional[conlist(Union[SFTDataset, DPODataset, KTODataset, StepwiseSupervisedDataset], min_length=1)] = None  # type: ignore
+    datasets: Optional[conlist(Union[SFTDataset, DPODataset, KTODataset], min_length=1)] = None  # type: ignore
+    test_datasets: Optional[conlist(Union[SFTDataset, DPODataset, KTODataset], min_length=1)] = None  # type: ignore
     shuffle_merged_datasets: Optional[bool] = True
     dataset_prepared_path: Optional[str] = None
     dataset_shard_num: Optional[int] = None
@@ -766,12 +709,6 @@ class AxolotlInputConfig(
     pad_to_sequence_len: Optional[bool] = None
     curriculum_sampling: Optional[bool] = None
     multipack_real_batches: Optional[bool] = None
-    pretraining_sample_concatenation: Optional[bool] = Field(
-        default=None,
-        json_schema_extra={
-            "description": "whether to soft pack/concatenate samples during pretraining",
-        },
-    )
 
     batch_flattening: Optional[Union[Literal["auto"], bool]] = None
 

src/axolotl/utils/config/models/input/v0_4_1/trl.py

@@ -1,32 +0,0 @@
-"""
-GRPO specific configuration args
-"""
-from typing import List, Optional
-
-from pydantic import BaseModel, Field
-
-
-class TrlConfig(BaseModel):
-    """
-    Input args for TRL.
-    """
-
-    beta: Optional[float] = None
-    max_completion_length: Optional[int] = Field(
-        default=None,
-        json_schema_extra={
-            "description": "Maximum length of the completion for RL training"
-        },
-    )
-
-    # GRPO specific args
-    use_vllm: Optional[bool] = False
-    vllm_device: Optional[str] = "auto"
-    vllm_gpu_memory_utilization: Optional[float] = 0.9
-    vllm_max_model_len: Optional[int] = None
-    vllm_dtype: Optional[str] = "auto"
-    reward_funcs: Optional[List[str]] = None
-    num_generations: Optional[int] = None
-    sync_ref_model: Optional[bool] = False
-    ref_model_mixup_alpha: Optional[float] = 0.9
-    ref_model_sync_steps: Optional[int] = 64

src/axolotl/utils/data/pretraining.py

@@ -22,7 +22,6 @@ def encode_pretraining(
     max_tokens: int,
     examples: Dict[str, List],
     text_column: str = "text",
-    concatenate: bool = True,
 ) -> Dict[str, List]:
     res = tokenizer(
         examples[text_column],
@@ -34,13 +33,6 @@ def encode_pretraining(
     input_ids = [torch.tensor(seq) for seq in res["input_ids"]]
     targets = [torch.tensor(seq) for seq in res["input_ids"]]
     attention_mask = [torch.tensor(seq) for seq in res["attention_mask"]]
-    if not concatenate:
-        return {
-            "input_ids": [seq.tolist() for seq in input_ids],
-            "labels": [seq.tolist() for seq in targets],
-            "attention_mask": [seq.tolist() for seq in attention_mask],
-        }
-
     new_input_ids = []
     new_labels = []
     new_attention_mask = []
@@ -191,7 +183,7 @@ def wrap_pretraining_dataset(
             tokenizer,
             return_tensors="pt",
             padding=True,
-            pad_to_multiple_of=max_tokens,
+            pad_to_multiple_of=max_tokens * batch_size,
             multipack_attn=cfg.pretrain_multipack_attn,
         )
         encode = functools.partial(
@@ -201,6 +193,8 @@ def wrap_pretraining_dataset(
             max_seq_length=max_tokens,
             batch_size=batch_size,
             multipack_attn=cfg.pretrain_multipack_attn,
+            group_size=cfg.sample_packing_group_size,
+            bin_size=cfg.sample_packing_bin_size,
         )
         # set this to 1 so downstream data_loader doesn't try to increase the batch again
         cfg.micro_batch_size = 1
@@ -210,7 +204,6 @@ def wrap_pretraining_dataset(
             tokenizer,
             max_tokens,
             text_column=cfg.pretraining_dataset[0].text_column or "text",
-            concatenate=cfg.pretraining_sample_concatenation is True,
         )
 
     if cfg.shuffle_merged_datasets:
@@ -245,7 +238,9 @@ def encode_packed_pretraining(
     examples: Dict[str, List],
     max_seq_length: int = 2048,
     batch_size: int = 4,
-    multipack_attn: Optional[bool] = True,
+    multipack_attn: Optional[bool] = False,
+    group_size: int = 100000,
+    bin_size: int = 200,
 ) -> Dict[str, List]:
     # pylint: disable=duplicate-code
     # tokenize all the examples
@@ -256,9 +251,6 @@ def encode_packed_pretraining(
         train_dataset,
         max_seq_length,
         skip_position_ids=not multipack_attn,
-        # FIXME using attention mask unpad/pad with trainer and packed pretraining is broken atm
-        # workaround by using the position id logic for now in trainer
-        drop_attention_mask=multipack_attn,
     )
 
     sampler = MultipackBatchSampler(
@@ -266,6 +258,8 @@ def encode_packed_pretraining(
         lengths=get_dataset_lengths(train_dataset),
         batch_size=1,
         batch_max_len=batch_size * max_seq_length,
+        group_size=group_size,
+        bin_size=bin_size,
         drop_last=True,
     )
 

src/axolotl/utils/data/rl.py

@@ -57,7 +57,7 @@ def _save_preprocessed_ds(cfg, sub_cfg, dataset):
         dataset.save_to_disk(str(prepared_ds_path))
 
 
-def map_dataset(cfg, data_set, ds_transform_fn, tokenizer, **map_kwargs):
+def map_dataset(cfg, data_set, ds_transform_fn, tokenizer):
     sig = inspect.signature(ds_transform_fn)
     if "tokenizer" in sig.parameters:
         if not tokenizer:
@@ -70,7 +70,6 @@ def map_dataset(cfg, data_set, ds_transform_fn, tokenizer, **map_kwargs):
     data_set = data_set.map(
         ds_transform_fn,
         desc="Mapping RL Dataset",
-        **map_kwargs,
     )
 
     return data_set
@@ -151,45 +150,36 @@ def load_prepare_preference_datasets(cfg):
                 else:
                     ds_transform_fn = load_dpo(_type, _cfg, dataset_idx=i)
 
-                map_kwargs = {}
-                if isinstance(ds_transform_fn, tuple):
-                    ds_transform_fn, map_kwargs = ds_transform_fn
                 split_datasets[i] = map_dataset(
-                    cfg, data_set, ds_transform_fn, tokenizer, **map_kwargs
+                    cfg, data_set, ds_transform_fn, tokenizer
                 )
             elif _cfg.rl == "kto":
                 ds_transform_fn = load_kto(_type, _cfg, dataset_idx=i)
-                map_kwargs = {}
-                if isinstance(ds_transform_fn, tuple):
-                    ds_transform_fn, map_kwargs = ds_transform_fn
                 split_datasets[i] = map_dataset(
-                    cfg, data_set, ds_transform_fn, tokenizer, **map_kwargs
+                    cfg, data_set, ds_transform_fn, tokenizer
                 )
             else:
                 # If no `type` is provided, assume the dataset is already in the expected format with
                 # "prompt", "chosen" and "rejected" already preprocessed
                 split_datasets[i] = data_set
 
-            if not cfg.skip_prepare_dataset:
-                drop_long = partial(
-                    drop_long_rl_seq,
-                    rl=_cfg.rl,
-                    tokenizer=tokenizer,
-                    sequence_len=cfg.sequence_len,
-                )
+            drop_long = partial(
+                drop_long_rl_seq,
+                rl=_cfg.rl,
+                tokenizer=tokenizer,
+                sequence_len=cfg.sequence_len,
+            )
 
-                prior_len = len(split_datasets[i])
-                split_datasets[i] = split_datasets[i].filter(
-                    drop_long,
-                    num_proc=cfg.dataset_processes,
-                    load_from_cache_file=not cfg.is_preprocess,
-                    desc="Dropping Long Sequences",
-                )
-                dropped = prior_len - len(split_datasets[i])
-                if dropped:
-                    LOG.warning(
-                        f"Dropped {dropped} long samples from dataset index {i}"
-                    )
+            prior_len = len(split_datasets[i])
+            split_datasets[i] = split_datasets[i].filter(
+                drop_long,
+                num_proc=cfg.dataset_processes,
+                load_from_cache_file=not cfg.is_preprocess,
+                desc="Dropping Long Sequences",
+            )
+            dropped = prior_len - len(split_datasets[i])
+            if dropped:
+                LOG.warning(f"Dropped {dropped} long samples from dataset index {i}")
 
         combined_datasets = concatenate_datasets(split_datasets)
         combined_datasets = combined_datasets.shuffle(seed=cfg.seed)

src/axolotl/utils/data/sft.py

@@ -9,8 +9,6 @@ from datasets import (
     Dataset,
     DatasetDict,
     IterableDataset,
-    Sequence,
-    Value,
     concatenate_datasets,
     load_dataset,
     load_from_disk,
@@ -18,7 +16,7 @@ from datasets import (
 from transformers import PreTrainedTokenizerBase
 
 from axolotl.common.const import DEFAULT_DATASET_PREPARED_PATH
-from axolotl.datasets import TokenizedPromptDataset, wrap_dataset_for_tokenized_prompt
+from axolotl.datasets import wrap_dataset_for_tokenized_prompt
 from axolotl.prompt_strategies import load
 from axolotl.prompt_strategies.bradley_terry import load as bradley_terry_load
 from axolotl.prompt_tokenizers import (
@@ -46,7 +44,6 @@ from axolotl.utils.data.pretraining import wrap_pretraining_dataset
 from axolotl.utils.data.shared import load_dataset_w_config
 from axolotl.utils.data.utils import (
     deduplicate_and_log_datasets,
-    drop_long_seq_in_dataset,
     md5,
     retry_on_request_exceptions,
 )
@@ -57,7 +54,7 @@ from axolotl.utils.trainer import (
     process_datasets_for_packing,
 )
 
-LOG = logging.getLogger(__name__)
+LOG = logging.getLogger("axolotl")
 
 
 @retry_on_request_exceptions(max_retries=3, delay=5)
@@ -340,11 +337,8 @@ def load_tokenized_prepared_datasets(
             else:
                 LOG.debug("NOT shuffling merged datasets")
 
-        if not cfg.skip_prepare_dataset:
-            dataset = drop_long_seq_in_dataset(dataset, cfg)
-
-            if cfg.sample_packing:
-                dataset, _ = process_datasets_for_packing(cfg, dataset, None)
+        if cfg.sample_packing and not cfg.skip_prepare_dataset:
+            dataset, _ = process_datasets_for_packing(cfg, dataset, None)
 
         if cfg.local_rank == 0 and not cfg.skip_prepare_dataset:
             LOG.info(f"Saving merged prepared dataset to disk... {prepared_ds_path}")
@@ -508,17 +502,6 @@ def get_dataset_wrapper(
             dataset,
             **ds_kwargs,
         )
-    elif config_dataset.type.startswith("stepwise_supervised"):
-        dataset_prompter = UnsupportedPrompter()
-        ds_strategy = load(config_dataset.type, tokenizer, cfg, config_dataset)
-        # we need to explicitly cast boolean labels to int
-        # for compatibility with how trl's PRMTrainer works
-        dataset = dataset.cast_column("labels", Sequence(Value("int64")))
-        dataset_wrapper = TokenizedPromptDataset(
-            ds_strategy,
-            dataset,
-            **ds_kwargs,
-        )
     elif ds_strategy := load(
         config_dataset.type, tokenizer, cfg, config_dataset, processor=processor
     ):

src/axolotl/utils/data/shared.py

@@ -109,13 +109,6 @@ def load_dataset_w_config(
     except (FileNotFoundError, ConnectionError):
         pass
 
-    # gather extra args from the config
-    load_ds_kwargs = {}
-    if config_dataset.split:
-        load_ds_kwargs["split"] = config_dataset.split
-    else:
-        load_ds_kwargs["split"] = None
-
     # prefer local dataset, even if hub exists
     local_path = Path(config_dataset.path)
     if local_path.exists():
@@ -127,7 +120,7 @@ def load_dataset_w_config(
                     name=config_dataset.name,
                     data_files=config_dataset.data_files,
                     streaming=streaming,
-                    **load_ds_kwargs,
+                    split=None,
                 )
             else:
                 try:
@@ -139,7 +132,7 @@ def load_dataset_w_config(
                         config_dataset.path,
                         name=config_dataset.name,
                         streaming=False,
-                        **load_ds_kwargs,
+                        split=None,
                     )
         elif local_path.is_file():
             ds_type = get_ds_type(config_dataset)
@@ -149,13 +142,16 @@ def load_dataset_w_config(
                 name=config_dataset.name,
                 data_files=config_dataset.path,
                 streaming=False,
-                **load_ds_kwargs,
+                split=None,
             )
         else:
             raise ValueError(
                 "unhandled dataset load: local path exists, but is neither a directory or a file"
             )
     elif ds_from_hub:
+        load_ds_kwargs = {}
+        if config_dataset.split:
+            load_ds_kwargs["split"] = config_dataset.split
         ds = load_dataset(
             config_dataset.path,
             name=config_dataset.name,
@@ -179,9 +175,9 @@ def load_dataset_w_config(
                 name=config_dataset.name,
                 data_files=config_dataset.path,
                 streaming=streaming,
+                split=None,
                 storage_options=storage_options,
                 trust_remote_code=config_dataset.trust_remote_code,
-                **load_ds_kwargs,
             )
     elif config_dataset.path.startswith("https://"):
         ds_type = get_ds_type(config_dataset)
@@ -190,9 +186,9 @@ def load_dataset_w_config(
             name=config_dataset.name,
             data_files=config_dataset.path,
             streaming=streaming,
+            split=None,
             storage_options=storage_options,
             trust_remote_code=config_dataset.trust_remote_code,
-            **load_ds_kwargs,
         )
     else:
         if isinstance(config_dataset.data_files, str):
@@ -220,7 +216,7 @@ def load_dataset_w_config(
             name=config_dataset.name,
             data_files=fp,
             streaming=streaming,
-            **load_ds_kwargs,
+            split=None,
         )
     if not ds:
         raise ValueError("unhandled dataset load")

src/axolotl/utils/data/utils.py

@@ -1,5 +1,4 @@
 """data handling helpers"""
-
 import functools
 import hashlib
 import logging
@@ -7,15 +6,10 @@ import time
 from enum import Enum
 
 import huggingface_hub
-import numpy as np
 import requests
-from datasets import Dataset, IterableDataset
+from datasets import Dataset
 
-from axolotl.utils.dict import DictDefault
-from axolotl.utils.samplers.utils import get_dataset_lengths
-from axolotl.utils.trainer import drop_long_seq
-
-LOG = logging.getLogger(__name__)
+LOG = logging.getLogger("axolotl")
 
 
 class RetryStrategy(Enum):
@@ -156,53 +150,3 @@ def deduplicate_and_log_datasets(
         )
 
     return train_dataset, eval_dataset, dataset
-
-
-def drop_long_seq_in_dataset(dataset: Dataset, cfg: DictDefault):
-    if "input_ids" not in dataset.column_names:
-        LOG.warning(
-            "Dataset does not contain 'input_ids' column. Skip drop long seq. This is expected for RewardModeling."
-        )
-        return dataset
-
-    drop_long = functools.partial(
-        drop_long_seq,
-        sequence_len=cfg.sequence_len,
-        min_sequence_len=cfg.min_sample_len,
-    )
-
-    try:
-        min_input_len = np.min(get_dataset_lengths(dataset))
-        LOG.debug(f"min_input_len: {min_input_len}")
-        max_input_len = np.max(get_dataset_lengths(dataset))
-        LOG.debug(f"max_input_len: {max_input_len}")
-    except AttributeError:
-        pass
-
-    try:
-        prior_len = len(dataset)
-    except TypeError:
-        # handle iterable datasets case
-        prior_len = None
-
-    filter_map_kwargs = {}
-    if not isinstance(dataset, IterableDataset):
-        filter_map_kwargs["num_proc"] = cfg.dataset_processes
-        filter_map_kwargs["load_from_cache_file"] = not cfg.is_preprocess
-
-    drop_long_kwargs = {}
-    if filter_map_kwargs:
-        drop_long_kwargs["desc"] = "Dropping Long Sequences"
-
-    dataset = dataset.filter(
-        drop_long,
-        batched=True,
-        **filter_map_kwargs,
-        **drop_long_kwargs,
-    )
-    if prior_len:
-        dropped = prior_len - len(dataset)
-        if dropped:
-            LOG.warning(f"Dropped {dropped} long samples from dataset")
-
-    return dataset

src/axolotl/utils/environment.py

@@ -10,7 +10,7 @@ from accelerate.utils.environment import get_gpu_info
 def check_cuda_p2p_ib_support():
     if not accelerate_check_cuda_p2p_ib_support():
         return False
-    unsupported_devices = {"RTX 6000 Ada", "L40S"}
+    unsupported_devices = {"RTX 6000 Ada"}
     try:
         device_names, device_count = get_gpu_info()
         if 1 < device_count < 8:

src/axolotl/utils/models.py

@@ -138,9 +138,7 @@ def load_model_config(cfg):
     config_kwargs = {}
     if cfg.revision_of_model:
         config_kwargs["revision"] = cfg.revision_of_model
-    if cfg.num_labels:
-        # num_labels is used to initialize classifier models
-        config_kwargs["num_labels"] = cfg.num_labels
+
     try:
         model_config = AutoConfig.from_pretrained(
             model_config_name,
@@ -382,19 +380,23 @@ class ModelLoader:
         plugin_manager = PluginManager.get_instance()
         plugin_manager.pre_model_load(self.cfg)
 
-        if self.cfg.adapter:
-            from axolotl.monkeypatch.transformers_fa_utils import (
-                patch_fa_peft_integration,
-            )
-
-            patch_fa_peft_integration()
-
         if self.cfg.gradient_checkpointing == "unsloth":
             transformers.modeling_utils.checkpoint = hf_grad_checkpoint_unsloth_wrapper
 
         if self.cfg.flash_attention:
             self.patch_attention()
 
+        if self.cfg.model_config_type == "llama":
+            from axolotl.monkeypatch.trainer_grad_accum import (
+                patch_flash_attention_forward,
+                patch_forward_for_ga,
+                patch_training_step_for_ga,
+            )
+
+            patch_flash_attention_forward()
+            patch_forward_for_ga()
+            patch_training_step_for_ga()
+
         if self.cfg.sample_packing and self.cfg.s2_attention:
             raise ValueError(
                 "Received `sample_packing=true` and `s2_attention=true`; however, \
@@ -1053,12 +1055,9 @@ class ModelLoader:
             if self.cfg.resize_token_embeddings_to_32x
             else len(self.tokenizer)
         )
-        if hasattr(self.model, "get_input_embeddings") and (
-            self.model.get_input_embeddings().num_embeddings < embeddings_len
-            or (
-                self.model.get_input_embeddings().num_embeddings > embeddings_len
-                and self.cfg.shrink_embeddings
-            )
+        if (
+            hasattr(self.model, "get_input_embeddings")
+            and self.model.get_input_embeddings().num_embeddings != embeddings_len
         ):
             resize_kwargs = {}
             if self.cfg.mean_resizing_embeddings is not None:

src/axolotl/utils/samplers/multipack.py

@@ -4,6 +4,7 @@ Multipack Batch Sampler
 """
 import logging
 import math
+import os
 from typing import Any, Iterable, List, Union
 
 import numba
@@ -116,7 +117,6 @@ class MultipackBatchSampler(BatchSampler):
         lengths: np.ndarray,
         packing_efficiency_estimate: float = 1.0,
         drop_last: bool = False,
-        num_count_samples: int = 16,
         **kwargs,
     ):
         super().__init__(sampler, batch_size, drop_last)
@@ -133,9 +133,6 @@ class MultipackBatchSampler(BatchSampler):
         self.eff_total_used = 0
         self.eff_total_slots = 0
 
-        # The number of times to calculate the batches to determine the minimum packed dataset length for the local rank
-        self.num_count_samples = num_count_samples
-        # the minimum packed dataset length across all ranks determined by a gather/broadcast
         self.len_across_ranks = None
 
     def set_epoch(self, epoch: int):
@@ -172,9 +169,6 @@ class MultipackBatchSampler(BatchSampler):
 
     def __iter__(self):
         batches = self.generate_batches(set_stats=True)
-        if self.len_across_ranks:
-            # make sure the batches we iterate over is truncated to the same min length across all ranks
-            batches = batches[: self.len_across_ranks]
         return iter(batches)
 
     def num_batches(self):
@@ -201,15 +195,42 @@ class MultipackBatchSampler(BatchSampler):
     def gather_len_batches(self, num):
         def calc_min_len(estimates: list[(int, float)]):
             LOG.info(f"gather_len_batches: {repr(estimates)}")
-            return math.floor(min(estimates))
+            return math.floor(0.998 * min(estimates))
 
         min_len_batches = reduce_and_broadcast(lambda: num, calc_min_len)
         return min_len_batches
 
     def __len__(self):
         if not self.len_across_ranks:
-            len_batches = min(
-                [self.num_batches() for _ in range(self.num_count_samples)]
-            )
+            len_batches = self.num_batches()
             self.len_across_ranks = self.gather_len_batches(len_batches)
         return self.len_across_ranks
+
+    def _len_est(self):
+        efficiency = (
+            self.packing_efficiency_estimate
+            if self.packing_efficiency_estimate
+            else self.gather_efficiency()
+        )
+        world_size = int(os.getenv("WORLD_SIZE", "1"))
+        lengths_sum = np.sum(self.lengths)
+        lengths_sum_per_device = lengths_sum // world_size
+        LOG.info(
+            f"packing_efficiency_estimate: {efficiency} "
+            f"total_num_tokens per device: {lengths_sum_per_device}"
+        )
+
+        # shave off 1% + 1 for dealing with variance in packing from random sampler to sampler
+        return max(
+            0,
+            (
+                world_size
+                * math.floor(
+                    0.99
+                    * lengths_sum_per_device
+                    / efficiency
+                    // (self.batch_max_len * self.batch_size)
+                )
+                - 1
+            ),
+        )

src/axolotl/utils/samplers/utils.py

@@ -13,4 +13,5 @@ def get_dataset_lengths(dataset):
     else:
         input_ids = dataset.data.column("input_ids")
         lengths = np.vectorize(len)(np.array(input_ids, dtype=object))
+        return lengths
     return lengths

src/axolotl/utils/trainer.py

@@ -1,5 +1,4 @@
 """Module containing the Trainer class and related functions"""
-
 import json
 import math
 import os
@@ -101,12 +100,17 @@ def add_position_ids(sample):
     - single example: sample['input_ids'] is a list[int]
     - batched data: sample['input_ids'] is a list[list[int]]
     """
-    # Return sample unchanged if "input_ids" is not present, or is empty
-    if "input_ids" not in sample or not sample["input_ids"]:
+    if "input_ids" not in sample:
+        # If there's no "input_ids", just return sample unchanged
         return sample
 
     input_ids = sample["input_ids"]
 
+    # Detect if it's a single example or a batch
+    if not input_ids:
+        # Edge case: empty
+        return sample
+
     # If first element is an int, it’s a single example
     # If first element is a list, it’s a batch
     if isinstance(input_ids[0], int):
@@ -211,8 +215,6 @@ def drop_long_seq(sample, sequence_len=2048, min_sequence_len=2):
 
     Works for both single-example (list[int]) or batched (list[list[int]]).
     """
-    min_sequence_len = min_sequence_len or 2
-
     input_ids = sample["input_ids"]
 
     # Edge case: if input_ids is empty
@@ -235,6 +237,20 @@ def drop_long_seq(sample, sequence_len=2048, min_sequence_len=2):
 
 
 def process_datasets_for_packing(cfg, train_dataset, eval_dataset):
+    drop_long = partial(
+        drop_long_seq,
+        sequence_len=cfg.sequence_len,
+        min_sequence_len=cfg.min_sample_len or 2,
+    )
+
+    try:
+        min_input_len = np.min(get_dataset_lengths(train_dataset))
+        LOG.debug(f"min_input_len: {min_input_len}", main_process_only=True)
+        max_input_len = np.max(get_dataset_lengths(train_dataset))
+        LOG.debug(f"max_input_len: {max_input_len}", main_process_only=True)
+    except AttributeError:
+        pass
+
     if cfg.model_config_type == "mamba":
         LOG.info("dropping attention_mask column")
         train_dataset = train_dataset.remove_columns("attention_mask")
@@ -248,6 +264,46 @@ def process_datasets_for_packing(cfg, train_dataset, eval_dataset):
         if eval_dataset and "token_type_ids" in eval_dataset.column_names:
             eval_dataset = eval_dataset.remove_columns("token_type_ids")
 
+    filter_map_kwargs = {}
+    if not isinstance(train_dataset, IterableDataset):
+        filter_map_kwargs["num_proc"] = cfg.dataset_processes
+        filter_map_kwargs["load_from_cache_file"] = not cfg.is_preprocess
+
+    try:
+        prior_len = len(train_dataset)
+    except TypeError:
+        # handle iterable datasets case
+        prior_len = None
+    drop_long_kwargs = {}
+    if filter_map_kwargs:
+        drop_long_kwargs["desc"] = "Dropping Long Sequences"
+    train_dataset = train_dataset.filter(
+        drop_long,
+        batched=True,
+        **filter_map_kwargs,
+        **drop_long_kwargs,
+    )
+    if prior_len:
+        dropped = prior_len - len(train_dataset)
+        if dropped:
+            LOG.warning(f"Dropped {dropped} long samples from train dataset")
+
+    if eval_dataset:
+        try:
+            prior_len = len(eval_dataset)
+        except TypeError:
+            # handle iterable datasets case
+            prior_len = None
+        eval_dataset = eval_dataset.filter(
+            drop_long,
+            **filter_map_kwargs,
+            **drop_long_kwargs,
+        )
+        if prior_len:
+            dropped = prior_len - len(eval_dataset)
+            if dropped:
+                LOG.warning(f"Dropped {dropped} long samples from eval dataset")
+
     def drop_no_trainable_tokens(sample):
         """
         Drop samples if all labels are -100 (i.e., zero trainable tokens).
@@ -262,11 +318,14 @@ def process_datasets_for_packing(cfg, train_dataset, eval_dataset):
         # If it's a list, we assume we're dealing with a batch
         if isinstance(labels[0], int):
             # Single example: return a single bool
-            return np.any(labels != -100)
+            return np.sum(np.array(labels) != -100) > 0
 
         # Batched: 'labels' is a list of lists
         # Return a list of booleans, one per sub-list
-        results = [np.any(row_labels != -100) for row_labels in labels]
+        results = []
+        for row_labels in labels:
+            # Each row_labels is a list[int]
+            results.append(np.sum(np.array(row_labels) != -100) > 0)
         return results
 
     try:
@@ -274,11 +333,6 @@ def process_datasets_for_packing(cfg, train_dataset, eval_dataset):
     except TypeError:
         # handle iterable datasets case
         prior_len = None
-    filter_map_kwargs = {}
-    if not isinstance(train_dataset, IterableDataset):
-        filter_map_kwargs["num_proc"] = cfg.dataset_processes
-        filter_map_kwargs["load_from_cache_file"] = not cfg.is_preprocess
-
     drop_long_kwargs = {}
     if filter_map_kwargs:
         drop_long_kwargs["desc"] = "Drop Samples with Zero Trainable Tokens"
@@ -368,22 +422,19 @@ def process_datasets_for_packing(cfg, train_dataset, eval_dataset):
 
 
 def process_pretraining_datasets_for_packing(
-    train_dataset, sequence_len, skip_position_ids=True, drop_attention_mask=False
+    train_dataset, sequence_len, skip_position_ids=True
 ):
     drop_long = partial(drop_long_seq, sequence_len=sequence_len)
 
     train_dataset = train_dataset.filter(
         drop_long,
         desc="Dropping Long Sequences",
-        load_from_cache_file=False,
     )
-    if not skip_position_ids:
+    if skip_position_ids:
         train_dataset = train_dataset.map(
             add_position_ids,
             desc="Add position_id column (Pretraining Sample Packing)",
         )
-    if drop_attention_mask:
-        train_dataset = train_dataset.remove_columns("attention_mask")
 
     return train_dataset
 
@@ -432,7 +483,7 @@ def calculate_total_num_steps(cfg, train_dataset, update=True):
         if cfg.sample_packing_eff_est:
             total_num_steps = (
                 # match count to len est in dataloader
-                int(
+                (
                     math.floor(
                         0.99
                         * cfg.total_num_tokens
@@ -576,7 +627,7 @@ def prepare_opinionated_env(cfg):
 def setup_trainer(
     cfg, train_dataset, eval_dataset, model, tokenizer, processor, total_num_steps
 ):
-    if cfg.rl in ("dpo", "grpo", "ipo", "orpo", "kto", "simpo"):
+    if cfg.rl in ("dpo", "ipo", "orpo", "kto", "simpo"):
         trainer_builder = HFRLTrainerBuilder(cfg, model[0], tokenizer, processor)
         trainer_builder.model_ref = model[1]
         trainer_builder.peft_config = model[2]