multipack support for phi moe

2024-11-15 22:56:25 -05:00
99 changed files with 767 additions and 3878 deletions
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -1,16 +1,6 @@
 name: ci-cd-base

 on:
-  push:
-    branches:
-      - "main"
-    paths:
-      - 'Dockerfile-base'
-      - '.github/workflows/base.yml'
-  pull_request:
-    paths:
-      - 'Dockerfile-base'
-      - '.github/workflows/base.yml'
  workflow_dispatch:

 jobs:
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -10,7 +10,7 @@ on:

 jobs:
  build-axolotl:
-    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]') && github.repository_owner == 'axolotl-ai-cloud' }}
+    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]]') && github.repository_owner == 'axolotl-ai-cloud' }}
    strategy:
      fail-fast: false
      matrix:
@@ -49,7 +49,7 @@ jobs:
            axolotlai/axolotl
          tags: |
            type=ref,event=branch
-            type=pep440,pattern={{version}}
+            type=semver,pattern={{version}}
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to Docker Hub
@@ -77,7 +77,7 @@ jobs:

  build-axolotl-cloud:
    needs: build-axolotl
-    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]') && github.repository_owner == 'axolotl-ai-cloud' }}
+    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]]') && github.repository_owner == 'axolotl-ai-cloud' }}
    # this job needs to be run on self-hosted GPU runners...
    strategy:
      matrix:
@@ -116,7 +116,7 @@ jobs:
            axolotlai/axolotl-cloud
          tags: |
            type=ref,event=branch
-            type=pep440,pattern={{version}}
+            type=semver,pattern={{version}}
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
@@ -140,7 +140,7 @@ jobs:

  build-axolotl-cloud-no-tmux:
    needs: build-axolotl
-    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]') && github.repository_owner == 'axolotl-ai-cloud' }}
+    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]]') && github.repository_owner == 'axolotl-ai-cloud' }}
    # this job needs to be run on self-hosted GPU runners...
    strategy:
      matrix:
@@ -163,7 +163,7 @@ jobs:
            axolotlai/axolotl-cloud-term
          tags: |
            type=ref,event=branch
-            type=pep440,pattern={{version}}
+            type=semver,pattern={{version}}
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
--- a/.github/workflows/multi-gpu-e2e.yml
+++ b/.github/workflows/multi-gpu-e2e.yml
@@ -15,7 +15,7 @@ concurrency:

 jobs:
  test-axolotl-multigpu:
-    if: ${{ ! contains(github.event.commits[0].message, '[skip e2e]') && github.repository_owner == 'axolotl-ai-cloud' }}
+    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]]') && github.repository_owner == 'axolotl-ai-cloud' }}
    strategy:
      fail-fast: false
      matrix:
--- a/.github/workflows/nightlies.yml
+++ b/.github/workflows/nightlies.yml
@@ -7,7 +7,7 @@ on:

 jobs:
  build-axolotl:
-    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]') && github.repository_owner == 'axolotl-ai-cloud' }}
+    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]]') && github.repository_owner == 'axolotl-ai-cloud' }}
    strategy:
      fail-fast: false
      matrix:
@@ -71,7 +71,7 @@ jobs:

  build-axolotl-cloud:
    needs: build-axolotl
-    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]') && github.repository_owner == 'axolotl-ai-cloud' }}
+    if: ${{ ! contains(github.event.commits[0].message, '[skip docker]]') && github.repository_owner == 'axolotl-ai-cloud' }}
    # this job needs to be run on self-hosted GPU runners...
    strategy:
      matrix:
--- a/.github/workflows/pypi.yml
+++ b/.github/workflows/pypi.yml
@@ -10,13 +10,20 @@ jobs:
  setup_release:
    name: Create Release
    runs-on: ubuntu-latest
-    permissions:
-      contents: write
    steps:
-      - name: Create release
+      - name: Get the tag version
+        id: extract_branch
+        run: echo ::set-output name=branch::${GITHUB_REF#refs/tags/}
+        shell: bash
+
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: gh release create "$GITHUB_REF_NAME" # GITHUB_REF_NAME is the tag name in `on.push.tags` workflows
+        with:
+          tag_name: ${{ steps.extract_branch.outputs.branch }}
+          release_name: ${{ steps.extract_branch.outputs.branch }}
  pypi-publish:
    name: Upload release to PyPI
    runs-on: ubuntu-latest
@@ -49,9 +56,9 @@ jobs:
        run: |
          sed -i -E 's/version="([0-9.]+)",/version="${{ steps.tag.outputs.TAG_NAME }}",/g' setup.py

-      - name: Build a source dist
+      - name: Build a binary wheel
        run: |
-          python setup.py sdist
+          python setup.py sdist bdist_wheel

      - name: Publish package distributions to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
--- a/.github/workflows/tests-nightly.yml
+++ b/.github/workflows/tests-nightly.yml
@@ -55,7 +55,6 @@ jobs:
          pip3 install --upgrade pip
          pip3 install --upgrade packaging
          pip3 install -U -e .
-          python scripts/cutcrossentropy_install.py | sh
          pip3 install -r requirements-dev.txt -r requirements-tests.txt

      - name: Run tests
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -8,15 +8,11 @@ on:
      - '**.py'
      - 'requirements.txt'
      - '.github/workflows/*.yml'
-      - 'requirements-tests.txt'
-      - 'cicd/cicd.sh'
  pull_request:
      paths:
       - '**.py'
       - 'requirements.txt'
       - '.github/workflows/*.yml'
-       - 'requirements-tests.txt'
-       - 'cicd/cicd.sh'
  workflow_dispatch:

 # Cancel jobs on the same ref if a new one is triggered
@@ -71,68 +67,22 @@ jobs:
        run: |
          pip3 show torch
          pip3 install -U -e .
-          python scripts/unsloth_install.py | sh
-          python scripts/cutcrossentropy_install.py | sh
          pip3 install -r requirements-dev.txt -r requirements-tests.txt

      - name: Run tests
        run: |
-          pytest -n8 --ignore=tests/e2e/ tests/
-
-      - name: cleanup pip cache
-        run: |
-          find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \;
-
-  pytest-sdist:
-    name: PyTest from Source Dist
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python_version: ["3.11"]
-        pytorch_version: ["2.4.1", "2.5.1"]
-    timeout-minutes: 20
-
-    steps:
-      - name: Check out repository code
-        uses: actions/checkout@v4
-
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python_version }}
-          cache: 'pip' # caching pip dependencies
-
-      - name: upgrade pip
-        run: |
-          pip3 install --upgrade pip
-          pip3 install --upgrade packaging setuptools wheel
-
-      - name: Install PyTorch
-        run: |
-          pip3 install torch==${{ matrix.pytorch_version }}
-
-      - name: Install dependencies
-        run: |
-          pip3 show torch
-          python3 setup.py sdist
-          pip3 install dist/axolotl*.tar.gz
-          pip3 install -r requirements-dev.txt -r requirements-tests.txt
-
-      - name: Run tests
-        run: |
-          pytest -n8 --ignore=tests/e2e/ tests/
+          pytest --ignore=tests/e2e/ tests/

      - name: cleanup pip cache
        run: |
          find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \;

  docker-e2e-tests-1st:
-    if: ${{ ! contains(github.event.commits[0].message, '[skip e2e]') && github.repository_owner == 'axolotl-ai-cloud' }}
+    if: github.repository_owner == 'axolotl-ai-cloud'
    # this job needs to be run on self-hosted GPU runners...
    runs-on: [self-hosted, modal]
    timeout-minutes: 90
-    needs: [pre-commit, pytest, pytest-sdist]
+    needs: [pre-commit, pytest]

    strategy:
      fail-fast: false
--- a/.gitignore
+++ b/.gitignore
@@ -182,6 +182,3 @@ submit.sh

 typings/
 out/
-
-# vim
-*.swp
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,4 +0,0 @@
-include requirements.txt
-include README.md
-include LICENSE
-recursive-include axolotl *.py
--- a/README.md
+++ b/README.md
@@ -1,21 +1,8 @@
-<p align="center">
-    <picture>
-        <source media="(prefers-color-scheme: dark)" srcset="image/axolotl_logo_digital_white.svg">
-        <source media="(prefers-color-scheme: light)" srcset="image/axolotl_logo_digital_black.svg">
-        <img alt="Axolotl" src="image/axolotl_logo_digital_black.svg" width="400" height="104" style="max-width: 100%;">
-    </picture>
-</p>
+# Axolotl

-<p align="center">
-    <img src="https://img.shields.io/github/license/axolotl-ai-cloud/axolotl.svg?color=blue" alt="GitHub License">
-    <img src="https://github.com/axolotl-ai-cloud/axolotl/actions/workflows/tests.yml/badge.svg" alt="tests">
-    <a href="https://github.com/axolotl-ai-cloud/axolotl/releases"><img src="https://img.shields.io/github/release/axolotl-ai-cloud/axolotl.svg" alt="Releases"></a>
-    <img src="https://img.shields.io/github/stars/axolotl-ai-cloud/axolotl" alt="GitHub Repo stars">
-</p>
-<p align="center">
-    <img src="https://github.com/axolotl-ai-cloud/axolotl/actions/workflows/tests-nightly.yml/badge.svg" alt="tests-nightly">
-    <img src="https://github.com/axolotl-ai-cloud/axolotl/actions/workflows/multi-gpu-e2e.yml/badge.svg" alt="multigpu-semi-weekly tests">
-</p>
+![tests](https://github.com/axolotl-ai-cloud/axolotl/actions/workflows/tests.yml/badge.svg)
+![tests-nightly](https://github.com/axolotl-ai-cloud/axolotl/actions/workflows/tests-nightly.yml/badge.svg)
+![multigpu-semi-weekly tests](https://github.com/axolotl-ai-cloud/axolotl/actions/workflows/multi-gpu-e2e.yml/badge.svg)

 Axolotl is a tool designed to streamline the fine-tuning of various AI models, offering support for multiple configurations and architectures.

@@ -88,7 +75,7 @@ Features:
 <td>

 <div align="center">
-  <img src="image/axolotl_symbol_digital_white.svg" alt="axolotl" width="160">
+  <img src="image/axolotl.png" alt="axolotl" width="160">
  <div>
    <p>
      <b>Axolotl provides a unified repository for fine-tuning <br />a variety of AI models with ease</b>
@@ -147,7 +134,7 @@ pip3 install -e '.[flash-attn,deepspeed]'
 ### Usage
 ```bash
 # preprocess datasets - optional but recommended
-CUDA_VISIBLE_DEVICES="0" python -m axolotl.cli.preprocess examples/openllama-3b/lora.yml
+CUDA_VISIBLE_DEVICES="" python -m axolotl.cli.preprocess examples/openllama-3b/lora.yml

 # finetune lora
 accelerate launch -m axolotl.cli.train examples/openllama-3b/lora.yml
--- a/cicd/Dockerfile.jinja
+++ b/cicd/Dockerfile.jinja
@@ -37,9 +37,6 @@ RUN if [ "$AXOLOTL_EXTRAS" != "" ] ; then \
        pip install -e .[deepspeed,flash-attn,optimizers] $AXOLOTL_ARGS; \
    fi

-RUN python scripts/unsloth_install.py | sh
-RUN python scripts/cutcrossentropy_install.py | sh
-
 # So we can test the Docker image
 RUN pip install -r requirements-dev.txt -r requirements-tests.txt

--- a/cicd/cicd.sh
+++ b/cicd/cicd.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 set -e

-pytest -v --durations=10 -n8 --ignore=tests/e2e/ /workspace/axolotl/tests/
-pytest -v --durations=10 -n1 --dist loadfile -v /workspace/axolotl/tests/e2e/patched/ /workspace/axolotl/tests/e2e/integrations/
-pytest -v --durations=10 --ignore=tests/e2e/patched/ --ignore=tests/e2e/multigpu/ --ignore=tests/e2e/integrations/ /workspace/axolotl/tests/e2e/
+pytest -n4 --ignore=tests/e2e/ /workspace/axolotl/tests/
+pytest -n1 --dist loadfile -v /workspace/axolotl/tests/e2e/patched/ /workspace/axolotl/tests/e2e/integrations/
+pytest --ignore=tests/e2e/patched/ --ignore=tests/e2e/multigpu/ --ignore=tests/e2e/integrations/ /workspace/axolotl/tests/e2e/
--- a/cicd/tests.py
+++ b/cicd/tests.py
@@ -40,7 +40,6 @@ with open(pathlib.Path(temp_dir) / "Dockerfile", "w", encoding="utf-8") as f:
 cicd_image = (
    Image.from_dockerfile(
        pathlib.Path(temp_dir) / "Dockerfile",
-        context_mount=None,
        force_build=True,
        gpu="A10G",
    )
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -26,9 +26,6 @@ RUN if [ "$AXOLOTL_EXTRAS" != "" ] ; then \
        pip install -e .[deepspeed,flash-attn,optimizers] $AXOLOTL_ARGS; \
    fi

-RUN python scripts/unsloth_install.py | sh
-RUN python scripts/cutcrossentropy_install.py | sh
-
 # So we can test the Docker image
 RUN pip install pytest

--- a/docker/Dockerfile-base
+++ b/docker/Dockerfile-base
@@ -29,9 +29,7 @@ ENV PATH="/root/miniconda3/envs/py${PYTHON_VERSION}/bin:${PATH}"
 WORKDIR /workspace

 RUN python3 -m pip install --upgrade pip && pip3 install packaging && \
-    python3 -m pip install --no-cache-dir -U torch==${PYTORCH_VERSION}+cu${CUDA} --extra-index-url https://download.pytorch.org/whl/cu$CUDA && \
-    python3 -m pip install --no-cache-dir "causal_conv1d @ git+https://github.com/Dao-AILab/causal-conv1d.git@main" && \
-    python3 -m pip install --no-cache-dir "mamba_ssm @ git+https://github.com/state-spaces/mamba.git@main"
+    python3 -m pip install --no-cache-dir -U torch==${PYTORCH_VERSION}+cu${CUDA} --extra-index-url https://download.pytorch.org/whl/cu$CUDA

 RUN git lfs install --skip-repo && \
    pip3 install awscli && \
--- a/docs/config.qmd
+++ b/docs/config.qmd
@@ -162,9 +162,6 @@ datasets:
 # The same applies to the `test_datasets` option and the `pretraining_dataset` option. Default is true.
 shuffle_merged_datasets: true

-Deduplicates datasets and test_datasets with identical entries.
-dataset_exact_deduplication: true
-
 # A list of one or more datasets to eval the model with.
 # You can use either test_datasets, or val_set_size, but not both.
 test_datasets:
@@ -409,7 +406,7 @@ lr_div_factor: # Learning rate div factor
 # - adamw_torch_fused
 # - adamw_torch_xla
 # - adamw_apex_fused
-# - adopt_adamw (an EXPERIMENTAL optimizer, only for torch version >= 2.5.1)
+# - adopt_adamw (only for torch version >= 2.5.1)
 # - adafactor
 # - adamw_anyprecision
 # - sgd
--- a/docs/unsloth.qmd
+++ b/docs/unsloth.qmd
@@ -11,10 +11,12 @@ standard industry baselines.

 ### Installation

-The following will install the correct unsloth and extras from source.
+The following will install unsloth from source and downgrade xformers as unsloth is incompatible with the most up
+to date libraries.

 ```bash
-python scripts/unsloth_install.py | sh
+pip install --no-deps "unsloth @ git+https://github.com/unslothai/unsloth.git"
+pip install --no-deps --force-reinstall xformers==0.0.26.post1
 ```

 ### Using unsloth w Axolotl
--- a/examples/colab-notebooks/colab-axolotl-example.ipynb
+++ b/examples/colab-notebooks/colab-axolotl-example.ipynb
@@ -2,15 +2,19 @@
 "cells": [
  {
   "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "id": "AKjdG7tbTb-n"
+   },
   "source": [
-    "## Setting up"
+    "# Example notebook for running Axolotl on google colab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "id": "RcbNpOgWRcii"
+   },
   "outputs": [],
   "source": [
    "import torch\n",
@@ -18,76 +22,82 @@
    "assert (torch.cuda.is_available()==True)"
   ]
  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "id": "h3nLav8oTRA5"
+   },
+   "source": [
+    "## Install Axolotl and dependencies"
+   ]
+  },
  {
   "cell_type": "code",
   "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "3c3yGAwnOIdi",
+    "outputId": "e3777b5a-40ef-424f-e181-62dfecd1dd01"
+   },
   "outputs": [],
   "source": [
-    "!pip install axolotl[deepspeed]"
+    "!pip install -e git+https://github.com/axolotl-ai-cloud/axolotl#egg=axolotl\n",
+    "!pip install flash-attn==\"2.7.0.post2\"\n",
+    "!pip install deepspeed==\"0.13.1\"!pip install mlflow==\"2.13.0\""
   ]
  },
  {
   "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "id": "BW2MFr7HTjub"
+   },
   "source": [
-    "## Hugging Face login (optional)"
+    "## Create an yaml config file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from huggingface_hub import notebook_login\n",
-    "notebook_login()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Example configuration"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "id": "9pkF2dSoQEUN"
+   },
   "outputs": [],
   "source": [
    "import yaml\n",
    "\n",
+    "# Your YAML string\n",
    "yaml_string = \"\"\"\n",
-    "base_model: NousResearch/Meta-Llama-3.1-8B\n",
+    "base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T\n",
+    "model_type: LlamaForCausalLM\n",
+    "tokenizer_type: LlamaTokenizer\n",
    "\n",
    "load_in_8bit: false\n",
    "load_in_4bit: true\n",
    "strict: false\n",
    "\n",
    "datasets:\n",
-    "  - path: tatsu-lab/alpaca\n",
+    "  - path: mhenrichsen/alpaca_2k_test\n",
    "    type: alpaca\n",
-    "dataset_prepared_path: last_run_prepared\n",
+    "dataset_prepared_path:\n",
    "val_set_size: 0.05\n",
-    "output_dir: ./outputs/lora-out\n",
-    "\n",
-    "sequence_len: 2048\n",
-    "sample_packing: true\n",
-    "eval_sample_packing: true\n",
-    "pad_to_sequence_len: true\n",
+    "output_dir: ./outputs/qlora-out\n",
    "\n",
    "adapter: qlora\n",
    "lora_model_dir:\n",
+    "\n",
+    "sequence_len: 4096\n",
+    "sample_packing: true\n",
+    "eval_sample_packing: false\n",
+    "pad_to_sequence_len: true\n",
+    "\n",
    "lora_r: 32\n",
    "lora_alpha: 16\n",
    "lora_dropout: 0.05\n",
+    "lora_target_modules:\n",
    "lora_target_linear: true\n",
    "lora_fan_in_fan_out:\n",
-    "lora_modules_to_save:\n",
-    "  - embed_tokens\n",
-    "  - lm_head\n",
    "\n",
    "wandb_project:\n",
    "wandb_entity:\n",
@@ -95,12 +105,12 @@
    "wandb_name:\n",
    "wandb_log_model:\n",
    "\n",
-    "gradient_accumulation_steps: 2\n",
-    "micro_batch_size: 1\n",
-    "num_epochs: 1\n",
-    "optimizer: paged_adamw_8bit\n",
+    "gradient_accumulation_steps: 4\n",
+    "micro_batch_size: 2\n",
+    "num_epochs: 4\n",
+    "optimizer: paged_adamw_32bit\n",
    "lr_scheduler: cosine\n",
-    "learning_rate: 2e-5\n",
+    "learning_rate: 0.0002\n",
    "\n",
    "train_on_inputs: false\n",
    "group_by_length: false\n",
@@ -111,15 +121,13 @@
    "gradient_checkpointing: true\n",
    "early_stopping_patience:\n",
    "resume_from_checkpoint:\n",
+    "local_rank:\n",
    "logging_steps: 1\n",
    "xformers_attention:\n",
-    "flash_attention: false\n",
-    "sdp_attention: true\n",
+    "flash_attention: true\n",
    "\n",
-    "warmup_steps: 1\n",
-    "max_steps: 25\n",
-    "evals_per_epoch: 1\n",
-    "eval_table_size:\n",
+    "warmup_steps: 10\n",
+    "evals_per_epoch: 4\n",
    "saves_per_epoch: 1\n",
    "debug:\n",
    "deepspeed:\n",
@@ -127,9 +135,8 @@
    "fsdp:\n",
    "fsdp_config:\n",
    "special_tokens:\n",
-    "  pad_token: <|end_of_text|>\n",
-    "\"\"\"\n",
    "\n",
+    "\"\"\"\n",
    "\n",
    "# Convert the YAML string to a Python dictionary\n",
    "yaml_dict = yaml.safe_load(yaml_string)\n",
@@ -139,124 +146,31 @@
    "\n",
    "# Write the YAML file\n",
    "with open(file_path, 'w') as file:\n",
-    "    yaml.dump(yaml_dict, file)"
+    "    yaml.dump(yaml_dict, file)\n"
   ]
  },
  {
   "cell_type": "markdown",
-   "metadata": {},
+   "metadata": {
+    "id": "bidoj8YLTusD"
+   },
   "source": [
-    "Above we have a configuration file with base LLM model and datasets specified, among many other things. Axolotl can automatically detect whether the specified datasets are on HuggingFace repo or local machine.\n",
-    "\n",
-    "The Axolotl configuration options encompass model and dataset selection, data pre-processing, and training. Let's go through them line by line:\n",
-    "\n",
-    "*   \"base model\": String value, specifies the underlying pre-trained LLM that will be used for finetuning\n",
-    "\n",
-    "Next we have options for model weights quantization. Quantization allows for reduction in occupied memory on GPUs.\n",
-    "\n",
-    "*   \"load_in_8bit\": Boolean value, whether to quantize the model weights into 8-bit integer.\n",
-    "\n",
-    "*   \"load_in_4bit\": Boolean value, whether to quantize the model weights into 4-bit integer.\n",
-    "\n",
-    "*   \"strict\": Boolean value. If false, it allows for overriding established configuration options in the yaml file when executing in command-line interface.\n",
-    "\n",
-    "*   \"datasets\": a list of dicts that contain path and type of data sets as well as other optional configurations where datasets are concerned. Supports multiple datasets.\n",
-    "\n",
-    "*   \"val_set_size\": Either a float value less than one or an integer less than the total size of dataset. Sets the size of validation set from the whole dataset. If float, sets the proportion of the dataset assigned for validation. If integer, sets the direct size of validation set.\n",
-    "\n",
-    "*   \"output_dir\": String value. Path of trained model.\n",
-    "\n",
-    "For data preprocessing:\n",
-    "\n",
-    "*   \"sequence_len\": Integer. Specifies the maximum sequence length of the input. Typically 2048 or less.\n",
-    "\n",
-    "*   \"pad_to_sequence_len\": Boolean. Padding input to maximum sequence length.\n",
-    "\n",
-    "*   \"sample_packing\": Boolean. Specifies whether to use multi-packing with block diagonal attention.\n",
-    "\n",
-    "*   \"special_tokens\": Python dict, optional. Allows users to specify the additional special tokens to be ignored by the tokenizer.\n",
-    "\n",
-    "For LoRA configuration and its hyperparamters:\n",
-    "\n",
-    "*   \"adapter\": String. Either \"lora\" or \"qlora\", depending on user's choice.\n",
-    "\n",
-    "*   \"lora_model_dir\": String, Optional. Path to directory that contains LoRA model, if there is already a trained LoRA model the user would like to use.\n",
-    "\n",
-    "*   \"lora_r\": Integer. Refers to the rank of LoRA decomposition matrices. Higher value will reduce LoRA efficiency. Recommended to be set to 8.\n",
-    "\n",
-    "*   \"lora_alpha\": Integer. Scale the weight matrices by $\\frac{\\text{lora_alpha}}{\\text{lora_r}}$Recommended to be fixed at 16.\n",
-    "\n",
-    "*   \"lora_dropout\": Float that is 1 or less. The dropout probability of a lora layer.\n",
-    "\n",
-    "*   \"lora_target_linear\": Boolean. If true, lora will target all linear modules in the transformers architecture.\n",
-    "\n",
-    "*   \"lora_modules_to_save\": If you added new tokens to the tokenizer, you may need to save some LoRA modules because they need to know the new tokens.\n",
-    "\n",
-    "See [LoRA](https://arxiv.org/abs/2106.09685) for detailed explanation of LoRA implementation.\n",
-    "\n",
-    "For the training configurations:\n",
-    "\n",
-    "*   \"gradient_accumulation_steps\": Integer. The number of steps over which to accumulate gradient for batch training. E.g. if 2, backprop is performed every two steps.\n",
-    "\n",
-    "*   \"micro_batch_size\": Integer. Batch size per gpu / gradient_accumulation_steps\n",
-    "\n",
-    "*   \"num_epochs\": Integer. Number of epochs. One epoch is when training has looped over every batch in the whole data set once.\n",
-    "\n",
-    "*   \"optimizer\": The optimizer to use for the training.\n",
-    "\n",
-    "*   \"learning_rate\": The learning rate.\n",
-    "\n",
-    "*   \"lr_scheduler\": The learning rate scheduler to use for adjusting learning rate during training.\n",
-    "\n",
-    "*   \"train_on_inputs\": Boolean. Whether to ignore or include the user's prompt from the training labels.\n",
-    "\n",
-    "*   \"group_by_length\": Boolean. Whether to group similarly sized data to minimize padding.\n",
-    "\n",
-    "*   \"bf16\": Either \"auto\", \"true\", or \"false\". Whether to use CUDA bf16 floating point format. If set to \"auto\", will automatically apply bf16 should the gpu supports it.\n",
-    "\n",
-    "*   \"fp16\": Optional. Specifies whether to use CUDA fp16. Automatically set to true if \"bf16\" is set to true. Otherwise false.\n",
-    "\n",
-    "*   \"tf32\": Boolean. Whether to use CUDA tf32. Will override bf16.\n",
-    "\n",
-    "*   \"gradient_checkpointing\": Boolean. Whether to use gradient checkpointing https://huggingface.co/docs/transformers/v4.18.0/en/performance#gradient-checkpointing\n",
-    "\n",
-    "*   \"gradient_checkpointing_kwargs\": Python Dict. Fed into the trainer.\n",
-    "\n",
-    "*   \"logging_steps\": Integer. Log training information over every specified number of steps.\n",
-    "\n",
-    "*   \"flash_attention\": Boolean. Whether to use the [flash attention](https://github.com/Dao-AILab/flash-attention) mechanism.\n",
-    "\n",
-    "*   \"sdp_attention\": Boolean. Whether to use the Scaled Dot Product attention mechanism (the attention mechanism in the [original implementation](https://arxiv.org/abs/1706.03762) of transformers.)\n",
-    "\n",
-    "*   \"warmup_steps\": Integer. The number of pre-training steps where a very low learning rate is used.\n",
-    "\n",
-    "*   \"evals_per_epoch\": Integer. Number of evaluations to be performed within one training epoch.\n",
-    "\n",
-    "*   \"saves_per_epoch\": Integer. Number of times the model is saved in one training epoch.\n",
-    "\n",
-    "*   \"weight_decay\": Positive Float. Sets the \"strength\" of weight decay (i.e. setting the coefficient of L2 regularization)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The above is but a snippet aiming to get users familiarized with the types of streamlined configuration options axolotl provides. For a full list of configuration options, see [here](https://axolotl-ai-cloud.github.io/axolotl/docs/config.html)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Train the model"
+    "## Launch the training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "ydTI2Jk2RStU",
+    "outputId": "d6d0df17-4b53-439c-c802-22c0456d301b"
+   },
   "outputs": [],
   "source": [
+    "# By using the ! the comand will be executed as a bash command\n",
    "!accelerate launch -m axolotl.cli.train /content/test_axolotl.yaml"
   ]
  },
@@ -264,7 +178,7 @@
   "cell_type": "markdown",
   "metadata": {},
   "source": [
-    "Predict with trained model"
+    "## Play with inference"
   ]
  },
  {
@@ -273,85 +187,36 @@
   "metadata": {},
   "outputs": [],
   "source": [
+    "# By using the ! the comand will be executed as a bash command\n",
    "!accelerate launch -m axolotl.cli.inference /content/test_axolotl.yaml \\\n",
-    "    --lora_model_dir=\"./outputs/lora-out\" --gradio"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Deeper Dive"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "It is also helpful to gain some familiarity over some of the core inner workings of axolotl"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Configuration Normalization"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Axolotl uses a custom Dict class, called ```DictDefault```\n",
-    "to store configurations specified in the yaml configuration file (into a Python variable named ```cfg```). The definition for this custom Dict can be found in the [utils/dict.py](https://github.com/axolotl-ai-cloud/axolotl/blob/main/src/axolotl/utils/dict.py)\n",
-    "\n",
-    "```DictDefault``` is amended such that calling a missing key from it will result in a ```None``` return type. This is important because if some configuration options aren't specified by the user, the ```None``` type allows Axolotl to perform boolean operations to determine the default settings for missing configurations. For more examples on how this is done, check out [utils/config/__init__.py](https://github.com/axolotl-ai-cloud/axolotl/blob/main/src/axolotl/utils/config/__init__.py)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Loading Models, Tokenizers, and Trainer"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "If we inspect [cli.train.py](https://github.com/axolotl-ai-cloud/axolotl/blob/main/src/axolotl/cli/train.py), we will find that most of the heavy lifting were done by the function ```train()``` which is itself imported from [src/axolotl/train.py](https://github.com/axolotl-ai-cloud/axolotl/blob/main/src/axolotl/train.py).\n",
-    "\n",
-    "```train()``` takes care of loading the appropriate tokenizer and pre-trained model through ```load_model()``` and ```load_tokenizer()``` from [src/axolotl/utils/models.py](https://github.com/axolotl-ai-cloud/axolotl/blob/main/src/axolotl/utils/models.py) respectively.\n",
-    "\n",
-    "```load_tokenizer()``` loads in the appropriate tokenizer given the desired model, as well as chat templates.\n",
-    "\n",
-    "```ModelLoader``` class follows after tokenizer has been selected. It will automatically discern the base model type, load in the desired model, as well as applying model-appropriate attention mechanism modifications (e.g. flash attention). Depending on which base model the user chooses in the configuration, ```ModelLoader``` will utilize the corresponding \"attention hijacking\" script. For example, if the user specified the base model to be ```NousResearch/Meta-Llama-3.1-8B```, which is of llama type, and set ```flash_attn``` to ```True```, ```ModelLoader``` will load in [llama_attn_hijack_flash.py](https://github.com/axolotl-ai-cloud/axolotl/blob/main/src/axolotl/monkeypatch/llama_attn_hijack_flash.py). For a list of supported attention hijacking, please refer to the directory [/src/axolotl/monkeypatch/](https://github.com/axolotl-ai-cloud/axolotl/tree/main/src/axolotl/monkeypatch)\n",
-    "\n",
-    "Another important operation encompassed in ```train()``` is setting up the training that takes into account of user-specified traning configurations (e.g. num_epochs, optimizer) through the use of ```setup_trainer()``` from [/src/axolotl/utils/trainer.py](https://github.com/axolotl-ai-cloud/axolotl/blob/main/src/axolotl/utils/trainer.py), which in turn relies on modules from [/src/axolotl/core/trainer_builder.py](https://github.com/axolotl-ai-cloud/axolotl/blob/main/src/axolotl/core/trainer_builder.py).\n",
-    "```trainer_builder.py``` provides a list of trainer object options bespoke for the task type (Causal or Reinforcement learning ('dpo', 'ipo', 'kto') )"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Monkey patch\n",
-    "\n",
-    "The [Monkey patch directory](https://github.com/axolotl-ai-cloud/axolotl/tree/main/src/axolotl/monkeypatch) is where model architecture/optimization patching scripts are stored (these are modifications that are not implemented in the official releases, hence the name monkey patch). It includes attention jacking, ReLoRA, and unsloth optimization."
+    "    --qlora_model_dir=\"./qlora-out\" --gradio"
   ]
  }
 ],
 "metadata": {
+  "accelerator": "GPU",
+  "colab": {
+   "gpuType": "T4",
+   "provenance": []
+  },
  "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
   "name": "python",
-   "version": "3.9.6"
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.1"
  }
 },
 "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }
--- a/examples/llama-3/lora-1b-deduplicate-dpo.yml
+++ b/examples/llama-3/lora-1b-deduplicate-dpo.yml
@@ -1,95 +0,0 @@
-base_model: meta-llama/Llama-3.2-1B
-model_type: LlamaForCausalLM
-tokenizer_type: AutoTokenizer
-
-load_in_8bit: true
-load_in_4bit: false
-strict: false
-
-chat_template: llama3
-rl: dpo
-datasets:
-  - path: fozziethebeat/alpaca_messages_2k_dpo_test
-    type: chat_template.default
-    field_messages: conversation
-    field_chosen: chosen
-    field_rejected: rejected
-    message_field_role: role
-    message_field_content: content
-    roles:
-      system:
-        - system
-      user:
-        - user
-      assistant:
-        - assistant
-  - path: fozziethebeat/alpaca_messages_2k_dpo_test
-    type: chat_template.default
-    field_messages: conversation
-    field_chosen: chosen
-    field_rejected: rejected
-    message_field_role: role
-    message_field_content: content
-    roles:
-      system:
-        - system
-      user:
-        - user
-      assistant:
-        - assistant
-
-dataset_exact_deduplication: true
-dataset_prepared_path:
-val_set_size: 0
-output_dir: ./outputs/lora-out
-
-sequence_len: 4096
-sample_packing: false
-pad_to_sequence_len: true
-
-adapter: lora
-lora_model_dir:
-lora_r: 32
-lora_alpha: 16
-lora_dropout: 0.05
-lora_target_linear: true
-lora_fan_in_fan_out:
-
-wandb_project:
-wandb_entity:
-wandb_watch:
-wandb_name:
-wandb_log_model:
-
-gradient_accumulation_steps: 4
-micro_batch_size: 2
-num_epochs: 4
-optimizer: adamw_bnb_8bit
-lr_scheduler: cosine
-learning_rate: 0.0002
-
-train_on_inputs: false
-group_by_length: false
-bf16: auto
-fp16:
-tf32: false
-
-gradient_checkpointing: true
-early_stopping_patience:
-resume_from_checkpoint:
-local_rank:
-logging_steps: 1
-xformers_attention:
-flash_attention: true
-s2_attention:
-
-warmup_steps: 10
-evals_per_epoch: 4
-eval_table_size:
-eval_max_new_tokens: 128
-saves_per_epoch: 1
-debug:
-deepspeed:
-weight_decay: 0.0
-fsdp:
-fsdp_config:
--- a/examples/llama-3/lora-1b-deduplicate-sft.yml
+++ b/examples/llama-3/lora-1b-deduplicate-sft.yml
@@ -1,76 +0,0 @@
-base_model: meta-llama/Llama-3.2-1B
-model_type: LlamaForCausalLM
-tokenizer_type: AutoTokenizer
-
-load_in_8bit: true
-load_in_4bit: false
-strict: false
-
-datasets:
-  - path: mhenrichsen/alpaca_2k_test
-    type: alpaca
-  - path: mhenrichsen/alpaca_2k_test
-    type: alpaca
-dataset_prepared_path:
-val_set_size: 0.0
-output_dir: ./outputs/lora-out
-
-dataset_exact_deduplication: true
-test_value: true
-
-sequence_len: 4096
-sample_packing: true
-eval_sample_packing: false
-pad_to_sequence_len: true
-
-adapter: lora
-lora_model_dir:
-lora_r: 32
-lora_alpha: 16
-lora_dropout: 0.05
-lora_target_linear: true
-lora_fan_in_fan_out:
-lora_modules_to_save:
-  - embed_tokens
-  - lm_head
-
-wandb_project:
-wandb_entity:
-wandb_watch:
-wandb_name:
-wandb_log_model:
-
-gradient_accumulation_steps: 4
-micro_batch_size: 2
-num_epochs: 4
-optimizer: adamw_bnb_8bit
-lr_scheduler: cosine
-learning_rate: 0.0002
-
-train_on_inputs: false
-group_by_length: false
-bf16: auto
-fp16:
-tf32: false
-
-gradient_checkpointing: true
-early_stopping_patience:
-resume_from_checkpoint:
-local_rank:
-logging_steps: 1
-xformers_attention:
-flash_attention: true
-s2_attention:
-
-warmup_steps: 10
-evals_per_epoch: 4
-eval_table_size:
-eval_max_new_tokens: 128
-saves_per_epoch: 1
-debug:
-deepspeed:
-weight_decay: 0.0
-fsdp:
-fsdp_config:
-special_tokens:
-   pad_token: <|end_of_text|>
--- a/examples/llava/lora-7b.yaml
+++ b/examples/llava/lora-7b.yaml
@@ -1,63 +0,0 @@
-base_model: llava-hf/llava-1.5-7b-hf
-processor_type: AutoProcessor
-strict: false
-
-# these 3 lines are needed for now to handle vision chat templates w images
-skip_prepare_dataset: true
-remove_unused_columns: false
-sample_packing: false
-
-chat_template: llava
-datasets:
-  - path: HuggingFaceH4/llava-instruct-mix-vsft
-    type: chat_template
-    split: train[:1%]
-    field_messages: messages
-dataset_prepared_path: last_run_prepared
-val_set_size: 0.0
-output_dir: ./outputs/out
-
-adapter: lora
-lora_model_dir:
-
-sequence_len: 8192
-pad_to_sequence_len: false
-
-lora_r: 32
-lora_alpha: 16
-lora_dropout: 0.05
-lora_target_modules: 'language_model.model.layers.[\d]+.(mlp|cross_attn|self_attn).(up|down|gate|q|k|v|o)_proj'
-
-wandb_project:
-wandb_entity:
-wandb_watch:
-wandb_name:
-wandb_log_model:
-
-gradient_accumulation_steps: 4
-micro_batch_size: 1
-num_epochs: 1
-optimizer: adamw_bnb_8bit
-lr_scheduler: cosine
-learning_rate: 0.0002
-
-train_on_inputs: false
-group_by_length: false
-bf16: true
-fp16:
-tf32: true
-
-gradient_checkpointing: true
-local_rank:
-logging_steps: 1
-flash_attention: true
-eager_attention:
-
-warmup_ratio: 0.1
-evals_per_epoch: 1
-saves_per_epoch: 1
-debug:
-deepspeed:
-weight_decay: 0.0
-fsdp:
-fsdp_config:
--- a/examples/pixtral/lora-12b.yml
+++ b/examples/pixtral/lora-12b.yml
@@ -1,65 +0,0 @@
-base_model: mistral-community/pixtral-12b
-processor_type: AutoProcessor
-strict: false
-
-# these 3 lines are needed for now to handle vision chat templates w images
-skip_prepare_dataset: true
-remove_unused_columns: false
-sample_packing: false
-
-chat_template: pixtral
-datasets:
-  - path: HuggingFaceH4/llava-instruct-mix-vsft
-    type: chat_template
-    split: train[:1%]
-    field_messages: messages
-dataset_prepared_path: last_run_prepared
-val_set_size: 0.0
-output_dir: ./outputs/out
-
-adapter: lora
-lora_model_dir:
-
-sequence_len: 8192
-pad_to_sequence_len: false
-
-lora_r: 32
-lora_alpha: 16
-lora_dropout: 0.05
-lora_target_modules: 'language_model.model.layers.[\d]+.(mlp|cross_attn|self_attn).(up|down|gate|q|k|v|o)_proj'
-
-wandb_project:
-wandb_entity:
-wandb_watch:
-wandb_name:
-wandb_log_model:
-
-gradient_accumulation_steps: 4
-micro_batch_size: 1
-num_epochs: 1
-optimizer: adamw_bnb_8bit
-lr_scheduler: cosine
-learning_rate: 0.0002
-
-train_on_inputs: false
-group_by_length: false
-bf16: true
-fp16:
-tf32: true
-
-gradient_checkpointing: true
-local_rank:
-logging_steps: 1
-flash_attention: false # PixtralVisionModel does not support Flash Attention 2.0 yet
-eager_attention:
-
-warmup_ratio: 0.1
-evals_per_epoch: 1
-saves_per_epoch: 1
-debug:
-deepspeed:
-weight_decay: 0.0
-fsdp:
-fsdp_config:
-special_tokens:
-  pad_token: <|end_of_text|>
--- a/examples/qwen2-vl/lora-7b.yaml
+++ b/examples/qwen2-vl/lora-7b.yaml
@@ -1,63 +0,0 @@
-base_model: Qwen/Qwen2-VL-7B-Instruct
-processor_type: AutoProcessor
-strict: false
-
-# these 3 lines are needed for now to handle vision chat templates w images
-skip_prepare_dataset: true
-remove_unused_columns: false
-sample_packing: false
-
-chat_template: qwen2_vl
-datasets:
-  - path: HuggingFaceH4/llava-instruct-mix-vsft
-    type: chat_template
-    split: train[:1%]
-    field_messages: messages
-dataset_prepared_path: last_run_prepared
-val_set_size: 0.0
-output_dir: ./outputs/out
-
-adapter: lora
-lora_model_dir:
-
-sequence_len: 8192
-pad_to_sequence_len: false
-
-lora_r: 32
-lora_alpha: 16
-lora_dropout: 0.05
-lora_target_modules: 'model.layers.[\d]+.(mlp|cross_attn|self_attn).(up|down|gate|q|k|v|o)_proj'
-
-wandb_project:
-wandb_entity:
-wandb_watch:
-wandb_name:
-wandb_log_model:
-
-gradient_accumulation_steps: 4
-micro_batch_size: 1
-num_epochs: 1
-optimizer: adamw_bnb_8bit
-lr_scheduler: cosine
-learning_rate: 0.0002
-
-train_on_inputs: false
-group_by_length: false
-bf16: true
-fp16:
-tf32: true
-
-gradient_checkpointing: true
-local_rank:
-logging_steps: 1
-flash_attention: true
-eager_attention:
-
-warmup_ratio: 0.1
-evals_per_epoch: 1
-saves_per_epoch: 1
-debug:
-deepspeed:
-weight_decay: 0.0
-fsdp:
-fsdp_config:
--- a/image/axolotl-badge-web-legacy.png
+++ b/image/axolotl-badge-web-legacy.png
--- a/image/axolotl-badge-web.png
+++ b/image/axolotl-badge-web.png
--- a/image/axolotl_logo_digital_black.svg
+++ b/image/axolotl_logo_digital_black.svg
@@ -1,19 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="0 0 1113 283.5">
-    <path fill="#141310" d="M435,234.3l-12.1-48.8h-54.4l-12.1,48.8h-24.7l48.2-185.1h31.6l47.9,185.1h-24.5ZM417.7,164.9l-13.8-55.6c-2.7-10.7-4.8-19.7-6.3-26.9-.9-4.2-1.5-7.5-2-9.9-.5,2.5-1.2,5.8-2,9.9-1.5,7.1-3.6,16.1-6.3,26.7l-13.8,55.9h44.3Z"/>
-    <path fill="#141310" d="M568.2,234.3l-29.9-45.6c-1.2-1.9-2.4-4.1-3.5-6.5-.8-1.7-1.5-3.3-2.1-4.5-.6,1.3-1.4,2.8-2.3,4.5-1.3,2.4-2.6,4.6-4,6.5l-29.9,45.6h-28.5l49.6-71.9-46.5-67.9h28.5l27.6,43.1c1.2,1.9,2.3,3.9,3.4,6.1.7,1.4,1.4,2.7,1.9,3.8.5-1.1,1.1-2.4,1.8-3.8,1.1-2.2,2.2-4.2,3.4-6.1l27.6-43.1h28.5l-46.5,68.2,49.3,71.7h-28.5Z"/>
-    <path fill="#141310" d="M658.6,236.3c-16.7,0-30.2-5-40.1-14.8-9.9-9.8-14.9-23.7-14.9-41.3v-31.7c0-17.7,5-31.7,14.8-41.4,9.8-9.7,23.4-14.7,40.3-14.7s30.4,4.9,40.3,14.7c9.8,9.7,14.8,23.7,14.8,41.4v31.7c0,17.6-5,31.5-14.9,41.3-9.9,9.8-23.4,14.8-40.1,14.8ZM658.6,114.1c-9.5,0-17.1,2.7-22.6,8.1-5.5,5.4-8.3,13.4-8.3,23.8v36.7c0,10.5,2.8,18.5,8.3,23.8,5.5,5.4,13.1,8.1,22.6,8.1s17.3-2.7,22.7-8.1c5.4-5.4,8.2-13.4,8.2-23.9v-36.7c0-10.5-2.8-18.5-8.2-23.9-5.4-5.4-13.1-8.1-22.7-8.1Z"/>
-    <path fill="#141310" d="M860.6,236.3c-16.7,0-30.2-5-40.1-14.8-9.9-9.8-14.9-23.7-14.9-41.3v-31.7c0-17.7,5-31.7,14.8-41.4,9.8-9.7,23.4-14.7,40.3-14.7s30.4,4.9,40.3,14.7c9.8,9.7,14.8,23.7,14.8,41.4v31.7c0,17.6-5,31.5-14.9,41.3-9.9,9.8-23.4,14.8-40.1,14.8ZM860.6,114.1c-9.5,0-17.1,2.7-22.6,8.1-5.5,5.4-8.3,13.4-8.3,23.8v36.7c0,10.5,2.8,18.5,8.3,23.8,5.5,5.4,13.1,8.1,22.6,8.1s17.3-2.7,22.7-8.1c5.4-5.4,8.2-13.4,8.2-23.9v-36.7c0-10.5-2.8-18.5-8.2-23.9-5.4-5.4-13.1-8.1-22.7-8.1Z"/>
-    <path fill="#141310" d="M773.9,234c-18,0-32.6-14.6-32.6-32.6V48.8h24.1v152.6c0,4.7,3.8,8.5,8.5,8.5h16.8v24.1h-16.8Z"/>
-    <path fill="#141310" d="M1036.2,234.3V81.4c0-4.7-3.8-8.5-8.5-8.5h-16.8v-24.1h16.8c18,0,32.6,14.6,32.6,32.6v152.9h-24.1Z"/>
-    <path fill="#141310" d="M978.6,234.3c-18,0-32.6-14.6-32.6-32.6v-85.1h-20.3v-22.1h20.3v-45.3h24.1v45.3h30.2v22.1h-30.2v85.1c0,4.7,3.8,8.5,8.5,8.5h21.7v24.1h-21.7Z"/>
-    <path fill="#141310" d="M51.5,49h12.2v-20.6h-12.2c-16,0-29,13-29,29v32.8h20.6v-32.8c0-4.7,3.8-8.4,8.4-8.4Z"/>
-    <path fill="#141310" d="M92.8,49h12.2v-20.6h-12.2c-16,0-29,13-29,29v12.2h20.6v-12.2c0-4.7,3.8-8.4,8.4-8.4Z"/>
-    <path fill="#141310" d="M249.3,57.4c0-16-13-29-29-29h-12.2v20.6h12.2c4.7,0,8.4,3.8,8.4,8.4v32.8h20.6v-32.8Z"/>
-    <path fill="#141310" d="M187.4,90.2v-20.6h-103.1v20.6h-41.2v20.6h-20.6v41.2c0,11.4,9.2,20.6,20.6,20.6h185.5c11.4,0,20.6-9.2,20.6-20.6v-41.2h-20.6v-20.6h-41.2ZM166.8,141.7c0-5.7-4.6-10.3-10.3-10.3s-10.3,4.6-10.3,10.3v10.3h-20.6v-20.6c0-11.4,9.2-20.6,20.6-20.6s20.6,9.2,20.6,20.6v10.3ZM228.7,141.7c0-5.7-4.6-10.3-10.3-10.3s-10.3,4.6-10.3,10.3v10.3h-20.6v-20.6c0-11.4,9.2-20.6,20.6-20.6s20.6,9.2,20.6,20.6v10.3Z"/>
-    <path fill="#141310" d="M208,57.4c0-16-13-29-29-29h-12.2v20.6h12.2c4.7,0,8.4,3.8,8.4,8.4v12.2h20.6v-12.2Z"/>
-    <rect fill="#141310" x="22.5" y="234.5" width="41.2" height="20.6"/>
-    <rect fill="#141310" x="84.3" y="234.5" width="164.9" height="20.6"/>
-    <rect fill="#141310" x="208" y="193.3" width="41.2" height="20.6"/>
-    <rect fill="#141310" x="22.5" y="193.3" width="164.9" height="20.6"/>
-</svg>
--- a/image/axolotl_logo_digital_white.svg
+++ b/image/axolotl_logo_digital_white.svg
@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="0 0 1113 283.5">
-    <path fill="#fff" d="M462.9,234.2l-12.1-48.8h-54.4l-12.1,48.8h-24.7l48.2-185h31.6l47.9,185h-24.4ZM445.7,164.8l-13.8-55.6c-2.7-10.7-4.8-19.7-6.3-26.9-.9-4.2-1.5-7.5-2-9.9-.5,2.5-1.2,5.8-2,9.9-1.5,7.1-3.6,16.1-6.3,26.7l-13.8,55.9h44.3Z"/>
-    <path fill="#fff" d="M596.1,234.2l-29.9-45.6c-1.2-1.9-2.4-4.1-3.5-6.5-.8-1.7-1.5-3.3-2.1-4.5-.6,1.3-1.4,2.8-2.3,4.5-1.3,2.4-2.6,4.6-4,6.5l-29.9,45.6h-28.5l49.5-71.9-46.5-67.9h28.5l27.6,43.1c1.2,1.9,2.3,3.9,3.4,6.1.7,1.4,1.3,2.7,1.9,3.8.5-1.1,1.1-2.4,1.8-3.8,1.1-2.2,2.2-4.2,3.4-6.1l27.6-43.1h28.5l-46.5,68.1,49.3,71.6h-28.5Z"/>
-    <path fill="#fff" d="M686.4,236.2c-16.7,0-30.2-5-40.1-14.8-9.9-9.8-14.9-23.7-14.9-41.3v-31.7c0-17.7,5-31.6,14.8-41.4,9.8-9.7,23.4-14.7,40.2-14.7s30.4,4.9,40.2,14.7c9.8,9.7,14.8,23.7,14.8,41.4v31.7c0,17.6-5,31.4-14.9,41.3-9.9,9.8-23.4,14.8-40.1,14.8ZM686.4,114.1c-9.5,0-17.1,2.7-22.6,8.1-5.5,5.4-8.3,13.4-8.3,23.8v36.7c0,10.5,2.8,18.5,8.3,23.8,5.5,5.4,13.1,8.1,22.6,8.1s17.3-2.7,22.7-8.1c5.4-5.4,8.2-13.4,8.2-23.8v-36.7c0-10.5-2.8-18.5-8.2-23.8-5.4-5.4-13.1-8.1-22.7-8.1Z"/>
-    <path fill="#fff" d="M888.3,236.2c-16.7,0-30.2-5-40.1-14.8-9.9-9.8-14.9-23.7-14.9-41.3v-31.7c0-17.7,5-31.6,14.8-41.4,9.8-9.7,23.4-14.7,40.2-14.7s30.4,4.9,40.2,14.7c9.8,9.7,14.8,23.7,14.8,41.4v31.7c0,17.6-5,31.4-14.9,41.3-9.9,9.8-23.4,14.8-40.1,14.8ZM888.3,114.1c-9.5,0-17.1,2.7-22.6,8.1-5.5,5.4-8.3,13.4-8.3,23.8v36.7c0,10.5,2.8,18.5,8.3,23.8,5.5,5.4,13.1,8.1,22.6,8.1s17.3-2.7,22.7-8.1c5.4-5.4,8.2-13.4,8.2-23.8v-36.7c0-10.5-2.8-18.5-8.2-23.8-5.4-5.4-13.1-8.1-22.7-8.1Z"/>
-    <path fill="#fff" d="M801.7,234c-18,0-32.6-14.6-32.6-32.6V48.8h24.1v152.5c0,4.7,3.8,8.5,8.5,8.5h16.7v24.1h-16.7Z"/>
-    <path fill="#fff" d="M1063.8,234.2V81.4c0-4.7-3.8-8.5-8.5-8.5h-16.7v-24.1h16.7c18,0,32.6,14.6,32.6,32.6v152.8h-24.1Z"/>
-    <path fill="#fff" d="M1006.2,234.2c-18,0-32.6-14.6-32.6-32.6v-85h-20.3v-22.1h20.3v-45.2h24.1v45.2h30.2v22.1h-30.2v85c0,4.7,3.8,8.5,8.5,8.5h21.7v24.1h-21.7Z"/>
-    <path fill="#fff" d="M160.2,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM222,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM222,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM160.2,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM160.2,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM222,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM222,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM160.2,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM160.2,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM222,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM222,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM160.2,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM277.3,57.4c0-23.8-19.3-43.1-43.1-43.1h-12.2c-3.9,0-7.6,1.6-10.2,4.4-5.9-2.9-12.3-4.4-18.9-4.4h-12.2c-7.7,0-14.1,6.3-14.1,14.1v20.6c0,2.4.6,4.6,1.6,6.6h-37c1-2,1.6-4.2,1.6-6.6v-20.6c0-7.7-6.3-14.1-14.1-14.1h-12.2c-6.5,0-13,1.5-18.9,4.4-2.6-2.8-6.3-4.4-10.2-4.4h-12.2c-23.8,0-43.1,19.3-43.1,43.1v32.8c0,4.1,1.7,7.7,4.5,10.3-2.8,2.6-4.5,6.2-4.5,10.3v41.2c0,11,5.2,20.8,13.2,27.2-7.3.4-13.2,6.6-13.2,14v20.6c0,4.1,1.7,7.7,4.5,10.3-2.8,2.6-4.5,6.2-4.5,10.3v20.6c0,7.7,6.3,14.1,14.1,14.1h41.2c4.1,0,7.7-1.7,10.3-4.5,2.6,2.8,6.2,4.5,10.3,4.5h164.9c7.7,0,14.1-6.3,14.1-14.1v-20.6c0-4.1-1.7-7.7-4.5-10.3,2.8-2.6,4.5-6.2,4.5-10.3v-20.6c0-7.5-5.8-13.6-13.2-14,8-6.4,13.2-16.2,13.2-27.2v-41.2c0-4.1-1.7-7.7-4.5-10.3,2.8-2.6,4.5-6.2,4.5-10.3v-32.8ZM77.8,255.1h-41.2v-20.6h41.2v20.6ZM36.5,213.9v-20.6h164.9v20.6H36.5ZM263.3,255.1H98.4v-20.6h164.9v20.6ZM263.3,213.9h-41.2v-20.6h41.2v20.6ZM263.3,90.2h-20.6v20.6h20.6v41.2c0,11.4-9.2,20.6-20.6,20.6H57.2c-11.4,0-20.6-9.2-20.6-20.6v-41.2h20.6v-20.6h-20.6v-32.8c0-16,13-29,29-29h12.2v20.6h-12.2c-4.7,0-8.4,3.8-8.4,8.4v32.8h41.2v-20.6h-20.6v-12.2c0-16,13-29,29-29h12.2v20.6h-12.2c-4.7,0-8.4,3.8-8.4,8.4v12.2h103.1v-12.2c0-4.7-3.8-8.4-8.4-8.4h-12.2v-20.6h12.2c16,0,29,13,29,29v12.2h-20.6v20.6h41.2v-32.8c0-4.7-3.8-8.4-8.4-8.4h-12.2v-20.6h12.2c16,0,29,13,29,29v32.8ZM201.4,152h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6s-20.6,9.2-20.6,20.6v20.6ZM160.2,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM160.2,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM222,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM222,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM160.2,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM160.2,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM222,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM222,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM160.2,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM160.2,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM222,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM222,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM160.2,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6Z"/>
-</svg>
--- a/image/axolotl_symbol_digital_black.svg
+++ b/image/axolotl_symbol_digital_black.svg
@@ -1,26 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="0 0 283.5 283.5">
-  <defs>
-    <style>
-      .cls-1 {
-        fill: #141310;
-      }
-    </style>
-  </defs>
-  <!-- Generator: Adobe Illustrator 28.7.1, SVG Export Plug-In . SVG Version: 1.2.0 Build 142)  -->
-  <g>
-    <g id="Layer_1">
-      <g>
-        <path class="cls-1" d="M46.9,37.4h13.7V14.2h-13.7c-18,0-32.7,14.6-32.7,32.7v36.9h23.2v-36.9c0-5.2,4.2-9.5,9.5-9.5Z"/>
-        <path class="cls-1" d="M93.2,37.4h13.7V14.2h-13.7c-18,0-32.7,14.6-32.7,32.7v13.7h23.2v-13.7c0-5.2,4.2-9.5,9.5-9.5Z"/>
-        <path class="cls-1" d="M269.3,46.9c0-18-14.6-32.7-32.7-32.7h-13.7v23.2h13.7c5.2,0,9.5,4.2,9.5,9.5v36.9h23.2v-36.9Z"/>
-        <path class="cls-1" d="M199.7,83.8v-23.2h-116v23.2h-46.4v23.2H14.2v46.4c0,12.8,10.4,23.2,23.2,23.2h208.7c12.8,0,23.2-10.4,23.2-23.2v-46.4h-23.2v-23.2h-46.4ZM176.5,141.7c0-6.4-5.2-11.6-11.6-11.6s-11.6,5.2-11.6,11.6v11.6h-23.2v-23.2c0-12.8,10.4-23.2,23.2-23.2s23.2,10.4,23.2,23.2v11.6ZM246.1,141.7c0-6.4-5.2-11.6-11.6-11.6s-11.6,5.2-11.6,11.6v11.6h-23.2v-23.2c0-12.8,10.4-23.2,23.2-23.2s23.2,10.4,23.2,23.2v11.6Z"/>
-        <path class="cls-1" d="M222.9,46.9c0-18-14.6-32.7-32.7-32.7h-13.7v23.2h13.7c5.2,0,9.5,4.2,9.5,9.5v13.7h23.2v-13.7Z"/>
-        <rect class="cls-1" x="14.2" y="246.1" width="46.4" height="23.2"/>
-        <rect class="cls-1" x="83.8" y="246.1" width="185.5" height="23.2"/>
-        <rect class="cls-1" x="222.9" y="199.7" width="46.4" height="23.2"/>
-        <rect class="cls-1" x="14.2" y="199.7" width="185.5" height="23.2"/>
-      </g>
-    </g>
-  </g>
-</svg>
--- a/image/axolotl_symbol_digital_white.svg
+++ b/image/axolotl_symbol_digital_white.svg
@@ -1,16 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="0 0 283.5 283.5">
-  <defs>
-    <style>
-      .cls-1 {
-        fill: #fff;
-      }
-    </style>
-  </defs>
-  <!-- Generator: Adobe Illustrator 28.7.1, SVG Export Plug-In . SVG Version: 1.2.0 Build 142)  -->
-  <g>
-    <g id="Layer_1">
-      <path class="cls-1" d="M152,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM214,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM214,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM152,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM152,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM214,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM214,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM152,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM152,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM214,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM214,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM152,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM269.3,57.3c0-23.8-19.4-43.1-43.1-43.1h-12.2c-3.9,0-7.6,1.6-10.2,4.4-5.9-2.9-12.3-4.4-18.9-4.4h-12.2c-7.8,0-14.1,6.3-14.1,14.1v20.6c0,2.4.6,4.6,1.6,6.6h-37c1-2,1.6-4.2,1.6-6.6v-20.6c0-7.8-6.3-14.1-14.1-14.1h-12.2c-6.6,0-13,1.5-18.9,4.4-2.6-2.8-6.3-4.4-10.2-4.4h-12.2c-23.8,0-43.1,19.4-43.1,43.1v32.8c0,4.1,1.7,7.7,4.5,10.3-2.8,2.6-4.5,6.2-4.5,10.3v41.3c0,11,5.2,20.9,13.2,27.2-7.4.4-13.2,6.6-13.2,14v20.6c0,4.1,1.7,7.7,4.5,10.3-2.8,2.6-4.5,6.2-4.5,10.3v20.6c0,7.8,6.3,14.1,14.1,14.1h41.3c4.1,0,7.7-1.7,10.3-4.5,2.6,2.8,6.2,4.5,10.3,4.5h165.1c7.8,0,14.1-6.3,14.1-14.1v-20.6c0-4.1-1.7-7.7-4.5-10.3,2.8-2.6,4.5-6.2,4.5-10.3v-20.6c0-7.5-5.9-13.6-13.2-14,8-6.4,13.2-16.2,13.2-27.2v-41.3c0-4.1-1.7-7.7-4.5-10.3,2.8-2.6,4.5-6.2,4.5-10.3v-32.8ZM69.5,255.2H28.2v-20.6h41.3v20.6ZM28.2,214v-20.6h165.1v20.6H28.2ZM255.2,255.2H90.1v-20.6h165.1v20.6ZM255.2,214h-41.3v-20.6h41.3v20.6ZM255.2,90.1h-20.6v20.6h20.6v41.3c0,11.4-9.2,20.6-20.6,20.6H48.9c-11.4,0-20.6-9.2-20.6-20.6v-41.3h20.6v-20.6h-20.6v-32.8c0-16.1,13-29.1,29.1-29.1h12.2v20.6h-12.2c-4.7,0-8.4,3.8-8.4,8.4v32.8h41.3v-20.6h-20.6v-12.2c0-16.1,13-29.1,29.1-29.1h12.2v20.6h-12.2c-4.7,0-8.4,3.8-8.4,8.4v12.2h103.2v-12.2c0-4.7-3.8-8.4-8.4-8.4h-12.2v-20.6h12.2c16.1,0,29.1,13,29.1,29.1v12.2h-20.6v20.6h41.3v-32.8c0-4.7-3.8-8.4-8.4-8.4h-12.2v-20.6h12.2c16.1,0,29.1,13,29.1,29.1v32.8ZM193.3,152h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6s-20.6,9.2-20.6,20.6v20.6ZM152,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM152,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM214,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM214,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM152,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM152,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM214,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM214,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM152,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM152,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM214,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM214,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6ZM152,110.8c-11.4,0-20.6,9.2-20.6,20.6v20.6h20.6v-10.3c0-5.7,4.6-10.3,10.3-10.3s10.3,4.6,10.3,10.3v-10.3c0-11.4-9.2-20.6-20.6-20.6Z"/>
-    </g>
-  </g>
-</svg>
--- a/image/axolotl_wordmark_digital_black.svg
+++ b/image/axolotl_wordmark_digital_black.svg
@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="0 0 765.4 212.6">
-  <!-- Generator: Adobe Illustrator 28.7.1, SVG Export Plug-In . SVG Version: 1.2.0 Build 142)  -->
-  <g>
-    <g id="Layer_1">
-      <g>
-        <path d="M121.6,198.1l-12.1-48.8h-54.4l-12.1,48.8h-24.7L66.6,12.9h31.6l47.9,185.1h-24.5ZM104.4,128.6l-13.8-55.6c-2.7-10.7-4.8-19.7-6.3-26.9-.9-4.2-1.5-7.5-2-9.9-.5,2.5-1.2,5.8-2,9.9-1.5,7.1-3.6,16.1-6.3,26.7l-13.8,55.9h44.3Z"/>
-        <path d="M254.9,198.1l-29.9-45.6c-1.2-1.9-2.4-4.1-3.5-6.5-.8-1.7-1.5-3.3-2.1-4.5-.6,1.3-1.4,2.8-2.3,4.5-1.3,2.4-2.6,4.6-4,6.5l-29.9,45.6h-28.5l49.6-71.9-46.5-67.9h28.5l27.6,43.1c1.2,1.9,2.3,3.9,3.4,6.1.7,1.4,1.4,2.7,1.9,3.8.5-1.1,1.1-2.4,1.8-3.8,1.1-2.2,2.2-4.2,3.4-6.1l27.6-43.1h28.5l-46.5,68.2,49.3,71.7h-28.5Z"/>
-        <path d="M345.2,200.1c-16.7,0-30.2-5-40.1-14.8-9.9-9.8-14.9-23.7-14.9-41.3v-31.7c0-17.7,5-31.7,14.8-41.4,9.8-9.7,23.4-14.7,40.3-14.7s30.4,4.9,40.3,14.7c9.8,9.7,14.8,23.7,14.8,41.4v31.7c0,17.6-5,31.5-14.9,41.3-9.9,9.8-23.4,14.8-40.1,14.8ZM345.2,77.8c-9.5,0-17.1,2.7-22.6,8.1-5.5,5.4-8.3,13.4-8.3,23.8v36.7c0,10.5,2.8,18.5,8.3,23.8,5.5,5.4,13.1,8.1,22.6,8.1s17.3-2.7,22.7-8.1c5.4-5.4,8.2-13.4,8.2-23.9v-36.7c0-10.5-2.8-18.5-8.2-23.9-5.4-5.4-13.1-8.1-22.7-8.1Z"/>
-        <path d="M547.3,200.1c-16.7,0-30.2-5-40.1-14.8-9.9-9.8-14.9-23.7-14.9-41.3v-31.7c0-17.7,5-31.7,14.8-41.4,9.8-9.7,23.4-14.7,40.3-14.7s30.4,4.9,40.3,14.7c9.8,9.7,14.8,23.7,14.8,41.4v31.7c0,17.6-5,31.5-14.9,41.3-9.9,9.8-23.4,14.8-40.1,14.8ZM547.3,77.8c-9.5,0-17.1,2.7-22.6,8.1-5.5,5.4-8.3,13.4-8.3,23.8v36.7c0,10.5,2.8,18.5,8.3,23.8,5.5,5.4,13.1,8.1,22.6,8.1s17.3-2.7,22.7-8.1c5.4-5.4,8.2-13.4,8.2-23.9v-36.7c0-10.5-2.8-18.5-8.2-23.9-5.4-5.4-13.1-8.1-22.7-8.1Z"/>
-        <path d="M460.6,197.8c-18,0-32.6-14.6-32.6-32.6V12.5h24.1v152.6c0,4.7,3.8,8.5,8.5,8.5h16.8v24.1h-16.8Z"/>
-        <path d="M722.8,198.1V45.2c0-4.7-3.8-8.5-8.5-8.5h-16.8V12.5h16.8c18,0,32.6,14.6,32.6,32.6v152.9h-24.1Z"/>
-        <path d="M665.2,198.1c-18,0-32.6-14.6-32.6-32.6v-85.1h-20.3v-22.1h20.3V12.9h24.1v45.3h30.2v22.1h-30.2v85.1c0,4.7,3.8,8.5,8.5,8.5h21.7v24.1h-21.7Z"/>
-      </g>
-    </g>
-  </g>
-</svg>
--- a/image/axolotl_wordmark_digital_white.svg
+++ b/image/axolotl_wordmark_digital_white.svg
@@ -1,24 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="0 0 765.4 212.6">
-  <defs>
-    <style>
-      .cls-1 {
-        fill: #fff;
-      }
-    </style>
-  </defs>
-  <!-- Generator: Adobe Illustrator 28.7.1, SVG Export Plug-In . SVG Version: 1.2.0 Build 142)  -->
-  <g>
-    <g id="Layer_1">
-      <g>
-        <path class="cls-1" d="M121.6,198.1l-12.1-48.8h-54.4l-12.1,48.8h-24.7L66.6,12.9h31.6l47.9,185.1h-24.5ZM104.4,128.6l-13.8-55.6c-2.7-10.7-4.8-19.7-6.3-26.9-.9-4.2-1.5-7.5-2-9.9-.5,2.5-1.2,5.8-2,9.9-1.5,7.1-3.6,16.1-6.3,26.7l-13.8,55.9h44.3Z"/>
-        <path class="cls-1" d="M254.9,198.1l-29.9-45.6c-1.2-1.9-2.4-4.1-3.5-6.5-.8-1.7-1.5-3.3-2.1-4.5-.6,1.3-1.4,2.8-2.3,4.5-1.3,2.4-2.6,4.6-4,6.5l-29.9,45.6h-28.5l49.6-71.9-46.5-67.9h28.5l27.6,43.1c1.2,1.9,2.3,3.9,3.4,6.1.7,1.4,1.4,2.7,1.9,3.8.5-1.1,1.1-2.4,1.8-3.8,1.1-2.2,2.2-4.2,3.4-6.1l27.6-43.1h28.5l-46.5,68.2,49.3,71.7h-28.5Z"/>
-        <path class="cls-1" d="M345.2,200.1c-16.7,0-30.2-5-40.1-14.8-9.9-9.8-14.9-23.7-14.9-41.3v-31.7c0-17.7,5-31.7,14.8-41.4,9.8-9.7,23.4-14.7,40.3-14.7s30.4,4.9,40.3,14.7c9.8,9.7,14.8,23.7,14.8,41.4v31.7c0,17.6-5,31.5-14.9,41.3-9.9,9.8-23.4,14.8-40.1,14.8ZM345.2,77.8c-9.5,0-17.1,2.7-22.6,8.1-5.5,5.4-8.3,13.4-8.3,23.8v36.7c0,10.5,2.8,18.5,8.3,23.8,5.5,5.4,13.1,8.1,22.6,8.1s17.3-2.7,22.7-8.1c5.4-5.4,8.2-13.4,8.2-23.9v-36.7c0-10.5-2.8-18.5-8.2-23.9-5.4-5.4-13.1-8.1-22.7-8.1Z"/>
-        <path class="cls-1" d="M547.3,200.1c-16.7,0-30.2-5-40.1-14.8-9.9-9.8-14.9-23.7-14.9-41.3v-31.7c0-17.7,5-31.7,14.8-41.4,9.8-9.7,23.4-14.7,40.3-14.7s30.4,4.9,40.3,14.7c9.8,9.7,14.8,23.7,14.8,41.4v31.7c0,17.6-5,31.5-14.9,41.3-9.9,9.8-23.4,14.8-40.1,14.8ZM547.3,77.8c-9.5,0-17.1,2.7-22.6,8.1-5.5,5.4-8.3,13.4-8.3,23.8v36.7c0,10.5,2.8,18.5,8.3,23.8,5.5,5.4,13.1,8.1,22.6,8.1s17.3-2.7,22.7-8.1c5.4-5.4,8.2-13.4,8.2-23.9v-36.7c0-10.5-2.8-18.5-8.2-23.9-5.4-5.4-13.1-8.1-22.7-8.1Z"/>
-        <path class="cls-1" d="M460.6,197.8c-18,0-32.6-14.6-32.6-32.6V12.5h24.1v152.6c0,4.7,3.8,8.5,8.5,8.5h16.8v24.1h-16.8Z"/>
-        <path class="cls-1" d="M722.8,198.1V45.2c0-4.7-3.8-8.5-8.5-8.5h-16.8V12.5h16.8c18,0,32.6,14.6,32.6,32.6v152.9h-24.1Z"/>
-        <path class="cls-1" d="M665.2,198.1c-18,0-32.6-14.6-32.6-32.6v-85.1h-20.3v-22.1h20.3V12.9h24.1v45.3h30.2v22.1h-30.2v85.1c0,4.7,3.8,8.5,8.5,8.5h21.7v24.1h-21.7Z"/>
-      </g>
-    </g>
-  </g>
-</svg>
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -2,3 +2,4 @@ pre-commit
 black
 mypy
 types-requests
+tbparse
--- a/requirements-tests.txt
+++ b/requirements-tests.txt
@@ -1,5 +1,3 @@
 pytest
 pytest-xdist
 pytest-retry
-pytest-sugar
-tbparse
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,12 +1,12 @@
 --extra-index-url https://huggingface.github.io/autogptq-index/whl/cu118/
 packaging==23.2
 peft==0.13.2
-transformers==4.46.3
+transformers==4.46.2
 tokenizers>=0.20.1
 bitsandbytes==0.44.1
 accelerate==1.1.0
 datasets==3.1.0
-deepspeed==0.15.4
+deepspeed==0.15.3
 pydantic==2.6.3
 addict
 fire
@@ -26,14 +26,14 @@ numpy>=1.24.4,<=2.0.1
 evaluate==0.4.1
 scipy
 scikit-learn==1.4.2
-nvidia-ml-py==12.560.30
+pynvml
 art
 gradio==3.50.2
 tensorboard
 python-dotenv==1.0.1
-autoawq==0.2.7.post2
+autoawq>=0.2.5
 triton>=2.3.0
-liger-kernel==0.4.2
+liger-kernel==0.4.1

 mamba-ssm==1.2.0.post1

--- a/scripts/cutcrossentropy_install.py
+++ b/scripts/cutcrossentropy_install.py
@@ -1,28 +0,0 @@
-"""Script to output the correct installation command for cut-cross-entropy."""
-import importlib.util
-import sys
-
-try:
-    import torch
-except ImportError as exc:
-    raise ImportError("Install torch via `pip install torch`") from exc
-from packaging.version import Version as V
-
-v = V(torch.__version__)
-
-# no cut-cross-entropy support for torch < 2.4.0
-if v < V("2.4.0"):
-    print("")
-    sys.exit(0)
-
-cce_spec = importlib.util.find_spec("cut_cross_entropy")
-cce_spec_transformers = importlib.util.find_spec("cut_cross_entropy.transformers")
-
-UNINSTALL_PREFIX = ""
-if cce_spec and not cce_spec_transformers:
-    UNINSTALL_PREFIX = "pip uninstall -y cut-cross-entropy && "
-
-print(
-    UNINSTALL_PREFIX
-    + 'pip install "cut-cross-entropy @ git+https://github.com/apple/ml-cross-entropy.git@9c297c905f55b73594b5d650722d1e78183b77bd"'
-)
--- a/scripts/unsloth_install.py
+++ b/scripts/unsloth_install.py
@@ -1,36 +0,0 @@
-# noqa
-# pylint: skip-file
-try:
-    import torch
-except ImportError:
-    raise ImportError("Install torch via `pip install torch`")
-from packaging.version import Version as V
-
-v = V(torch.__version__)
-cuda = str(torch.version.cuda)
-try:
-    is_ampere = torch.cuda.get_device_capability()[0] >= 8
-except RuntimeError:
-    is_ampere = False
-if cuda != "12.1" and cuda != "11.8" and cuda != "12.4":
-    raise RuntimeError(f"CUDA = {cuda} not supported!")
-if v <= V("2.1.0"):
-    raise RuntimeError(f"Torch = {v} too old!")
-elif v <= V("2.1.1"):
-    x = "cu{}{}-torch211"
-elif v <= V("2.1.2"):
-    x = "cu{}{}-torch212"
-elif v < V("2.3.0"):
-    x = "cu{}{}-torch220"
-elif v < V("2.4.0"):
-    x = "cu{}{}-torch230"
-elif v < V("2.5.0"):
-    x = "cu{}{}-torch240"
-elif v < V("2.6.0"):
-    x = "cu{}{}-torch250"
-else:
-    raise RuntimeError(f"Torch = {v} too new!")
-x = x.format(cuda.replace(".", ""), "-ampere" if is_ampere else "")
-print(
-    f'pip install unsloth-zoo==2024.11.7 && pip install --no-deps "unsloth[{x}]==2024.11.9"'
-)
--- a/setup.py
+++ b/setup.py
@@ -96,11 +96,11 @@ install_requires, dependency_links = parse_requirements()

 setup(
    name="axolotl",
-    version="0.5.2",
+    version="0.5.0",
    description="LLM Trainer",
    long_description="Axolotl is a tool designed to streamline the fine-tuning of various AI models, offering support for multiple configurations and architectures.",
    package_dir={"": "src"},
-    packages=find_packages("src"),
+    packages=find_packages(),
    install_requires=install_requires,
    dependency_links=dependency_links,
    extras_require={
@@ -108,7 +108,7 @@ setup(
            "flash-attn==2.7.0.post2",
        ],
        "deepspeed": [
-            "deepspeed==0.15.4",
+            "deepspeed==0.14.4",
            "deepspeed-kernels",
        ],
        "mamba-ssm": [
--- a/src/axolotl/cli/init.py
+++ b/src/axolotl/cli/init.py
@@ -27,17 +27,14 @@ from transformers.utils import is_torch_bf16_gpu_available
 from transformers.utils.import_utils import _is_package_available

 from axolotl.common.cli import TrainerCliArgs, load_model_and_tokenizer
+from axolotl.integrations.base import PluginManager
 from axolotl.logging_config import configure_logging
 from axolotl.train import TrainDatasetMeta
-from axolotl.utils.chat_templates import (
-    get_chat_template,
-    get_chat_template_from_config,
-)
+from axolotl.utils.chat_templates import get_chat_template
 from axolotl.utils.comet_ import setup_comet_env_vars
 from axolotl.utils.config import (
    normalize_cfg_datasets,
    normalize_config,
-    prepare_plugins,
    validate_config,
 )
 from axolotl.utils.data import load_prepare_dpo_datasets, prepare_dataset
@@ -100,8 +97,8 @@ def print_dep_versions():
        print("*" * 40)
        print("**** Axolotl Dependency Versions *****")
        for pkg in packages:
-            pkg_version = _is_package_available(pkg, return_version=True)
-            print(f"{pkg: >{max_len}}: {pkg_version[1]: <15}")
+            version = _is_package_available(pkg, return_version=True)
+            print(f"{pkg: >{max_len}}: {version[1]: <15}")
        print("*" * 40)


@@ -139,7 +136,7 @@ def check_remote_config(config: Union[str, Path]):
        with open(output_path, "wb") as file:
            file.write(content)
        LOG.info(
-            f"Using the following config obtained from {config}: \n\n{content.decode('utf-8')}\n"
+            f"Using the following config obtained from {config}:\n\n{content.decode('utf-8')}\n"
        )
        return output_path

@@ -202,10 +199,6 @@ def do_inference(
        )
    elif cfg.chat_template:
        chat_template_str = get_chat_template(cfg.chat_template)
-    elif cfg.datasets[0].type == "chat_template":
-        chat_template_str = get_chat_template_from_config(
-            cfg=cfg, ds_cfg=cfg.datasets[0], tokenizer=tokenizer
-        )

    model = model.to(cfg.device, dtype=cfg.torch_dtype)

@@ -426,6 +419,11 @@ def load_cfg(config: Union[str, Path] = Path("examples/"), **kwargs):

    cfg.axolotl_config_path = config

+    if cfg.get("plugins"):
+        plugin_manager = PluginManager.get_instance()
+        for plugin_name in cfg["plugins"]:
+            plugin_manager.register(plugin_name)
+
    try:
        device_props = torch.cuda.get_device_properties("cuda")
        gpu_version = "sm_" + str(device_props.major) + str(device_props.minor)
@@ -439,13 +437,8 @@ def load_cfg(config: Union[str, Path] = Path("examples/"), **kwargs):
            "n_gpu": int(os.environ.get("WORLD_SIZE", 1)),
            "compute_capability": gpu_version,
        },
-        env_capabilities={
-            "torch_version": str(torch.__version__).split("+", maxsplit=1)[0]
-        },
    )

-    prepare_plugins(cfg)
-
    prepare_optim_env(cfg)

    prepare_opinionated_env(cfg)
--- a/src/axolotl/cli/inference.py
+++ b/src/axolotl/cli/inference.py
@@ -19,7 +19,7 @@ from axolotl.common.cli import TrainerCliArgs
 def do_cli(config: Path = Path("examples/"), gradio=False, **kwargs):
    # pylint: disable=duplicate-code
    print_axolotl_text_art()
-    parsed_cfg = load_cfg(config, inference=True, **kwargs)
+    parsed_cfg = load_cfg(config, **kwargs)
    parsed_cfg.sample_packing = False
    parser = transformers.HfArgumentParser((TrainerCliArgs))
    parsed_cli_args, _ = parser.parse_args_into_dataclasses(
--- a/src/axolotl/common/architectures.py
+++ b/src/axolotl/common/architectures.py
@@ -10,6 +10,7 @@ MOE_ARCH_BLOCK = {
        "JetMoeMoE",
    ],
    "mixtral": "MixtralSparseMoeBlock",
+    "phimoe": "PhiMoESparseMoeBlock",
    "qwen2_moe": "Qwen2MoeSparseMoeBlock",
    "deepseek_v2": "DeepseekV2MoE",
 }
--- a/src/axolotl/core/trainer_builder.py
+++ b/src/axolotl/core/trainer_builder.py
@@ -107,22 +107,6 @@ def _sanitize_kwargs_for_tagging(tag_names, kwargs=None):
    return kwargs


-def _sanitize_kwargs_for_ds_tagging(dataset_tags, kwargs=None):
-    if isinstance(dataset_tags, str):
-        dataset_tags = [dataset_tags]
-
-    if (dataset_tags is not None) and (kwargs is not None):
-        if "dataset_tags" not in kwargs:
-            kwargs["dataset_tags"] = dataset_tags
-        elif "dataset_tags" in kwargs and isinstance(kwargs["dataset_tags"], list):
-            kwargs["dataset_tags"].extend(dataset_tags)
-        elif "dataset_tags" in kwargs and isinstance(kwargs["dataset_tags"], str):
-            dataset_tags.append(kwargs["dataset_tags"])
-            kwargs["dataset_tags"] = dataset_tags
-
-    return kwargs
-
-
@dataclass
 class AxolotlTrainingMixins:
    """
@@ -236,14 +220,6 @@ class AxolotlTrainingMixins:
        default=1e-6,
        metadata={"help": "loraplus learning rate for lora embedding layers."},
    )
-    embedding_lr_scale: Optional[float] = field(
-        default=None,
-        metadata={"help": "Scale the learning rate for the embedding layers."},
-    )
-    embedding_lr: Optional[float] = field(
-        default=None,
-        metadata={"help": "absolute learning rate for the embedding layers."},
-    )
    qlora: bool = field(
        default=False,
        metadata={"help": "whether this is a qlora training"},
@@ -410,7 +386,7 @@ class SchedulerMixin(Trainer):
                    min_lr_ratio=self.args.cosine_min_lr_ratio,
                )
            else:
-                return super().create_scheduler(num_training_steps, optimizer=optimizer)
+                return super().create_scheduler(num_training_steps, optimizer)
        else:
            if use_cosine_quadratic:
                LOG.warning("axolotl's cosine scheduler with quadratic warmup not used (e.g., because of deepspeed).")
@@ -434,12 +410,10 @@ class AxolotlTrainer(SchedulerMixin, Trainer):
        *_args,
        bench_data_collator=None,
        eval_data_collator=None,
-        dataset_tags=None,
        **kwargs,
    ):
        self.bench_data_collator = bench_data_collator
        self.eval_data_collator = eval_data_collator
-        self.dataset_tags = dataset_tags
        super().__init__(*_args, **kwargs)
        self.train_data_collator = self.data_collator
        self._stored_metrics = defaultdict(lambda: defaultdict(list))
@@ -461,8 +435,6 @@ class AxolotlTrainer(SchedulerMixin, Trainer):
    def create_optimizer(self):
        if (
            self.args.loraplus_lr_ratio is None
-            and self.args.embedding_lr_scale is None
-            and self.args.embedding_lr is None
            and self.args.alternate_optimizer
            not in [
                "optimi_adamw",
@@ -477,59 +449,30 @@ class AxolotlTrainer(SchedulerMixin, Trainer):
        opt_model = self.model_wrapped if is_sagemaker_mp_enabled() else self.model
        if self.optimizer is None:  # pylint: disable=access-member-before-definition
            decay_parameters = self.get_decay_parameter_names(opt_model)
-            params = {
-                "to_weight_decay": {},  # LayerNorm and bias
-                "embeddings": {},  # lm_head, embed_tokens,
-                "no_weight_decay": {},
-            }
+            optimizer_grouped_parameters = [
+                {
+                    "params": [
+                        p
+                        for n, p in opt_model.named_parameters()
+                        if (n in decay_parameters and p.requires_grad)
+                    ],
+                    "weight_decay": self.args.weight_decay,
+                },
+                {
+                    "params": [
+                        p
+                        for n, p in opt_model.named_parameters()
+                        if (n not in decay_parameters and p.requires_grad)
+                    ],
+                    "weight_decay": 0.0,
+                },
+            ]

            optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(
                self.args,
                opt_model,
            )

-            for name, param in opt_model.named_parameters():
-                if not param.requires_grad:
-                    continue
-                if name.endswith("modules_to_save.default.weight") or any(
-                    embed_name in name for embed_name in ["embed_tokens", "lm_head"]
-                ):
-                    params["embeddings"][name] = param
-                elif name in decay_parameters:
-                    params["to_weight_decay"][name] = param
-                else:
-                    params["no_weight_decay"][name] = param
-            optimizer_grouped_parameters = []
-            if params["to_weight_decay"]:
-                optimizer_grouped_parameters.append(
-                    {
-                        "params": list(params["to_weight_decay"].values()),
-                        "weight_decay": self.args.weight_decay,
-                        "lr": optimizer_kwargs["lr"],
-                    }
-                )
-            if params["embeddings"]:
-                lr = optimizer_kwargs["lr"]  # pylint: disable=invalid-name
-                if self.args.embedding_lr_scale:
-                    lr *= self.args.embedding_lr_scale  # pylint: disable=invalid-name
-                elif self.args.embedding_lr:
-                    lr = self.args.embedding_lr  # pylint: disable=invalid-name
-                optimizer_grouped_parameters.append(
-                    {
-                        "params": list(params["embeddings"].values()),
-                        "weight_decay": 0.0,
-                        "lr": lr,
-                    }
-                )
-            if params["no_weight_decay"]:
-                optimizer_grouped_parameters.append(
-                    {
-                        "params": list(params["no_weight_decay"].values()),
-                        "weight_decay": 0.0,
-                        "lr": optimizer_kwargs["lr"],
-                    }
-                )
-
            if self.args.loraplus_lr_ratio is not None:
                loraplus_lr_ratio = getattr(self.args, "loraplus_lr_ratio", None)
                loraplus_lr_embedding = getattr(
@@ -542,13 +485,6 @@ class AxolotlTrainer(SchedulerMixin, Trainer):
                    loraplus_lr_embedding=loraplus_lr_embedding,
                    **optimizer_kwargs,
                )
-            elif (
-                self.args.embedding_lr_scale is not None
-                or self.args.embedding_lr is not None
-            ):
-                self.optimizer = (  # pylint: disable=attribute-defined-outside-init
-                    optimizer_cls(optimizer_grouped_parameters, **optimizer_kwargs)
-                )
            elif self.args.alternate_optimizer == "optimi_adamw":
                from optimi import AdamW

@@ -580,9 +516,7 @@ class AxolotlTrainer(SchedulerMixin, Trainer):

                self.optimizer = (  # pylint: disable=attribute-defined-outside-init
                    ADOPT(
-                        optimizer_grouped_parameters,
-                        decouple=True,
-                        **optimizer_kwargs,
+                        optimizer_grouped_parameters, decoupled=True, **optimizer_kwargs
                    )
                )

@@ -937,9 +871,6 @@ class AxolotlTrainer(SchedulerMixin, Trainer):
        Overwrite the `push_to_hub` method in order to force-add the tags when pushing the
        model on the Hub. Please refer to `~transformers.Trainer.push_to_hub` for more details.
        """
-        kwargs = _sanitize_kwargs_for_ds_tagging(
-            dataset_tags=self.dataset_tags, kwargs=kwargs
-        )
        kwargs = _sanitize_kwargs_for_tagging(tag_names=self.tag_names, kwargs=kwargs)

        return super().push_to_hub(*args, **kwargs)
@@ -1063,9 +994,8 @@ class AxolotlDPOTrainer(SchedulerMixin, DPOTrainer):

    tag_names = ["axolotl", "dpo"]

-    def __init__(self, *args, dataset_tags=None, **kwargs):
+    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
-        self.dataset_tags = dataset_tags
        self.optimizer = None

    def create_optimizer(self):
@@ -1104,9 +1034,6 @@ class AxolotlDPOTrainer(SchedulerMixin, DPOTrainer):
        Overwrite the `push_to_hub` method in order to force-add the tags when pushing the
        model on the Hub. Please refer to `~transformers.Trainer.push_to_hub` for more details.
        """
-        kwargs = _sanitize_kwargs_for_ds_tagging(
-            dataset_tags=self.dataset_tags, kwargs=kwargs
-        )
        kwargs = _sanitize_kwargs_for_tagging(tag_names=self.tag_names, kwargs=kwargs)

        return super().push_to_hub(*args, **kwargs)
@@ -1285,17 +1212,11 @@ class TrainerBuilderBase(abc.ABC):
        Callbacks added after the trainer is created, usually b/c these need access to the trainer
        """
        callbacks = []
-        if self.cfg.plugins:
-            plugin_manager = PluginManager.get_instance()
-            callbacks.extend(
-                [
-                    cb
-                    for cb in plugin_manager.add_callbacks_post_trainer(
-                        self.cfg, trainer
-                    )
-                    if cb
-                ]
-            )
+
+        plugin_manager = PluginManager.get_instance()
+        callbacks.extend(
+            plugin_manager.add_callbacks_post_trainer(cfg=self.cfg, trainer=trainer)
+        )
        return callbacks

    def hook_pre_create_training_args(self, training_arguments_kwargs):
@@ -1342,7 +1263,7 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
        return callbacks

    def get_post_trainer_create_callbacks(self, trainer):
-        callbacks = []
+        callbacks = super().get_post_trainer_create_callbacks(trainer=trainer)
        if self.cfg.use_wandb and self.cfg.eval_table_size > 0:
            LogPredictionCallback = log_prediction_callback_factory(
                trainer, self.tokenizer, "wandb"
@@ -1380,7 +1301,17 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
        if self.cfg.lisa_step_interval and self.cfg.lisa_n_layers:
            callbacks.append(lisa_callback_factory(trainer))

-        callbacks.extend(super().get_post_trainer_create_callbacks(trainer=trainer))
+        if self.cfg.plugins:
+            plugin_manager = PluginManager.get_instance()
+            callbacks.extend(
+                [
+                    cb
+                    for cb in plugin_manager.add_callbacks_post_trainer(
+                        self.cfg, trainer
+                    )
+                    if cb
+                ]
+            )
        return callbacks

    def _get_trainer_cls(self):
@@ -1644,9 +1575,6 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
        training_arguments_kwargs[
            "loraplus_lr_embedding"
        ] = self.cfg.loraplus_lr_embedding
-        training_arguments_kwargs["embedding_lr"] = self.cfg.embedding_lr
-        training_arguments_kwargs["embedding_lr_scale"] = self.cfg.embedding_lr_scale
-
        if self.cfg.lr_scheduler in ["one_cycle", "log_sweep"]:
            training_arguments_kwargs["lr_scheduler_type"] = "cosine"
            training_arguments_kwargs[
@@ -1831,10 +1759,6 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
        else:
            trainer_kwargs["tokenizer"] = self.tokenizer

-        if (trainer_cls is not AxolotlRewardTrainer) and self.cfg.datasets is not None:
-            trainer_kwargs["dataset_tags"] = [
-                d["path"] for d in self.cfg.datasets if not Path(d["path"]).is_dir()
-            ]
        trainer = trainer_cls(
            model=self.model,
            train_dataset=self.train_dataset,
@@ -1897,7 +1821,6 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
                collator = MultiModalChatDataCollator
                kwargs["processor"] = self.processor
                kwargs["chat_template"] = training_args.chat_template
-                kwargs["chat_template_type"] = self.cfg.chat_template
            else:
                collator = DataCollatorForSeq2Seq

@@ -2109,10 +2032,6 @@ class HFRLTrainerBuilder(TrainerBuilderBase):
        else:
            dpo_trainer_kwargs["tokenizer"] = self.tokenizer

-        if self.cfg.datasets is not None and (trainer_cls is AxolotlDPOTrainer):
-            dpo_trainer_kwargs["dataset_tags"] = [
-                d["path"] for d in self.cfg.datasets if not Path(d["path"]).is_dir()
-            ]
        dpo_trainer = trainer_cls(
            *trainer_cls_args,
            args=training_args,
--- a/src/axolotl/core/trainers/trl.py
+++ b/src/axolotl/core/trainers/trl.py
@@ -40,7 +40,7 @@ class TRLPPOTrainer(PPOTrainer):
                query_tensors,
                return_prompt=False,
                generate_ref_response=True,
-                **generation_kwargs,
+                **generation_kwargs
            )
            batch["response"] = self.tokenizer.batch_decode(response_tensors)
            batch["ref_response"] = self.tokenizer.batch_decode(ref_response_tensors)
--- a/src/axolotl/integrations/cut_cross_entropy/ACKNOWLEDGEMENTS.md
+++ b/src/axolotl/integrations/cut_cross_entropy/ACKNOWLEDGEMENTS.md
@@ -1,325 +0,0 @@
-Acknowledgements
-
-Portions of this Cut Cross Entropy Software may utilize the following copyrighted
-material, the use of which is hereby acknowledged.
-
-
------
-
-
-PyTorch
-
-    From PyTorch:
-
-    Copyright (c) 2016-     Facebook, Inc            (Adam Paszke)
-    Copyright (c) 2014-     Facebook, Inc            (Soumith Chintala)
-    Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)
-    Copyright (c) 2012-2014 Deepmind Technologies    (Koray Kavukcuoglu)
-    Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)
-    Copyright (c) 2011-2013 NYU                      (Clement Farabet)
-    Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston)
-    Copyright (c) 2006      Idiap Research Institute (Samy Bengio)
-    Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz)
-
-    From Caffe2:
-
-    Copyright (c) 2016-present, Facebook Inc. All rights reserved.
-
-    All contributions by Facebook:
-    Copyright (c) 2016 Facebook Inc.
-
-    All contributions by Google:
-    Copyright (c) 2015 Google Inc.
-    All rights reserved.
-
-    All contributions by Yangqing Jia:
-    Copyright (c) 2015 Yangqing Jia
-    All rights reserved.
-
-    All contributions by Kakao Brain:
-    Copyright 2019-2020 Kakao Brain
-
-    All contributions by Cruise LLC:
-    Copyright (c) 2022 Cruise LLC.
-    All rights reserved.
-
-    All contributions by Arm:
-    Copyright (c) 2021, 2023-2024 Arm Limited and/or its affiliates
-
-    All contributions from Caffe:
-    Copyright(c) 2013, 2014, 2015, the respective contributors
-    All rights reserved.
-
-    All other contributions:
-    Copyright(c) 2015, 2016 the respective contributors
-    All rights reserved.
-
-    Caffe2 uses a copyright model similar to Caffe: each contributor holds
-    copyright over their contributions to Caffe2. The project versioning records
-    all such contribution and copyright details. If a contributor wants to further
-    mark their specific copyright on a particular contribution, they should
-    indicate their copyright solely in the commit message of the change when it is
-    committed.
-
-    All rights reserved.
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are met:
-
-    1. Redistributions of source code must retain the above copyright
-    notice, this list of conditions and the following disclaimer.
-
-    2. Redistributions in binary form must reproduce the above copyright
-    notice, this list of conditions and the following disclaimer in the
-    documentation and/or other materials provided with the distribution.
-
-    3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America
-    and IDIAP Research Institute nor the names of its contributors may be
-    used to endorse or promote products derived from this software without
-    specific prior written permission.
-
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-    AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-    IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-    ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-    LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-    CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-    POSSIBILITY OF SUCH DAMAGE.
-
-
-Triton
-
-    /*
-    * Copyright 2018-2020 Philippe Tillet
-    * Copyright 2020-2022 OpenAI
-    *
-    * Permission is hereby granted, free of charge, to any person obtaining
-    * a copy of this software and associated documentation files
-    * (the "Software"), to deal in the Software without restriction,
-    * including without limitation the rights to use, copy, modify, merge,
-    * publish, distribute, sublicense, and/or sell copies of the Software,
-    * and to permit persons to whom the Software is furnished to do so,
-    * subject to the following conditions:
-    *
-    * The above copyright notice and this permission notice shall be
-    * included in all copies or substantial portions of the Software.
-    *
-    * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-    * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-    * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-    * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-    * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-    * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-    * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-    */
-
-
-Transformers
-
-    Copyright 2018- The Hugging Face team. All rights reserved.
-
-                                    Apache License
-                            Version 2.0, January 2004
-                            http://www.apache.org/licenses/
-
-    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-    1. Definitions.
-
-        "License" shall mean the terms and conditions for use, reproduction,
-        and distribution as defined by Sections 1 through 9 of this document.
-
-        "Licensor" shall mean the copyright owner or entity authorized by
-        the copyright owner that is granting the License.
-
-        "Legal Entity" shall mean the union of the acting entity and all
-        other entities that control, are controlled by, or are under common
-        control with that entity. For the purposes of this definition,
-        "control" means (i) the power, direct or indirect, to cause the
-        direction or management of such entity, whether by contract or
-        otherwise, or (ii) ownership of fifty percent (50%) or more of the
-        outstanding shares, or (iii) beneficial ownership of such entity.
-
-        "You" (or "Your") shall mean an individual or Legal Entity
-        exercising permissions granted by this License.
-
-        "Source" form shall mean the preferred form for making modifications,
-        including but not limited to software source code, documentation
-        source, and configuration files.
-
-        "Object" form shall mean any form resulting from mechanical
-        transformation or translation of a Source form, including but
-        not limited to compiled object code, generated documentation,
-        and conversions to other media types.
-
-        "Work" shall mean the work of authorship, whether in Source or
-        Object form, made available under the License, as indicated by a
-        copyright notice that is included in or attached to the work
-        (an example is provided in the Appendix below).
-
-        "Derivative Works" shall mean any work, whether in Source or Object
-        form, that is based on (or derived from) the Work and for which the
-        editorial revisions, annotations, elaborations, or other modifications
-        represent, as a whole, an original work of authorship. For the purposes
-        of this License, Derivative Works shall not include works that remain
-        separable from, or merely link (or bind by name) to the interfaces of,
-        the Work and Derivative Works thereof.
-
-        "Contribution" shall mean any work of authorship, including
-        the original version of the Work and any modifications or additions
-        to that Work or Derivative Works thereof, that is intentionally
-        submitted to Licensor for inclusion in the Work by the copyright owner
-        or by an individual or Legal Entity authorized to submit on behalf of
-        the copyright owner. For the purposes of this definition, "submitted"
-        means any form of electronic, verbal, or written communication sent
-        to the Licensor or its representatives, including but not limited to
-        communication on electronic mailing lists, source code control systems,
-        and issue tracking systems that are managed by, or on behalf of, the
-        Licensor for the purpose of discussing and improving the Work, but
-        excluding communication that is conspicuously marked or otherwise
-        designated in writing by the copyright owner as "Not a Contribution."
-
-        "Contributor" shall mean Licensor and any individual or Legal Entity
-        on behalf of whom a Contribution has been received by Licensor and
-        subsequently incorporated within the Work.
-
-    2. Grant of Copyright License. Subject to the terms and conditions of
-        this License, each Contributor hereby grants to You a perpetual,
-        worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-        copyright license to reproduce, prepare Derivative Works of,
-        publicly display, publicly perform, sublicense, and distribute the
-        Work and such Derivative Works in Source or Object form.
-
-    3. Grant of Patent License. Subject to the terms and conditions of
-        this License, each Contributor hereby grants to You a perpetual,
-        worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-        (except as stated in this section) patent license to make, have made,
-        use, offer to sell, sell, import, and otherwise transfer the Work,
-        where such license applies only to those patent claims licensable
-        by such Contributor that are necessarily infringed by their
-        Contribution(s) alone or by combination of their Contribution(s)
-        with the Work to which such Contribution(s) was submitted. If You
-        institute patent litigation against any entity (including a
-        cross-claim or counterclaim in a lawsuit) alleging that the Work
-        or a Contribution incorporated within the Work constitutes direct
-        or contributory patent infringement, then any patent licenses
-        granted to You under this License for that Work shall terminate
-        as of the date such litigation is filed.
-
-    4. Redistribution. You may reproduce and distribute copies of the
-        Work or Derivative Works thereof in any medium, with or without
-        modifications, and in Source or Object form, provided that You
-        meet the following conditions:
-
-        (a) You must give any other recipients of the Work or
-            Derivative Works a copy of this License; and
-
-        (b) You must cause any modified files to carry prominent notices
-            stating that You changed the files; and
-
-        (c) You must retain, in the Source form of any Derivative Works
-            that You distribute, all copyright, patent, trademark, and
-            attribution notices from the Source form of the Work,
-            excluding those notices that do not pertain to any part of
-            the Derivative Works; and
-
-        (d) If the Work includes a "NOTICE" text file as part of its
-            distribution, then any Derivative Works that You distribute must
-            include a readable copy of the attribution notices contained
-            within such NOTICE file, excluding those notices that do not
-            pertain to any part of the Derivative Works, in at least one
-            of the following places: within a NOTICE text file distributed
-            as part of the Derivative Works; within the Source form or
-            documentation, if provided along with the Derivative Works; or,
-            within a display generated by the Derivative Works, if and
-            wherever such third-party notices normally appear. The contents
-            of the NOTICE file are for informational purposes only and
-            do not modify the License. You may add Your own attribution
-            notices within Derivative Works that You distribute, alongside
-            or as an addendum to the NOTICE text from the Work, provided
-            that such additional attribution notices cannot be construed
-            as modifying the License.
-
-        You may add Your own copyright statement to Your modifications and
-        may provide additional or different license terms and conditions
-        for use, reproduction, or distribution of Your modifications, or
-        for any such Derivative Works as a whole, provided Your use,
-        reproduction, and distribution of the Work otherwise complies with
-        the conditions stated in this License.
-
-    5. Submission of Contributions. Unless You explicitly state otherwise,
-        any Contribution intentionally submitted for inclusion in the Work
-        by You to the Licensor shall be under the terms and conditions of
-        this License, without any additional terms or conditions.
-        Notwithstanding the above, nothing herein shall supersede or modify
-        the terms of any separate license agreement you may have executed
-        with Licensor regarding such Contributions.
-
-    6. Trademarks. This License does not grant permission to use the trade
-        names, trademarks, service marks, or product names of the Licensor,
-        except as required for reasonable and customary use in describing the
-        origin of the Work and reproducing the content of the NOTICE file.
-
-    7. Disclaimer of Warranty. Unless required by applicable law or
-        agreed to in writing, Licensor provides the Work (and each
-        Contributor provides its Contributions) on an "AS IS" BASIS,
-        WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-        implied, including, without limitation, any warranties or conditions
-        of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-        PARTICULAR PURPOSE. You are solely responsible for determining the
-        appropriateness of using or redistributing the Work and assume any
-        risks associated with Your exercise of permissions under this License.
-
-    8. Limitation of Liability. In no event and under no legal theory,
-        whether in tort (including negligence), contract, or otherwise,
-        unless required by applicable law (such as deliberate and grossly
-        negligent acts) or agreed to in writing, shall any Contributor be
-        liable to You for damages, including any direct, indirect, special,
-        incidental, or consequential damages of any character arising as a
-        result of this License or out of the use or inability to use the
-        Work (including but not limited to damages for loss of goodwill,
-        work stoppage, computer failure or malfunction, or any and all
-        other commercial damages or losses), even if such Contributor
-        has been advised of the possibility of such damages.
-
-    9. Accepting Warranty or Additional Liability. While redistributing
-        the Work or Derivative Works thereof, You may choose to offer,
-        and charge a fee for, acceptance of support, warranty, indemnity,
-        or other liability obligations and/or rights consistent with this
-        License. However, in accepting such obligations, You may act only
-        on Your own behalf and on Your sole responsibility, not on behalf
-        of any other Contributor, and only if You agree to indemnify,
-        defend, and hold each Contributor harmless for any liability
-        incurred by, or claims asserted against, such Contributor by reason
-        of your accepting any such warranty or additional liability.
-
-    END OF TERMS AND CONDITIONS
-
-    APPENDIX: How to apply the Apache License to your work.
-
-        To apply the Apache License to your work, attach the following
-        boilerplate notice, with the fields enclosed by brackets "[]"
-        replaced with your own identifying information. (Don't include
-        the brackets!)  The text should be enclosed in the appropriate
-        comment syntax for the file format. We also recommend that a
-        file or class name and description of purpose be included on the
-        same "printed page" as the copyright notice for easier
-        identification within third-party archives.
-
-    Copyright [yyyy] [name of copyright owner]
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-        http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
--- a/src/axolotl/integrations/cut_cross_entropy/LICENSE
+++ b/src/axolotl/integrations/cut_cross_entropy/LICENSE
@@ -1,47 +0,0 @@
-Copyright (C) 2024 Apple Inc. All Rights Reserved.
-
-IMPORTANT:  This Apple software is supplied to you by Apple
-Inc. ("Apple") in consideration of your agreement to the following
-terms, and your use, installation, modification or redistribution of
-this Apple software constitutes acceptance of these terms.  If you do
-not agree with these terms, please do not use, install, modify or
-redistribute this Apple software.
-
-In consideration of your agreement to abide by the following terms, and
-subject to these terms, Apple grants you a personal, non-exclusive
-license, under Apple's copyrights in this original Apple software (the
-"Apple Software"), to use, reproduce, modify and redistribute the Apple
-Software, with or without modifications, in source and/or binary forms;
-provided that if you redistribute the Apple Software in its entirety and
-without modifications, you must retain this notice and the following
-text and disclaimers in all such redistributions of the Apple Software.
-Neither the name, trademarks, service marks or logos of Apple Inc. may
-be used to endorse or promote products derived from the Apple Software
-without specific prior written permission from Apple.  Except as
-expressly stated in this notice, no other rights or licenses, express or
-implied, are granted by Apple herein, including but not limited to any
-patent rights that may be infringed by your derivative works or by other
-works in which the Apple Software may be incorporated.
-
-The Apple Software is provided by Apple on an "AS IS" basis.  APPLE
-MAKES NO WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION
-THE IMPLIED WARRANTIES OF NON-INFRINGEMENT, MERCHANTABILITY AND FITNESS
-FOR A PARTICULAR PURPOSE, REGARDING THE APPLE SOFTWARE OR ITS USE AND
-OPERATION ALONE OR IN COMBINATION WITH YOUR PRODUCTS.
-
-IN NO EVENT SHALL APPLE BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL
-OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) ARISING IN ANY WAY OUT OF THE USE, REPRODUCTION,
-MODIFICATION AND/OR DISTRIBUTION OF THE APPLE SOFTWARE, HOWEVER CAUSED
-AND WHETHER UNDER THEORY OF CONTRACT, TORT (INCLUDING NEGLIGENCE),
-STRICT LIABILITY OR OTHERWISE, EVEN IF APPLE HAS BEEN ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-
-------------------------------------------------------------------------------
-SOFTWARE DISTRIBUTED WITH CUT CROSS ENTROPY:
-
-The Cut Cross Entropy software includes a number of subcomponents with separate
-copyright notices and license terms - please see the file ACKNOWLEDGEMENTS.md.
-------------------------------------------------------------------------------
--- a/src/axolotl/integrations/cut_cross_entropy/README.md
+++ b/src/axolotl/integrations/cut_cross_entropy/README.md
@@ -1,10 +0,0 @@
-# Cut Cross Entropy
-
-### Usage
-
-```yaml
-plugins:
-  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
-
-cut_cross_entropy: true
-```
--- a/src/axolotl/integrations/cut_cross_entropy/init.py
+++ b/src/axolotl/integrations/cut_cross_entropy/init.py
@@ -1,83 +0,0 @@
-# Copyright 2024 Axolotl AI. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Module for the Plugin for Cut Cross Entropy integration with Axolotl.
-
-Cut Cross Entropy is an optimized implementation of cross entropy loss
-from Apple's ML team.
-"""
-import importlib
-import logging
-
-import torch
-
-from axolotl.integrations.base import BasePlugin
-from axolotl.utils import get_pytorch_version
-
-from ...utils.distributed import zero_only
-from .args import CutCrossEntropyArgs  # pylint: disable=unused-import. # noqa: F401
-
-LOG = logging.getLogger("axolotl.integrations.cut_cross_entropy")
-
-_CCE_INSTALL_MESSAGE = (
-    "Please install cut_cross_entropy with transformers support using "
-    '`pip install "cut-cross-entropy[transformers]==24.11.4"`'
-)
-
-
-class CutCrossEntropyPlugin(BasePlugin):
-    """
-    Plugin for Cut Cross Entropy integration with Axolotl.
-    """
-
-    def get_input_args(self):
-        return "axolotl.integrations.cut_cross_entropy.CutCrossEntropyArgs"
-
-    def _check_requirements(self):
-        """Check if all requirements are met."""
-        # Check PyTorch version
-
-        major, minor, _ = get_pytorch_version()
-        if (major, minor) < (2, 4):
-            raise ImportError(
-                "Cut Cross Entropy requires PyTorch >= 2.4.0. "
-                f"Current version: {torch.__version__}"
-            )
-
-        # Check if cut_cross_entropy is installed
-        cce_spec = importlib.util.find_spec("cut_cross_entropy")
-        if cce_spec is None:
-            raise ImportError(_CCE_INSTALL_MESSAGE)
-
-        cce_spec_transformers = importlib.util.find_spec(
-            "cut_cross_entropy.transformers"
-        )
-        if cce_spec_transformers is None:
-            raise ImportError(_CCE_INSTALL_MESSAGE)
-
-    def pre_model_load(self, cfg):
-        """Apply cut cross entropy before model loading if enabled."""
-        if cfg.cut_cross_entropy:
-            self._check_requirements()
-
-            from cut_cross_entropy.transformers import cce_patch
-
-            with zero_only():
-                LOG.info(
-                    f"Applying Cut Cross Entropy to model type: {cfg.model_config_type}"
-                )
-
-            # The patch checks model_type internally
-            cce_patch(cfg.model_config_type)
--- a/src/axolotl/integrations/cut_cross_entropy/args.py
+++ b/src/axolotl/integrations/cut_cross_entropy/args.py
@@ -1,42 +0,0 @@
-# Copyright 2024 Axolotl AI. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""
-Module for handling Cut Cross Entropy input arguments.
-"""
-import logging
-from typing import Optional
-
-from pydantic import BaseModel, model_validator
-
-LOG = logging.getLogger("axolotl.integrations.cut_cross_entropy.args")
-
-
-class CutCrossEntropyArgs(BaseModel):
-    """
-    Input args for Cut Cross Entropy.
-    """
-
-    cut_cross_entropy: Optional[bool] = None
-
-    @model_validator(mode="before")
-    @classmethod
-    def check_dtype_is_half(cls, data):
-        if not (data.get("bf16") or data.get("fp16")):
-            raise ValueError(
-                "Cut Cross Entropy requires fp16/bf16 training for backward pass. "
-                "Please set `bf16` or `fp16` to `True`."
-            )
-
-        return data
--- a/src/axolotl/monkeypatch/init.py
+++ b/src/axolotl/monkeypatch/init.py
--- a/src/axolotl/monkeypatch/attention/init.py
+++ b/src/axolotl/monkeypatch/attention/init.py
--- a/src/axolotl/monkeypatch/llama_attn_hijack_flash.py
+++ b/src/axolotl/monkeypatch/llama_attn_hijack_flash.py
@@ -4,6 +4,7 @@

 import logging
 import warnings
+from functools import partial
 from typing import List, Optional, Tuple, Union

 import torch
@@ -93,32 +94,13 @@ def replace_llama_qkv_with_fused(model):
            set_module_name(model, name, qkv)


-def patch_fa_llama_cross_entropy():
-    LOG.info(
-        "patching transformers.loss.loss_utils.fixed_cross_entropy with flash_attn.ops.triton.cross_entropy"
-    )
-    from flash_attn.ops.triton.cross_entropy import (
-        cross_entropy_loss as flash_attn_cross_entropy_loss,
-    )
+def patch_llama_cross_entropy():
+    from flash_attn.losses.cross_entropy import CrossEntropyLoss

-    def fa2_fixed_cross_entropy(
-        source,
-        target,
-        num_items_in_batch: int = None,
-        ignore_index: int = -100,
-        **kwargs,
-    ):  # pylint: disable=unused-argument
-        reduction = "sum" if num_items_in_batch is not None else "mean"
-        loss, _ = flash_attn_cross_entropy_loss(
-            source, target, ignore_index=ignore_index
-        )
-        if reduction == "sum":
-            loss = loss.sum() / num_items_in_batch
-        else:
-            loss = loss.sum() / (target != ignore_index).sum()
-        return loss
-
-    transformers.loss.loss_utils.fixed_cross_entropy = fa2_fixed_cross_entropy
+    LOG.info("patching with flash_attn.losses.cross_entropy")
+    transformers.models.llama.modeling_llama.CrossEntropyLoss = partial(
+        CrossEntropyLoss, inplace_backward=True
+    )


 def patch_llama_rms_norm():
@@ -165,7 +147,7 @@ def replace_llama_attn_with_flash_attn(

    # skip only if explicitly disabled
    if cross_entropy:
-        patch_fa_llama_cross_entropy()
+        patch_llama_cross_entropy()

    # skip only if explicitly disabled
    if rms_norm:
--- a/src/axolotl/monkeypatch/multipack.py
+++ b/src/axolotl/monkeypatch/multipack.py
@@ -20,6 +20,7 @@ SUPPORTED_MULTIPACK_MODEL_TYPES = [
    "falcon",
    "phi",
    "phi3",
+    "phimoe",
    "gemma",
    "gemma2",
    "gemmoe",
--- a/src/axolotl/monkeypatch/relora.py
+++ b/src/axolotl/monkeypatch/relora.py
@@ -46,10 +46,9 @@ def reset_optimizer(
    *,
    reset_params: List[str],  # where str is the key to a torch.nn.Parameter
    optimizer_state_keys: List[str],
-    optimizer_magnitude_pruning: float = 0.9,
+    prune_ratio: float = 0.9,
 ):
-    # pylint:disable=unused-argument
-    pruning_fn = partial(magnitude_pruning_, prune_ratio=optimizer_magnitude_pruning)
+    pruning_fn = partial(magnitude_pruning_, prune_ratio=prune_ratio)
    n_zeros = 0
    n_total = 0

@@ -57,22 +56,16 @@ def reset_optimizer(
    if isinstance(optimizer, ZeroRedundancyOptimizer):
        optimizer_state = optimizer.optim.state

-    for group in optimizer.param_groups:
-        for param in group["params"]:
-            state = optimizer_state[param]
-            for key, value in state.items():
-                if key not in optimizer_state_keys:
-                    continue
-                if torch.is_tensor(value):
-                    try:
-                        pruning_fn(value)
-                        n_total += value.numel()
-                        n_zeros += torch.sum(value == 0).item()
-                    except RuntimeError as exc:
-                        if "quantile() input tensor is too large" in str(exc):
-                            pass
-                        else:
-                            raise exc
+    for param in reset_params:
+        param_state = optimizer_state[param]
+        if len(param_state) == 0:  # no state for this param, happens for ZeRo optimizer
+            continue
+        for key in optimizer_state_keys:
+            pruning_fn(
+                param_state[key]
+            )  # pruning fn has to be inplace to keep the same keys in the dict
+            n_total += param_state[key].numel()
+            n_zeros += torch.sum(param_state[key] == 0).item()

    _zeroed = n_zeros / (1e-7 + n_total) * 100
    LOG.info(f"Percent of optimizer states zeroed: {_zeroed:.2f}")
@@ -136,9 +129,6 @@ class ReLoRACallback(TrainerCallback):

            if "adam" in args.optim.lower():
                optimizer_state_keys = ["exp_avg", "exp_avg_sq"]
-                if "8bit" in args.optim.lower():
-                    optimizer_state_keys.append("state1")
-                    optimizer_state_keys.append("state2")
            else:
                raise ValueError(f"Optimizer {args.optim} not supported with ReLoRA")

@@ -170,7 +160,7 @@ class ReLoRACallback(TrainerCallback):
                    optimizer,
                    reset_params=lora_params,
                    optimizer_state_keys=optimizer_state_keys,
-                    optimizer_magnitude_pruning=args.relora_prune_ratio,
+                    prune_ratio=args.relora_prune_ratio,
                )

            if self.quantized:
--- a/src/axolotl/monkeypatch/trainer_fsdp_grad_accum.py
+++ b/src/axolotl/monkeypatch/trainer_fsdp_grad_accum.py
@@ -0,0 +1,83 @@
+"""
+fix for FSDP gradient accumulation
+see https://github.com/huggingface/transformers/pull/34645
+"""
+import inspect
+
+from accelerate.logging import get_logger
+from transformers.trainer import Trainer
+
+from axolotl.monkeypatch.unsloth_ import detab_code
+
+LOG = get_logger("axolotl.monkeypatch.trainer_fsdp_grad_accumulation")
+
+ORIGINAL_CONTEXT_CODE = """
+                context = (
+                    functools.partial(self.accelerator.no_sync, model=model)
+                    if i == len(batch_samples) - 1
+                    else contextlib.nullcontext
+                )
+"""
+
+PATCHED_CONTEXT_CODE = """
+                context = (
+                    functools.partial(self.accelerator.no_sync, model=model)
+                    if i != len(batch_samples) - 1
+                    else contextlib.nullcontext
+                )
+"""
+
+
+def get_training_loop_code() -> str:
+    training_loop = inspect.getsource(
+        Trainer._inner_training_loop  # pylint: disable=protected-access
+    )
+    return training_loop
+
+
+def check_training_loop_is_patchable() -> bool:
+    train_loop = get_training_loop_code()
+    train_loop, _ = detab_code(train_loop)
+    return ORIGINAL_CONTEXT_CODE in train_loop
+
+
+def patch_training_loop_for_fsdp_grad_accum():
+    """
+    monkeypatch for fixing the training loop for FSDP gradient accumulation
+    """
+
+    train_loop = get_training_loop_code()
+    Trainer._original_inner_training_loop = (  # pylint: disable=protected-access
+        train_loop
+    )
+    train_loop, _ = detab_code(train_loop)
+    assert (
+        ORIGINAL_CONTEXT_CODE in train_loop
+    ), "Original _inner_training_loop code not found"
+
+    train_loop = train_loop.replace(ORIGINAL_CONTEXT_CODE, PATCHED_CONTEXT_CODE)
+    train_loop = train_loop.replace(
+        "def _inner_training_loop(",
+        "def _fixed_inner_training_loop(",
+        1,
+    )
+
+    # load imports necessary
+    import transformers.trainer
+
+    items_to_import = []
+    for item in dir(transformers.trainer):
+        if item in train_loop:
+            items_to_import.append(item)
+
+    exec(  # pylint: disable=exec-used  # nosec B102
+        "from transformers.trainer import ("
+        + ", ".join(x for x in items_to_import)
+        + ")",
+        globals(),
+    )
+    exec(train_loop, globals())  # pylint: disable=exec-used  # nosec B102
+    LOG.info("patching _inner_training_loop", main_process_only=True)
+    Trainer._inner_training_loop = (  # pylint: disable=protected-access
+        _fixed_inner_training_loop  # pylint: disable=undefined-variable  # noqa: F821
+    )
--- a/src/axolotl/monkeypatch/unsloth_.py
+++ b/src/axolotl/monkeypatch/unsloth_.py
@@ -188,7 +188,7 @@ def integrate_lora_mlp_patch(peft_model: PeftModelForCausalLM):
            for module in layer_modules
        )
        mlp_not_dora = all(
-            len(getattr(module, "lora_magnitude_vector", []) or []) == 0
+            getattr(module, "lora_magnitude_vector", None) is None
            for module in layer_modules
        )

@@ -213,7 +213,7 @@ def integrate_lora_patch(peft_model: PeftModelForCausalLM, cfg):
                for module in layer_modules
            )
            qkv_not_dora = all(
-                len(getattr(module, "lora_magnitude_vector", []) or []) == 0
+                getattr(module, "lora_magnitude_vector", None) is None
                for module in layer_modules
            )

@@ -232,7 +232,7 @@ def integrate_lora_patch(peft_model: PeftModelForCausalLM, cfg):
                for module in layer_modules
            )
            o_not_dora = all(
-                len(getattr(module, "lora_magnitude_vector", []) or []) == 0
+                getattr(module, "lora_magnitude_vector", None) is None
                for module in layer_modules
            )

--- a/src/axolotl/train.py
+++ b/src/axolotl/train.py
@@ -259,31 +259,11 @@ def train(
        model.save_pretrained(cfg.output_dir, safe_serialization=safe_serialization)

    if not cfg.hub_model_id:
-        from huggingface_hub import HfApi
-        from huggingface_hub.utils import RepositoryNotFoundError
-
        try:
-            # Check to make sure the base model is from HuggingFace not a local directory
-            hf_api = HfApi()
-            hf_api.model_info(cfg.base_model)
-
-            model_card_kwarg = {
-                "model_name": cfg.output_dir.lstrip("./")
-                .encode("utf-8")
-                .decode("utf-8")
-            }
-            if cfg.datasets is not None:
-                if cfg.rl is not None or cfg.reward_model:
-                    model_card_kwarg["dataset_name"] = [
-                        d["path"] for d in cfg.datasets if not Path(d["path"]).is_dir()
-                    ]
-                else:
-                    model_card_kwarg["dataset_tags"] = [
-                        d["path"] for d in cfg.datasets if not Path(d["path"]).is_dir()
-                    ]
-
-            trainer.create_model_card(**model_card_kwarg)
-        except (AttributeError, UnicodeDecodeError, RepositoryNotFoundError):
+            trainer.create_model_card(
+                model_name=cfg.output_dir.lstrip("./").encode("utf-8").decode("utf-8")
+            )
+        except (AttributeError, UnicodeDecodeError):
            pass
    elif cfg.hub_model_id:
        # defensively push to the hub to ensure the model card is updated
--- a/src/axolotl/utils/init.py
+++ b/src/axolotl/utils/init.py
@@ -1,11 +1,7 @@
 """
 Basic utils for Axolotl
 """
-
 import importlib.util
-import re
-
-import torch


 def is_mlflow_available():
@@ -14,23 +10,3 @@ def is_mlflow_available():

 def is_comet_available():
    return importlib.util.find_spec("comet_ml") is not None
-
-
-# pylint: disable=duplicate-code
-def get_pytorch_version() -> tuple[int, int, int]:
-    """
-    Get Pytorch version as a tuple of (major, minor, patch).
-    """
-    torch_version = torch.__version__
-    version_match = re.match(r"^(\d+)\.(\d+)(?:\.(\d+))?", torch_version)
-
-    if not version_match:
-        raise ValueError("Invalid version format")
-
-    major, minor, patch = version_match.groups()
-    major, minor = int(major), int(minor)
-    patch = int(patch) if patch is not None else 0  # Default patch to 0 if not present
-    return major, minor, patch
-
-
-# pylint: enable=duplicate-code
--- a/src/axolotl/utils/bench.py
+++ b/src/axolotl/utils/bench.py
@@ -1,23 +1,9 @@
 """Benchmarking and measurement utilities"""
 import functools

+import pynvml
 import torch
-from transformers.utils.import_utils import is_torch_npu_available
-
-from axolotl.utils.distributed import get_device_type
-
-try:
-    from pynvml import (
-        NVMLError,
-        nvmlDeviceGetHandleByIndex,
-        nvmlDeviceGetMemoryInfo,
-        nvmlInit,
-    )
-except ImportError:
-    NVMLError = None
-    nvmlDeviceGetHandleByIndex = None
-    nvmlDeviceGetMemoryInfo = None
-    nvmlInit = None
+from pynvml.nvml import NVMLError


 def check_cuda_device(default_value):
@@ -67,35 +53,24 @@ def mps_memory_usage_all():
    return usage, reserved - usage, 0


-def npu_memory_usage_all(device=0):
-    usage = torch.npu.memory_allocated(device) / 1024.0**3
-    reserved = torch.npu.memory_reserved(device) / 1024.0**3
-    return usage, reserved - usage, 0
-
-
@check_cuda_device(0.0)
 def gpu_memory_usage_smi(device=0):
    if isinstance(device, torch.device):
        device = device.index
    if isinstance(device, str) and device.startswith("cuda:"):
        device = int(device[5:])
-    if not nvmlInit:
-        return 0.0
    try:
-        nvmlInit()
-        handle = nvmlDeviceGetHandleByIndex(device)
-        info = nvmlDeviceGetMemoryInfo(handle)
+        pynvml.nvmlInit()
+        handle = pynvml.nvmlDeviceGetHandleByIndex(device)
+        info = pynvml.nvmlDeviceGetMemoryInfo(handle)
        return info.used / 1024.0**3
    except NVMLError:
        return 0.0


 def log_gpu_memory_usage(log, msg, device):
-    cur_device = get_device_type()
    if torch.backends.mps.is_available():
        usage, cache, misc = mps_memory_usage_all()
-    elif "npu" in str(cur_device) and is_torch_npu_available():
-        usage, cache, misc = npu_memory_usage_all(device)
    else:
        usage, cache, misc = gpu_memory_usage_all(device)
    extras = []
@@ -104,7 +79,6 @@ def log_gpu_memory_usage(log, msg, device):
    if misc > 0:
        extras.append(f"+{misc:.03f}GB misc")
    log.info(
-        f"{str(cur_device)} memory usage {msg}: {usage:.03f}GB ({', '.join(extras)})",
-        stacklevel=2,
+        f"GPU memory usage {msg}: {usage:.03f}GB ({', '.join(extras)})", stacklevel=2
    )
    return usage, cache, misc
--- a/src/axolotl/utils/callbacks/init.py
+++ b/src/axolotl/utils/callbacks/init.py
@@ -28,7 +28,6 @@ from transformers import (
    TrainingArguments,
 )
 from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR, IntervalStrategy
-from trl.models import unwrap_model_for_generation

 from axolotl.utils import is_comet_available, is_mlflow_available
 from axolotl.utils.bench import log_gpu_memory_usage
@@ -47,7 +46,6 @@ from axolotl.utils.distributed import (
 if TYPE_CHECKING:
    from axolotl.core.trainer_builder import AxolotlTrainingArguments

-
 IGNORE_INDEX = -100
 LOG = logging.getLogger("axolotl.callbacks")

@@ -66,10 +64,7 @@ class EvalFirstStepCallback(
        control: TrainerControl,
        **kwargs,
    ):
-        if (
-            args.evaluation_strategy == IntervalStrategy.STEPS
-            and state.global_step == 1
-        ):
+        if args.eval_strategy == IntervalStrategy.STEPS and state.global_step == 1:
            control.should_evaluate = True
        return control

@@ -380,10 +375,7 @@ def causal_lm_bench_eval_callback_factory(trainer: Trainer, tokenizer):
            for metric in self.cfg.eval_causal_lm_metrics:
                if metric == "perplexity":
                    max_seq_len = self.cfg.eval_max_new_tokens
-                    metrics[metric] = Perplexity(
-                        tokenizer=tokenizer,
-                        max_seq_len=max_seq_len,
-                    )
+                    metrics[metric] = Perplexity(trainer.model, tokenizer, max_seq_len)
                else:
                    try:
                        metrics[metric] = evaluate.load(metric)
@@ -400,11 +392,8 @@ def causal_lm_bench_eval_callback_factory(trainer: Trainer, tokenizer):
            eval_dataloader,
            **kwargs,  # pylint: disable=unused-argument
        ):
-            trainer.model_wrapped.eval()
-
-            device = torch.device(
-                self.cfg.device
-            )  # Use this instead of trainer.model_wrapped.device as it may return cpu if fsdp offloaded
+            trainer.model.eval()
+            device = torch.device(self.cfg.device)

            # pylint: disable=duplicate-code
            generation_config = GenerationConfig(
@@ -441,10 +430,6 @@ def causal_lm_bench_eval_callback_factory(trainer: Trainer, tokenizer):
                        for k in metric._feature_names()  # pylint: disable=protected-access
                        if k in kwargs
                    }
-
-                    if isinstance(metric, Perplexity):
-                        metric_kwargs["model"] = trainer.model_wrapped
-
                    metric_score = metric.compute(**metric_kwargs)
                    return (
                        metric_score["score"]
@@ -480,97 +465,89 @@ def causal_lm_bench_eval_callback_factory(trainer: Trainer, tokenizer):
            def predict_with_generate():
                eval_src, eval_pred, eval_ref = [], [], []

-                with unwrap_model_for_generation(
-                    trainer.model_wrapped, trainer.accelerator
-                ) as unwrapped_model:
-                    for batch in tqdm(eval_dataloader, disable=not is_main_process()):
-                        batch_labels = batch["labels"].to(device)
-                        batch_input_ids = batch["input_ids"].to(device)
+                for batch in tqdm(eval_dataloader):
+                    batch_labels = batch["labels"].to(device)
+                    batch_input_ids = batch["input_ids"].to(device)

-                        if "position_ids" in batch:
-                            batch_pos_ids = batch["position_ids"].tolist()
+                    if "position_ids" in batch:
+                        batch_pos_ids = batch["position_ids"].tolist()
+                    else:
+                        batch_pos_ids = [None] * len(batch["input_ids"])
+
+                    prompt_token_ids_list = []
+                    completion_token_ids_list = []
+
+                    for input_ids_all, labels_all, pos_ids in zip(
+                        batch_input_ids,
+                        batch_labels,
+                        batch_pos_ids,
+                    ):
+                        if pos_ids is None:
+                            pos_ranges = [(0, len(input_ids_all) - 1)]
                        else:
-                            batch_pos_ids = [None] * len(batch["input_ids"])
+                            pos_ranges = find_ranges(pos_ids)

-                        prompt_token_ids_list = []
-                        completion_token_ids_list = []
+                        for pos_range in pos_ranges:
+                            start, end = pos_range
+                            if start == end:
+                                continue

-                        for input_ids_all, labels_all, pos_ids in zip(
-                            batch_input_ids,
-                            batch_labels,
-                            batch_pos_ids,
-                        ):
-                            if pos_ids is None:
-                                pos_ranges = [(0, len(input_ids_all) - 1)]
-                            else:
-                                pos_ranges = find_ranges(pos_ids)
+                            input_ids = input_ids_all[start : end + 1]
+                            labels = labels_all[start : end + 1]

-                            for pos_range in pos_ranges:
-                                start, end = pos_range
-                                if start == end:
-                                    continue
-
-                                input_ids = input_ids_all[start : end + 1]
-                                labels = labels_all[start : end + 1]
-
-                                tokens_without_loss = labels == IGNORE_INDEX
-                                tokens_with_loss = labels != IGNORE_INDEX
-                                tokens_exclude_padding = (
-                                    input_ids != tokenizer.pad_token_id
-                                )
-                                prompt_token_includes = (
-                                    tokens_without_loss & tokens_exclude_padding
-                                )
-
-                                prompt_token_ids = input_ids[prompt_token_includes]
-                                prompt_token_ids_list.append(prompt_token_ids)
-
-                                completion_token_ids = input_ids[tokens_with_loss]
-                                completion_token_ids_list.append(completion_token_ids)
-
-                        prompt_texts = tokenizer.batch_decode(
-                            prompt_token_ids_list, skip_special_tokens=True
-                        )
-                        completion_texts = tokenizer.batch_decode(
-                            completion_token_ids_list, skip_special_tokens=True
-                        )
-
-                        with torch.no_grad():
-                            prompt_encoding = tokenizer(
-                                prompt_texts, padding=True, return_tensors="pt"
-                            ).to(device)
-
-                            predictions = unwrapped_model.generate(
-                                **prompt_encoding, generation_config=generation_config
+                            tokens_without_loss = labels == IGNORE_INDEX
+                            tokens_with_loss = labels != IGNORE_INDEX
+                            tokens_exclude_padding = input_ids != tokenizer.pad_token_id
+                            prompt_token_includes = (
+                                tokens_without_loss & tokens_exclude_padding
                            )

-                            del prompt_encoding
+                            prompt_token_ids = input_ids[prompt_token_includes]
+                            prompt_token_ids_list.append(prompt_token_ids)

-                        prediction_all_tokens = predictions["sequences"].cpu().tolist()
-                        prediction_without_prompt_tokens_list = []
-                        for prompt_token_ids, prediction_tokens in zip(
-                            prompt_token_ids_list, prediction_all_tokens
-                        ):
-                            prediction_without_prompt_tokens = prediction_tokens[
-                                len(prompt_token_ids) :
-                            ]
-                            prediction_without_prompt_tokens_list.append(
-                                prediction_without_prompt_tokens
-                            )
+                            completion_token_ids = input_ids[tokens_with_loss]
+                            completion_token_ids_list.append(completion_token_ids)

-                        predicted_texts = tokenizer.batch_decode(
-                            prediction_without_prompt_tokens_list,
-                            skip_special_tokens=True,
+                    prompt_texts = tokenizer.batch_decode(
+                        prompt_token_ids_list, skip_special_tokens=True
+                    )
+                    completion_texts = tokenizer.batch_decode(
+                        completion_token_ids_list, skip_special_tokens=True
+                    )
+
+                    with torch.no_grad():
+                        prompt_encoding = tokenizer(
+                            prompt_texts, padding=True, return_tensors="pt"
+                        ).to(self.cfg.device)
+                        predictions = trainer.model.generate(
+                            **prompt_encoding, generation_config=generation_config
                        )

-                        eval_src.extend(prompt_texts)
-                        eval_pred.extend(predicted_texts)
-                        eval_ref.extend(completion_texts)
+                    prediction_all_tokens = predictions["sequences"].cpu().tolist()
+                    prediction_without_prompt_tokens_list = []
+                    for prompt_token_ids, prediction_tokens in zip(
+                        prompt_token_ids_list, prediction_all_tokens
+                    ):
+                        prediction_without_prompt_tokens = prediction_tokens[
+                            len(prompt_token_ids) :
+                        ]
+                        prediction_without_prompt_tokens_list.append(
+                            prediction_without_prompt_tokens
+                        )
+
+                    predicted_texts = tokenizer.batch_decode(
+                        prediction_without_prompt_tokens_list, skip_special_tokens=True
+                    )
+
+                    eval_src.extend(prompt_texts)
+                    eval_pred.extend(predicted_texts)
+                    eval_ref.extend(completion_texts)

                return eval_src, eval_pred, eval_ref

-            eval_preds = predict_with_generate()
-            trainer.log(evaluate_preds(*eval_preds))
+            if is_main_process():
+                eval_preds = predict_with_generate()
+                trainer.log(evaluate_preds(*eval_preds))

            return control

--- a/src/axolotl/utils/callbacks/perplexity.py
+++ b/src/axolotl/utils/callbacks/perplexity.py
@@ -8,8 +8,6 @@ from transformers.modeling_outputs import CausalLMOutput
 from transformers.modeling_utils import PreTrainedModel
 from transformers.tokenization_utils import PreTrainedTokenizer

-from axolotl.utils.distributed import is_main_process
-

 class Perplexity:
    """
@@ -19,13 +17,16 @@ class Perplexity:

    def __init__(
        self,
+        model: PreTrainedModel,
        tokenizer: PreTrainedTokenizer,
        max_seq_len: int,
        stride: int = 512,
    ) -> None:
        self.max_seq_len = max_seq_len
        self.stride = stride
+        self.model = model
        self.tokenizer = tokenizer
+        self.device = model.device
        self.name = "perplexity"

    def _feature_names(self) -> List[str]:
@@ -33,7 +34,6 @@ class Perplexity:

    def compute(
        self,
-        model: PreTrainedModel,
        references: Optional[List[str]] = None,
    ) -> Dict[str, float]:
        """
@@ -41,21 +41,17 @@ class Perplexity:
        """
        assert references is not None, "Missing parameter: references"

-        model.eval()
-
        references_tokenized = self.tokenizer(
            references, return_tensors="pt", padding=True, truncation=True
        )
        input_ids: Tensor = references_tokenized["input_ids"]  # type: ignore
-        input_ids = input_ids.to(model.device)
+        input_ids = input_ids.to(self.device)

        sequence_length = input_ids.size(1)

        losses = []
        prev_end_loc = 0
-        for begin_loc in tqdm(
-            range(0, sequence_length, self.stride), disable=not is_main_process()
-        ):
+        for begin_loc in tqdm(range(0, sequence_length, self.stride)):
            end_loc = min(begin_loc + self.max_seq_len, sequence_length)
            trg_len = end_loc - prev_end_loc
            input_ids_slice = input_ids[:, begin_loc:end_loc]
@@ -63,7 +59,7 @@ class Perplexity:
            labels_slice[:, :-trg_len] = -100

            with torch.no_grad():
-                outputs: CausalLMOutput = model(
+                outputs: CausalLMOutput = self.model(
                    input_ids=input_ids_slice, labels=labels_slice
                )

--- a/src/axolotl/utils/chat_templates.py
+++ b/src/axolotl/utils/chat_templates.py
--- a/src/axolotl/utils/collators/mm_chat.py
+++ b/src/axolotl/utils/collators/mm_chat.py
@@ -1,10 +1,8 @@
 """
 Collators for multi-modal chat messages and packing
 """
-
-from copy import deepcopy
 from dataclasses import dataclass
-from typing import Any, Optional, Union
+from typing import Any, Dict, List, Optional, Union

 from PIL import Image
 from transformers import PreTrainedTokenizerBase, ProcessorMixin
@@ -22,7 +20,6 @@ class MultiModalChatDataCollator(DataCollatorMixin):
    processor: ProcessorMixin
    return_tensors: str = "pt"
    chat_template: Optional[str] = None
-    chat_template_type: Optional[str] = None
    packing: bool = False
    max_images: int = -1
    padding: Union[bool, str, PaddingStrategy] = True
@@ -33,190 +30,38 @@ class MultiModalChatDataCollator(DataCollatorMixin):
            raise ValueError("Packing is currently not supported.")

    def torch_call(
-        self, examples: list[Union[list[int], Any, dict[str, Any]]]
-    ) -> dict[str, Any]:
+        self, examples: List[Union[List[int], Any, Dict[str, Any]]]
+    ) -> Dict[str, Any]:
        # Handle dict or lists with proper padding and conversion to tensor.
+
        return self.__class__.process_rows(
-            examples,
-            self.processor,
-            self.chat_template,
-            self.max_images,
-            chat_template_type=self.chat_template_type,
+            examples, self.processor, self.chat_template, self.max_images
        )

    @staticmethod
-    def preprocess(examples: list[dict]) -> list[dict]:
-        """
-        Preprocess conversation examples to ensure consistent format.
-        Converts different conversation formats to OpenAI format with 'messages'.
-        Supports two formats:
-        1. OpenAI format with 'messages'
-        2. Legacy format with 'conversations'
-
-        Args:
-        examples: list of conversation dictionaries
-        Returns:
-            dict in OpenAI format with 'messages' key
-
-        Raises:
-            ValueError: If the conversation format is not supported
-        """
-        role_mapping = {
-            "human": "user",
-            "gpt": "assistant",
-        }
-
-        def normalize_role(role: str) -> str:
-            """Normalize role names to OpenAI format. Default to original role if not found."""
-            return role_mapping.get(role, role)
-
-        def convert_legacy_format(example: dict) -> dict:
-            """Convert legacy 'conversations' format to OpenAI 'messages' format."""
-            messages = [
-                {
-                    "role": normalize_role(convo["from"]),
-                    "content": convo["value"],
-                }
-                for convo in example["conversations"]
-            ]
-
-            # Create new dict without 'conversations' key
-            result = deepcopy(example)
-            result.pop("conversations")
-            return {"messages": messages, **result}
-
-        processed_examples = []
-        for example in examples:
-            # OpenAI format
-            if "messages" in example:
-                processed_examples.append(example)
-
-            # Legacy format
-            elif "conversations" in example:
-                processed_examples.append(convert_legacy_format(example))
-
-            else:
-                raise ValueError(
-                    "Only `messages` and `conversations` message keys are currently supported."
-                )
-
-        return processed_examples
-
-    @staticmethod
-    def process_images(examples, max_images):
-        """
-        Process images from examples, ensuring consistency in image presence and applying max_images limit.
-
-        Args:
-            examples: List of dictionaries that may contain 'images' key
-            max_images: Maximum number of images to keep per example (0 means no limit)
-
-        Returns:
-            Either None (if no images) or List[Image objects] (if all examples have images)
-
-        Raises:
-            ValueError: If there's a mix of None and non-None images
-        """
-
-        def get_image(example):
-            if "images" not in example:
-                return None
-            images = example["images"]
-            if isinstance(images, str):
-                return Image.open(images)
-            return images
-
-        images = [get_image(example) for example in examples]
-
-        # Count None and non-None images
-        none_count = sum(1 for img in images if img is None)
-
-        # All images are None
-        if none_count == len(images):
-            return None
-
-        # Mix of None and non-None images
-        if none_count > 0:
-            raise ValueError(
-                "All images should be either None or not None. "
-                "Please provide images for all examples or None."
-            )
-
-        # Apply max_images limit if specified
-        if max_images > 0:
-            images = [
-                (
-                    img_batch[:max_images]
-                    if isinstance(img_batch, (list, tuple))
-                    else img_batch
-                )
-                for img_batch in images
-            ]
-
-        return images
-
-    @staticmethod
-    def pixtral_chat_conversion(messages):
-        is_single_message = not isinstance(messages, list)
-        if is_single_message:
-            messages = [messages]
-
-        for i, message in enumerate(messages):
-            if message["role"] == "user":
-                for j, content in enumerate(message["content"]):
-                    if "type" in content and content["type"] == "text":
-                        messages[i]["content"][j] = {
-                            "type": "text",
-                            "content": content["text"],
-                        }
-
-            if message["role"] == "assistant":
-                messages[i]["content"] = message["content"][0]["text"]
-
-        if is_single_message:
-            return messages[0]
-        return messages
-
-    @staticmethod
-    def process_rows(
-        examples,
-        processor,
-        chat_template,
-        max_images,
-        length_only=False,
-        chat_template_type=None,
-    ):
+    def process_rows(examples, processor, chat_template, max_images, length_only=False):
        # HINT: use `_torch_collate_batch` to stack and pad tensors
        # see also DataCollatorWithFlattening and DefaultDataCollator

        # *** This is COPIED from the trl example sft_vlm.py code ***
        # use this as a starting point

-        # Preprocess the examples
-        examples = __class__.preprocess(examples)
-
        # Get the texts and images, and apply the chat template
-        if chat_template_type == "pixtral":
-            texts = [
-                processor.apply_chat_template(
-                    __class__.pixtral_chat_conversion(example["messages"]),
-                    chat_template=chat_template,
-                    tokenize=False,
-                )
-                for example in examples
-            ]
-        else:
-            texts = [
-                processor.apply_chat_template(
-                    example["messages"], chat_template=chat_template, tokenize=False
-                )
-                for example in examples
-            ]
+        texts = [
+            processor.apply_chat_template(
+                example["messages"], chat_template=chat_template, tokenize=False
+            )
+            for example in examples
+        ]
+        images = [
+            Image.open(example["images"])
+            if isinstance(example["images"], str)
+            else example["images"]
+            for example in examples
+        ]

-        images = __class__.process_images(examples, max_images=max_images)
-        if chat_template_type == "llava":
-            # LLava1.5 does not support multiple images
-            images = [image[0] for image in images]
+        if max_images > 0:
+            images = [img_batch[:max_images] for img_batch in images]

        # Tokenize the texts and process the images
        batch = processor(text=texts, images=images, return_tensors="pt", padding=True)
@@ -225,12 +70,9 @@ class MultiModalChatDataCollator(DataCollatorMixin):
        labels = batch["input_ids"].clone()
        labels[labels == processor.tokenizer.pad_token_id] = -100  #
        # Ignore the image token index in the loss computation (model specific)
-        if chat_template_type == "qwen2_vl":
-            image_token_id = processor.tokenizer.convert_tokens_to_ids("<|image_pad|>")
-        else:
-            image_token_id = processor.tokenizer.convert_tokens_to_ids(
-                processor.image_token
-            )
+        image_token_id = processor.tokenizer.convert_tokens_to_ids(
+            processor.image_token
+        )
        labels[labels == image_token_id] = -100
        batch["labels"] = labels

--- a/src/axolotl/utils/config/init.py
+++ b/src/axolotl/utils/config/init.py
@@ -5,9 +5,7 @@ from typing import Optional

 import torch
 from transformers.utils import is_torch_bf16_gpu_available
-from transformers.utils.import_utils import is_torch_npu_available

-from axolotl.integrations.base import PluginManager
 from axolotl.integrations.config import merge_input_args
 from axolotl.utils.bench import log_gpu_memory_usage
 from axolotl.utils.config.models.input.v0_4_1 import (
@@ -31,10 +29,7 @@ def choose_device(cfg):
            if torch.backends.mps.is_available():
                return "mps"

-            if is_torch_npu_available():
-                return f"npu:{cfg.local_rank}"
-
-            raise SystemError("No CUDA/mps/npu device found")
+            raise SystemError("No CUDA/mps device found")
        except Exception:  # pylint: disable=broad-exception-caught
            return "cpu"

@@ -44,8 +39,6 @@ def choose_device(cfg):
    else:
        if cfg.device.startswith("cuda"):
            cfg.device_map = {"": torch.cuda.current_device()}
-        elif cfg.device.startswith("npu"):
-            cfg.device_map = {"npu": torch.npu.current_device()}
        else:
            cfg.device_map = {"": cfg.device}

@@ -132,7 +125,7 @@ def normalize_config(cfg):

    cfg.is_multimodal = (
        hasattr(model_config, "model_type")
-        and model_config.model_type in ["llava", "mllama", "qwen2_vl", "qwen2_5_vl"]
+        and model_config.model_type in ["llava", "mllama"]
        or any(
            multimodal_name in cfg.base_model.lower()
            for multimodal_name in [
@@ -145,12 +138,7 @@ def normalize_config(cfg):
        cfg.processor_config = (
            cfg.processor_config or cfg.base_model_config or cfg.base_model
        )
-
-        try:
-            model_config = model_config.text_config
-        except AttributeError:
-            # for qwen2_vl
-            model_config = model_config.get_text_config()
+        model_config = model_config.text_config

    cfg.model_config_type = model_config.model_type

@@ -235,11 +223,7 @@ def normalize_cfg_datasets(cfg):
                    cfg.datasets[idx].chat_template_jinja = cfg.chat_template_jinja


-def validate_config(
-    cfg: DictDefault,
-    capabilities: Optional[dict] = None,
-    env_capabilities: Optional[dict] = None,
-):
+def validate_config(cfg: DictDefault, capabilities: Optional[dict] = None):
    AxolotlConfigWCapabilities = AxolotlConfigWCapabilitiesBase
    AxolotlInputConfig = AxolotlInputConfigBase

@@ -249,35 +233,14 @@ def validate_config(
            AxolotlInputConfig,  # pylint: disable=invalid-name
        ) = merge_input_args()

-    if capabilities or env_capabilities:
-        if (capabilities and not env_capabilities) or (
-            env_capabilities and not capabilities
-        ):
-            raise ValueError(
-                "Both capabilities and env_capabilities must be provided or not provided."
-            )
-
+    if capabilities:
        return DictDefault(
            dict(
                AxolotlConfigWCapabilities(
-                    **cfg.to_dict(),
-                    capabilities=capabilities,
-                    env_capabilities=env_capabilities,
+                    **cfg.to_dict(), capabilities=capabilities
                ).model_dump(exclude_none=True)
            )
        )
-
    return DictDefault(
        dict(AxolotlInputConfig(**cfg.to_dict()).model_dump(exclude_none=True))
    )
-
-
-def prepare_plugins(cfg):
-    """
-    Prepare the plugins for the configuration
-    """
-
-    if cfg.get("plugins"):
-        plugin_manager = PluginManager.get_instance()
-        for plugin_name in cfg["plugins"]:
-            plugin_manager.register(plugin_name)
--- a/src/axolotl/utils/config/models/input/v0_4_1/init.py
+++ b/src/axolotl/utils/config/models/input/v0_4_1/init.py
@@ -7,9 +7,9 @@ Module for pydantic models for configuration
 import logging
 import os
 from enum import Enum
+from importlib.metadata import version
 from typing import Annotated, Any, Dict, List, Literal, Optional, Tuple, Union

-from packaging import version
 from pydantic import (
    BaseModel,
    Field,
@@ -20,9 +20,8 @@ from pydantic import (
 )
 from transformers import SchedulerType
 from transformers.training_args import OptimizerNames
-from transformers.utils.import_utils import is_torch_npu_available

-from axolotl.utils.config.models.internals import EnvCapabilities, GPUCapabilities
+from axolotl.utils.config.models.internals import GPUCapabilities

 LOG = logging.getLogger("axolotl.utils.config.models.input")

@@ -51,7 +50,6 @@ class ChatTemplate(str, Enum):
    cohere = "cohere"  # pylint: disable=invalid-name
    llama3 = "llama3"  # pylint: disable=invalid-name
    llama3_2_vision = "llama3_2_vision"  # pylint: disable=invalid-name
-    llava = "llava"  # pylint: disable=invalid-name
    phi_3 = "phi_3"  # pylint: disable=invalid-name
    phi_35 = "phi_35"  # pylint: disable=invalid-name
    deepseek_v2 = "deepseek_v2"  # pylint: disable=invalid-name
@@ -61,8 +59,6 @@ class ChatTemplate(str, Enum):
    tokenizer_default = "tokenizer_default"  # pylint: disable=invalid-name
    exaone = "exaone"  # pylint: disable=invalid-name
    metharme = "metharme"  # pylint: disable=invalid-name
-    pixtral = "pixtral"  # pylint: disable=invalid-name
-    qwen2_vl = "qwen2_vl"  # pylint: disable=invalid-name


 class DeprecatedParameters(BaseModel):
@@ -254,10 +250,8 @@ class KTODataset(BaseModel):
 class LoftQConfig(BaseModel):
    """LoftQ configuration subset"""

-    loftq_bits: int = Field(
-        default=4, json_schema_extra={"description": "Quantization bits for LoftQ"}
-    )
-    # loftq_iter: int = Field(default=1, json_schema_extra={"description": "Alternating iterations for LoftQ"})
+    loftq_bits: int = Field(default=4, metadata={"help": "Quantization bits for LoftQ"})
+    # loftq_iter: int = Field(default=1, metadata={"help": "Alternating iterations for LoftQ"})


 class PeftConfig(BaseModel):
@@ -300,8 +294,8 @@ class LoraConfig(BaseModel):

    qlora_sharded_model_loading: Optional[bool] = Field(
        default=False,
-        json_schema_extra={
-            "description": "load qlora model in sharded format for FSDP using answer.ai technique."
+        metadata={
+            "help": "load qlora model in sharded format for FSDP using answer.ai technique."
        },
    )
    lora_on_cpu: Optional[bool] = None
@@ -310,15 +304,13 @@ class LoraConfig(BaseModel):

    loraplus_lr_ratio: Optional[float] = Field(
        default=None,
-        json_schema_extra={
-            "description": "loraplus learning rate ratio lr_B / lr_A. Recommended value is 2^4."
+        metadata={
+            "help": "loraplus learning rate ratio lr_B / lr_A. Recommended value is 2^4."
        },
    )
    loraplus_lr_embedding: Optional[float] = Field(
        default=1e-6,
-        json_schema_extra={
-            "description": "loraplus learning rate for lora embedding layers."
-        },
+        metadata={"help": "loraplus learning rate for lora embedding layers."},
    )

    merge_lora: Optional[bool] = None
@@ -326,13 +318,11 @@ class LoraConfig(BaseModel):
    @model_validator(mode="before")
    @classmethod
    def validate_adapter(cls, data):
-        if (
-            not data.get("adapter")
-            and not data.get("inference")
-            and (data.get("load_in_8bit") or data.get("load_in_4bit"))
+        if not data.get("adapter") and (
+            data.get("load_in_8bit") or data.get("load_in_4bit")
        ):
            raise ValueError(
-                "load_in_8bit and load_in_4bit are not supported without setting an adapter for training."
+                "load_in_8bit and load_in_4bit are not supported without setting an adapter."
                "If you want to full finetune, please turn off load_in_8bit and load_in_4bit."
            )
        return data
@@ -390,10 +380,10 @@ class ModelInputConfig(BaseModel):
    tokenizer_use_fast: Optional[bool] = None
    tokenizer_legacy: Optional[bool] = None
    tokenizer_type: Optional[str] = Field(
-        default=None, json_schema_extra={"description": "transformers tokenizer class"}
+        default=None, metadata={"help": "transformers tokenizer class"}
    )
    processor_type: Optional[str] = Field(
-        default=None, json_schema_extra={"description": "transformers processor class"}
+        default=None, metadata={"help": "transformers processor class"}
    )
    trust_remote_code: Optional[bool] = None

@@ -415,18 +405,18 @@ class HyperparametersConfig(BaseModel):
    gradient_accumulation_steps: Optional[int] = Field(default=1)
    micro_batch_size: Optional[int] = Field(
        default=1,
-        json_schema_extra={"description": "per gpu micro batch size for training"},
+        metadata={"help": "per gpu micro batch size for training"},
    )
    batch_size: Optional[int] = Field(
        default=None,
-        json_schema_extra={
-            "description": "Total batch size, we do not recommended setting this manually"
+        metadata={
+            "help": "Total batch size, we do not recommended setting this manually"
        },
    )
    eval_batch_size: Optional[int] = Field(
        default=None,
-        json_schema_extra={
-            "description": "per gpu micro batch size for evals, defaults to value of micro_batch_size"
+        metadata={
+            "help": "per gpu micro batch size for evals, defaults to value of micro_batch_size"
        },
    )

@@ -436,8 +426,6 @@ class HyperparametersConfig(BaseModel):
    group_by_length: Optional[bool] = None

    learning_rate: Union[str, float]
-    embedding_lr: Optional[float] = None
-    embedding_lr_scale: Optional[float] = None
    weight_decay: Optional[float] = 0.0
    optimizer: Optional[
        Union[
@@ -453,13 +441,12 @@ class HyperparametersConfig(BaseModel):
        ]
    ] = OptimizerNames.ADAMW_HF.value
    optim_args: Optional[Union[str, Dict[str, Any]]] = Field(
-        default=None,
-        json_schema_extra={"description": "Optional arguments to supply to optimizer."},
+        default=None, metadata={"help": "Optional arguments to supply to optimizer."}
    )
    optim_target_modules: Optional[Union[List[str], Literal["all_linear"]]] = Field(
        default=None,
-        json_schema_extra={
-            "description": "The target modules to optimize, i.e. the module names that you would like to train."
+        metadata={
+            "help": "The target modules to optimize, i.e. the module names that you would like to train."
        },
    )
    torchdistx_path: Optional[str] = None
@@ -519,15 +506,15 @@ class LISAConfig(BaseModel):

    lisa_n_layers: Optional[int] = Field(
        default=None,
-        json_schema_extra={"description": "the number of activate layers in LISA"},
+        metadata={"help": "the number of activate layers in LISA"},
    )
    lisa_step_interval: Optional[int] = Field(
        default=None,
-        json_schema_extra={"description": "how often to switch layers in LISA"},
+        metadata={"help": "how often to switch layers in LISA"},
    )
    lisa_layers_attribute: Optional[str] = Field(
        default="model.layers",
-        json_schema_extra={"description": "path under the model to access the layers"},
+        metadata={"help": "path under the model to access the layers"},
    )


@@ -626,11 +613,9 @@ class AxolotlInputConfig(
    pretraining_dataset: Optional[  # type: ignore
        conlist(Union[PretrainingDataset, SFTDataset], min_length=1)
    ] = Field(
-        default=None,
-        json_schema_extra={"description": "streaming dataset to use for pretraining"},
+        default=None, metadata={"help": {"streaming dataset to use for pretraining"}}
    )
    dataset_processes: Optional[int] = Field(default=os.cpu_count())
-    dataset_exact_deduplication: Optional[bool] = None
    dataset_keep_in_memory: Optional[bool] = None
    dataloader_pin_memory: Optional[bool] = None
    dataloader_num_workers: Optional[int] = None
@@ -688,8 +673,7 @@ class AxolotlInputConfig(
    sequence_len: int = Field(default=512)
    min_sample_len: Optional[int] = None
    max_prompt_len: int = Field(
-        default=512,
-        json_schema_extra={"description": "maximum prompt length for RL training"},
+        default=512, metadata={"help": "maximum prompt length for RL training"}
    )
    sample_packing: Optional[bool] = None
    sample_packing_group_size: Optional[int] = 100_000
@@ -708,8 +692,8 @@ class AxolotlInputConfig(
    pretrain_multipack_buffer_size: Optional[int] = 10_000
    pretrain_multipack_attn: Optional[bool] = Field(
        default=True,
-        json_schema_extra={
-            "description": "whether to prevent cross attention for packed sequences during pretraining",
+        metadata={
+            "help": "whether to prevent cross attention for packed sequences during pretraining",
        },
    )

@@ -1323,7 +1307,6 @@ class AxolotlInputConfig(
            and data.get("gradient_checkpointing_kwargs", {})
            and data.get("gradient_checkpointing_kwargs", {}).get("use_reentrant")
            is False
-            and data.get("deepspeed", "") is not None
            and "zero3" in data.get("deepspeed", "")
        ):
            # may result in:
@@ -1435,6 +1418,21 @@ class AxolotlInputConfig(
                )
        return data

+    @model_validator(mode="before")
+    @classmethod
+    def check_unsloth_xformers_version(cls, data):
+        if (
+            data.get("unsloth_lora_mlp")
+            or data.get("unsloth_lora_qkv")
+            or data.get("unsloth_lora_o")
+        ):
+            xformers_version = version("xformers")
+            if xformers_version == "0.0.27":
+                raise ValueError(
+                    "xformers version 0.0.27 is not supported with unsloth. Please downgrade to 0.0.26.post1"
+                )
+        return data
+
    @model_validator(mode="before")
    @classmethod
    def check_torch_compile_deepspeed(cls, data):
@@ -1444,46 +1442,11 @@ class AxolotlInputConfig(
            )
        return data

-    @model_validator(mode="before")
-    @classmethod
-    def check_npu_config(cls, data):
-        if is_torch_npu_available():
-            # check attention config
-            attn_list = ["flash_attention", "sdp_attention", "s2_attention"]
-            for attn in attn_list:
-                if data.get(attn):
-                    raise NotImplementedError(
-                        f"{attn} is currently not supported in Ascend npu, please disable this configuration."
-                    )
-
-            # check quant config
-            if data.get("optimizer") is not None and "bit" in data.get("optimizer"):
-                optimizer = data.get("optimizer")
-                raise NotImplementedError(
-                    f"{optimizer} is currently not supported in Ascend npu, choose another one please."
-                )
-
-            quant_list = ["load_in_8bit", "load_in_4bit"]
-            for quant in quant_list:
-                if data.get(quant):
-                    raise NotImplementedError(
-                        f"Quantification is currently not supported in Ascend npu, please disable {quant}."
-                    )
-
-            # check dtype config
-            if data.get("tf32"):
-                raise NotImplementedError(
-                    "tf32 dtype is currently not supported in Ascend npu, please disable this configuration"
-                )
-
-        return data
-

 class AxolotlConfigWCapabilities(AxolotlInputConfig):
    """wrapper to valdiate gpu capabilities with the configured options"""

    capabilities: GPUCapabilities
-    env_capabilities: EnvCapabilities

    @model_validator(mode="after")
    def check_bf16(self):
@@ -1558,21 +1521,3 @@ class AxolotlConfigWCapabilities(AxolotlInputConfig):
                    "unsloth_lora_mlp, unsloth_lora_qkv, and unsloth_lora_o are not compatible with multi-GPU training."
                )
        return data
-
-    @model_validator(mode="before")
-    @classmethod
-    def check_adopt_torch_version(cls, data):
-        if (data.get("optimizer") is not None) and ("adopt" in data.get("optimizer")):
-            env_capabilities = data.get("env_capabilities", {})
-            torch_version = env_capabilities.get("torch_version")
-
-            if torch_version is None:
-                import torch
-
-                torch_version = str(torch.__version__).split("+", maxsplit=1)[0]
-
-            if version.parse(torch_version) < version.parse("2.5.1"):
-                raise ValueError(
-                    "ADOPT optimizer is incompatible with torch version < 2.5.1"
-                )
-        return data
--- a/src/axolotl/utils/config/models/internals/init.py
+++ b/src/axolotl/utils/config/models/internals/init.py
@@ -12,9 +12,3 @@ class GPUCapabilities(BaseModel):
    n_gpu: int = Field(default=1)
    n_node: int = Field(default=1)
    compute_capability: Optional[str] = Field(default=None)
-
-
-class EnvCapabilities(BaseModel):
-    """model to manage the environment capabilities statically"""
-
-    torch_version: Optional[str] = Field(default=None)
--- a/src/axolotl/utils/data/rl.py
+++ b/src/axolotl/utils/data/rl.py
@@ -13,7 +13,7 @@ from axolotl.common.const import DEFAULT_DATASET_PREPARED_PATH
 from axolotl.prompt_strategies.dpo import load as load_dpo
 from axolotl.prompt_strategies.kto import load as load_kto
 from axolotl.prompt_strategies.orpo import load as load_orpo
-from axolotl.utils.data.utils import deduplicate_and_log_datasets, md5
+from axolotl.utils.data.utils import md5
 from axolotl.utils.dict import DictDefault
 from axolotl.utils.distributed import is_main_process, zero_first
 from axolotl.utils.models import load_tokenizer
@@ -64,57 +64,15 @@ def map_dataset(cfg, data_set, ds_transform_fn, tokenizer):
            tokenizer = load_tokenizer(cfg)
        ds_transform_fn = partial(ds_transform_fn, tokenizer=tokenizer)

-    if isinstance(data_set, DatasetDict):
-        data_set = data_set["train"]
-
    data_set = data_set.map(
        ds_transform_fn,
        desc="Mapping RL Dataset",
    )
-
+    if isinstance(data_set, DatasetDict):
+        data_set = data_set["train"]
    return data_set


-def drop_long_rl_seq(
-    sample, rl, tokenizer, sequence_len  # pylint: disable=invalid-name
-):
-    if rl in ("dpo", "ipo", "orpo", "simpo"):
-        if not (
-            sample.get("prompt") and sample.get("chosen") and sample.get("rejected")
-        ):
-            raise ValueError(
-                "Prompt, chosen and rejected keys are required for DPO/ORPO datasets"
-            )
-
-        prompt = sample["prompt"]
-        chosen = sample["chosen"]
-        rejected = sample["rejected"]
-
-        len_prompt = len(tokenizer(prompt, add_special_tokens=False)["input_ids"])
-        len_chosen = len(tokenizer(chosen, add_special_tokens=False)["input_ids"])
-        len_rejected = len(tokenizer(rejected, add_special_tokens=False)["input_ids"])
-
-        return (len_prompt + len_chosen) <= sequence_len and (
-            len_prompt + len_rejected
-        ) <= sequence_len
-
-    if rl == "kto":
-        if not (sample.get("prompt") and sample.get("completion")):
-            raise ValueError("Prompt and completion keys are required for KTO datasets")
-
-        prompt = sample["prompt"]
-        completion = sample["completion"]
-
-        len_prompt = len(tokenizer(prompt, add_special_tokens=False)["input_ids"])
-        len_completion = len(
-            tokenizer(completion, add_special_tokens=False)["input_ids"]
-        )
-
-        return (len_prompt + len_completion) <= sequence_len
-
-    raise ValueError("Unknown RL type")
-
-
 def load_prepare_dpo_datasets(cfg):
    def load_split(dataset_cfgs, _cfg):
        split_datasets: List[Any] = []
@@ -136,7 +94,7 @@ def load_prepare_dpo_datasets(cfg):
                )
                split_datasets.insert(i, ds)

-        tokenizer = load_tokenizer(cfg)
+        tokenizer = None

        for i, data_set in enumerate(split_datasets):
            _type = dataset_cfgs[i]["type"]
@@ -163,28 +121,7 @@ def load_prepare_dpo_datasets(cfg):
                # "prompt", "chosen" and "rejected" already preprocessed
                split_datasets[i] = data_set

-            drop_long = partial(
-                drop_long_rl_seq,
-                rl=_cfg.rl,
-                tokenizer=tokenizer,
-                sequence_len=cfg.sequence_len,
-            )
-
-            prior_len = len(split_datasets[i])
-            split_datasets[i] = split_datasets[i].filter(
-                drop_long,
-                num_proc=cfg.dataset_processes,
-                load_from_cache_file=not cfg.is_preprocess,
-                desc="Dropping Long Sequences",
-            )
-            dropped = prior_len - len(split_datasets[i])
-            if dropped:
-                LOG.warning(f"Dropped {dropped} long samples from dataset index {i}")
-
-        combined_datasets = concatenate_datasets(split_datasets)
-        combined_datasets = combined_datasets.shuffle(seed=cfg.seed)
-
-        return combined_datasets
+        return concatenate_datasets(split_datasets)

    with zero_first(is_main_process()):
        train_is_preprocessed = False
@@ -208,9 +145,4 @@ def load_prepare_dpo_datasets(cfg):
        if eval_dataset and not eval_is_preprocessed:
            _save_preprocessed_ds(cfg, cfg.test_datasets, eval_dataset)

-    if cfg.dataset_exact_deduplication:
-        train_dataset, eval_dataset, _ = deduplicate_and_log_datasets(
-            train_dataset=train_dataset, eval_dataset=eval_dataset
-        )
-
    return train_dataset, eval_dataset
--- a/src/axolotl/utils/data/sft.py
+++ b/src/axolotl/utils/data/sft.py
@@ -44,7 +44,7 @@ from axolotl.prompters import (
    UnsupportedPrompter,
 )
 from axolotl.utils.data.pretraining import wrap_pretraining_dataset
-from axolotl.utils.data.utils import deduplicate_and_log_datasets, md5
+from axolotl.utils.data.utils import md5
 from axolotl.utils.dict import DictDefault
 from axolotl.utils.distributed import is_local_main_process, zero_first
 from axolotl.utils.trainer import (
@@ -136,9 +136,8 @@ def prepare_dataset(cfg, tokenizer, processor=None):
        # https://discuss.huggingface.co/t/how-to-use-huggingface-trainer-streaming-datasets-without-wrapping-it-with-torchdatas-iterablewrapper/25230
        train_dataset = train_dataset.with_format("torch")
        eval_dataset = None
-        if cfg.dataset_exact_deduplication:
-            LOG.info("Deduplication not available for pretrained datasets")
        return train_dataset, eval_dataset, cfg.max_steps, prompters
+
    if eval_dataset and cfg.sample_packing and cfg.eval_sample_packing is not False:
        total_eval_steps = calculate_total_num_steps(cfg, eval_dataset, update=False)
        if total_eval_steps == 0:
@@ -585,8 +584,7 @@ def load_prepare_datasets(
        )
        train_fingerprint = md5(to_hash_train)
        test_fingerprint = md5(to_hash_test)
-        if cfg.dataset_exact_deduplication:
-            _, _, dataset = deduplicate_and_log_datasets(dataset=dataset)
+
        dataset = dataset.train_test_split(
            test_size=val_set_size,
            shuffle=False,
@@ -598,17 +596,12 @@ def load_prepare_datasets(
        train_dataset = dataset["train"]
        eval_dataset = dataset["test"]
    elif split == "test":
-        if cfg.dataset_exact_deduplication:
-            _, eval_dataset, _ = deduplicate_and_log_datasets(eval_dataset=dataset)
-        else:
-            eval_dataset = dataset
        train_dataset = None
+        eval_dataset = dataset
    else:
-        if cfg.dataset_exact_deduplication:
-            train_dataset, _, _ = deduplicate_and_log_datasets(train_dataset=dataset)
-        else:
-            train_dataset = dataset
+        train_dataset = dataset
        eval_dataset = None
+
    return train_dataset, eval_dataset, prompters


--- a/src/axolotl/utils/data/utils.py
+++ b/src/axolotl/utils/data/utils.py
@@ -1,11 +1,6 @@
 """data handling helpers"""

 import hashlib
-import logging
-
-from datasets import Dataset
-
-LOG = logging.getLogger("axolotl")


 def md5(to_hash: str, encoding: str = "utf-8") -> str:
@@ -13,96 +8,3 @@ def md5(to_hash: str, encoding: str = "utf-8") -> str:
        return hashlib.md5(to_hash.encode(encoding), usedforsecurity=False).hexdigest()
    except TypeError:
        return hashlib.md5(to_hash.encode(encoding)).hexdigest()  # nosec
-
-
-def sha256(to_hash: str, encoding: str = "utf-8") -> str:
-    return hashlib.sha256(to_hash.encode(encoding)).hexdigest()
-
-
-def deduplicate_dataset(
-    dataset: Dataset, seen_hashes: dict[str, list[int]], other_dataset: Dataset = None
-) -> Dataset:
-    unique_indices = []
-
-    for idx, row in enumerate(dataset):
-        row_hash = sha256(str(row))  # Using SHA256 for collision resistance.
-        if row_hash not in seen_hashes:
-            seen_hashes[row_hash] = [idx]
-            unique_indices.append(idx)
-        else:
-            # Check for collision by looking up the original dataset indices
-            original_indices = seen_hashes[row_hash]
-            is_duplicate = False
-            for original_idx in original_indices:
-                if (
-                    not idx == original_idx
-                    and original_idx < len(dataset)
-                    and str(dataset[original_idx]) == str(row)
-                ):
-                    is_duplicate = True
-                    break
-                # Check in the other dataset if provided
-                if other_dataset is not None:
-                    if original_idx < len(other_dataset) and str(
-                        other_dataset[original_idx]
-                    ) == str(row):
-                        is_duplicate = True
-                        break
-            if not is_duplicate:
-                seen_hashes[row_hash].append(idx)
-                unique_indices.append(idx)
-                continue
-    return dataset.select(unique_indices)
-
-
-def deduplicate_and_log_datasets(
-    *,
-    train_dataset: Dataset = None,
-    eval_dataset: Dataset = None,
-    dataset: Dataset = None,
-) -> tuple[Dataset, Dataset, Dataset]:
-    """
-    Deduplicates train, eval, and an optional dataset if provided, logging original and new sizes.
-
-    Returns:
-        tuple: Deduplicated train, eval, and additional datasets.
-    """
-    seen_hashes: dict[str, list[int]] = {}
-
-    # Handle cases where datasets are None
-    if train_dataset is not None:
-        LOG.info(
-            f"Starting deduplication for train dataset. Original size: {len(train_dataset)}"
-        )
-        train_dataset = deduplicate_dataset(
-            dataset=train_dataset, seen_hashes=seen_hashes
-        )
-        LOG.info(
-            f"Deduplication complete for train dataset. New size: {len(train_dataset)}"
-        )
-    else:
-        LOG.info("Train dataset is None. Skipping deduplication.")
-
-    if eval_dataset is not None:
-        LOG.info(
-            f"Starting deduplication for eval dataset. Original size: {len(eval_dataset)}"
-        )
-        eval_dataset = deduplicate_dataset(
-            dataset=eval_dataset, seen_hashes=seen_hashes, other_dataset=train_dataset
-        )
-        LOG.info(
-            f"Deduplication complete for eval dataset. New size: {len(eval_dataset)}"
-        )
-    else:
-        LOG.info("Eval dataset is None. Skipping deduplication.")
-
-    if dataset is not None and (eval_dataset is None and train_dataset is None):
-        LOG.info(
-            f"Starting deduplication for combined dataset. Original size: {len(dataset)}"
-        )
-        dataset = deduplicate_dataset(dataset=dataset, seen_hashes=seen_hashes)
-        LOG.info(
-            f"Deduplication complete for combined dataset. New size: {len(dataset)}"
-        )
-
-    return train_dataset, eval_dataset, dataset
--- a/src/axolotl/utils/distributed.py
+++ b/src/axolotl/utils/distributed.py
@@ -9,44 +9,10 @@ from datetime import timedelta
 import torch
 import torch.distributed as dist
 from accelerate import PartialState
-from transformers.utils.import_utils import (
-    is_torch_cuda_available,
-    is_torch_mps_available,
-    is_torch_npu_available,
-)

 distributed_state = None  # pylint: disable=invalid-name


-def get_device_type():
-    device = torch.device("cpu")
-    if is_torch_cuda_available():
-        device = torch.device("cuda")
-    elif is_torch_mps_available():
-        device = torch.device("mps")
-    elif is_torch_npu_available():
-        device = torch.device("npu")
-    return device
-
-
-def get_device_count():
-    cur_device = get_device_type()
-    if "cuda" in str(cur_device):
-        return torch.cuda.device_count()
-    if "npu" in str(cur_device):
-        return torch.npu.device_count()
-    return 1
-
-
-def get_current_device():
-    cur_device = get_device_type()
-    if "cuda" in str(cur_device):
-        return torch.cuda.current_device()
-    if "npu" in str(cur_device):
-        return torch.npu.current_device()
-    return 0
-
-
 def is_distributed():
    """
    Check if distributed training is initialized.
@@ -125,7 +91,7 @@ def gather_scalar_from_all_ranks(fn, world_size=1):  # pylint: disable=invalid-n
    if not is_distributed():
        return [value_scalar]
    value_tensor = torch.tensor(
-        value_scalar, device=f"{get_device_type()}:{get_current_device()}"
+        value_scalar, device=torch.cuda.current_device()
    ).float()

    if not is_main_process():
@@ -149,14 +115,13 @@ def broadcast_dict(vals: dict):
    if not is_distributed():
        return vals

-    cur_device = get_device_type()
    if is_main_process():
        data_byte = pickle.dumps(vals)
-        data_tensor = torch.ByteTensor(list(data_byte)).to(cur_device)
-        data_size = torch.IntTensor([len(data_byte)]).to(cur_device)
+        data_tensor = torch.ByteTensor(list(data_byte)).to("cuda")
+        data_size = torch.IntTensor([len(data_byte)]).to("cuda")
    else:
-        data_tensor = torch.empty([1024], dtype=torch.uint8, device=cur_device)
-        data_size = torch.IntTensor([0]).to(cur_device)
+        data_tensor = torch.empty([1024], dtype=torch.uint8, device="cuda")
+        data_size = torch.IntTensor([0]).to("cuda")

    dist.broadcast(data_size, 0)
    if not is_main_process():
@@ -185,15 +150,14 @@ def compute_and_broadcast(fn):  # pylint: disable=invalid-name
    Returns:
    - The computed value (int or float).
    """
-    cur_device = f"{get_device_type()}:{get_current_device()}"
    if is_main_process():
        value_scalar = fn()
        value_tensor = torch.tensor(
-            value_scalar, device=cur_device, dtype=torch.float32
+            value_scalar, device=torch.cuda.current_device(), dtype=torch.float32
        )
    else:
        value_tensor = torch.tensor(
-            0.0, device=cur_device, dtype=torch.float32
+            0.0, device=torch.cuda.current_device(), dtype=torch.float32
        )  # Placeholder tensor

    # Broadcast the tensor to all processes.
@@ -220,7 +184,7 @@ def gather_from_all_ranks(fn, world_size=1):  # pylint: disable=invalid-name
    """
    value_scalar = fn()
    value_tensor = torch.tensor(
-        value_scalar, device=f"{get_device_type()}:{get_current_device()}"
+        value_scalar, device=torch.cuda.current_device()
    ).float()

    # Placeholder tensor for gathering results
--- a/src/axolotl/utils/models.py
+++ b/src/axolotl/utils/models.py
@@ -2,12 +2,10 @@

 # pylint: disable=too-many-lines
 import gc
-import importlib
 import logging
 import math
 import os
 import types
-from functools import cached_property
 from typing import Any, Dict, Optional, Tuple, Union  # noqa: F401

 import addict
@@ -30,7 +28,6 @@ from transformers import (  # noqa: F401
    AddedToken,
    AutoConfig,
    AutoModelForCausalLM,
-    AutoModelForImageTextToText,
    AutoModelForVision2Seq,
    AutoProcessor,
    AutoTokenizer,
@@ -58,7 +55,7 @@ from axolotl.prompt_tokenizers import LLAMA_DEFAULT_EOS_TOKEN
 from axolotl.utils.bench import log_gpu_memory_usage
 from axolotl.utils.chat_templates import get_chat_template_from_config
 from axolotl.utils.dict import DictDefault
-from axolotl.utils.distributed import get_device_count, get_device_type, zero_only
+from axolotl.utils.distributed import zero_only
 from axolotl.utils.gradient_checkpointing import hf_grad_checkpoint_unsloth_wrapper
 from axolotl.utils.lora_embeddings import get_linear_embedding_layers
 from axolotl.utils.model_shard_quant import load_sharded_model, load_sharded_model_quant
@@ -92,11 +89,7 @@ def get_module_class_from_name(module, name):

 def check_model_config(cfg: DictDefault, model_config: Union[AutoConfig, DictDefault]):
    if cfg.is_multimodal:
-        try:
-            model_config = model_config.text_config
-        except AttributeError:
-            # for qwen2_vl
-            model_config = model_config.get_text_config()
+        model_config = model_config.text_config

    quant_config_exists = (
        hasattr(model_config, "quantization_config")
@@ -372,11 +365,7 @@ class ModelLoader:
        # init model config
        self.model_config = load_model_config(cfg)
        if cfg.is_multimodal:
-            try:
-                self.text_model_config = self.model_config.text_config
-            except AttributeError:
-                # for qwen2_vl
-                self.text_model_config = self.model_config.get_text_config()
+            self.text_model_config = self.model_config.text_config
        else:
            self.text_model_config = self.model_config

@@ -420,7 +409,7 @@ class ModelLoader:
            )

            if self.cfg.is_llama_derived_model:
-                self.patch_loss_llama()
+                self.patch_loss()
                if self.cfg.unsloth_lora_qkv or self.cfg.unsloth_lora_o:
                    from axolotl.monkeypatch.unsloth_ import patch_self_attn_lora

@@ -462,34 +451,27 @@ class ModelLoader:

                replace_stablelm_attn_with_flash_attn(self.cfg.base_model)

-    @cached_property
-    def has_flash_attn(self) -> bool:
-        """Check if flash attention is installed"""
-        return importlib.util.find_spec("flash_attn") is not None
-
-    def patch_loss_llama(self) -> None:
+    def patch_loss(self) -> None:
        """
        Patch loss functions
        """
-        if self.has_flash_attn:
-            from axolotl.monkeypatch.llama_attn_hijack_flash import (
-                patch_fa_llama_cross_entropy,
-                patch_llama_rms_norm,
-            )
+        from axolotl.monkeypatch.llama_attn_hijack_flash import (
+            patch_llama_cross_entropy,
+            patch_llama_rms_norm,
+        )

-        if self.cfg.flash_attn_cross_entropy and self.has_flash_attn:
-            patch_fa_llama_cross_entropy()
-        elif self.cfg.unsloth_cross_entropy_loss:
-            from axolotl.monkeypatch.unsloth_ import integrate_cross_entropy_loss_patch
-
-            integrate_cross_entropy_loss_patch(model_type="llama")
-
-        if self.cfg.flash_attn_rms_norm and self.has_flash_attn:
+        if self.cfg.flash_attn_cross_entropy:
+            patch_llama_cross_entropy()
+        if self.cfg.flash_attn_rms_norm:
            patch_llama_rms_norm()
        elif self.cfg.unsloth_rms_norm:
            from axolotl.monkeypatch.unsloth_ import patch_unsloth_layernorm

            patch_unsloth_layernorm()
+        if self.cfg.unsloth_cross_entropy_loss:
+            from axolotl.monkeypatch.unsloth_ import integrate_cross_entropy_loss_patch
+
+            integrate_cross_entropy_loss_patch(model_type="llama")
        if self.cfg.unsloth_lora_qkv or self.cfg.unsloth_lora_o:
            from axolotl.monkeypatch.unsloth_ import patch_self_attn_lora

@@ -499,7 +481,6 @@ class ModelLoader:
        """
        Modify all llama derived models in one block
        """
-        self.patch_loss_llama()

        if self.cfg.flash_attention:
            from axolotl.monkeypatch.llama_attn_hijack_flash import (
@@ -547,6 +528,16 @@ class ModelLoader:
                "Shifted-sparse attention not currently implemented without flash attention."
            )

+        if self.cfg.unsloth_cross_entropy_loss:
+            from axolotl.monkeypatch.unsloth_ import integrate_cross_entropy_loss_patch
+
+            integrate_cross_entropy_loss_patch(model_type="llama")
+
+        if self.cfg.unsloth_lora_qkv or self.cfg.unsloth_lora_o:
+            from axolotl.monkeypatch.unsloth_ import patch_self_attn_lora
+
+            patch_self_attn_lora()
+
    def set_auto_model_loader(self) -> None:
        """set self.AutoModelLoader
        - default value: AutoModelForCausalLM (set at __init__)
@@ -562,10 +553,6 @@ class ModelLoader:
                self.AutoModelLoader = (  # pylint: disable=invalid-name
                    MllamaForConditionalGeneration
                )
-            elif self.model_config.model_type == "qwen2_vl":
-                self.AutoModelLoader = (  # pylint: disable=invalid-name
-                    AutoModelForImageTextToText
-                )
            else:
                self.AutoModelLoader = (
                    AutoModelForVision2Seq  # pylint: disable=invalid-name
@@ -583,8 +570,7 @@ class ModelLoader:
            )

            max_memory = {}
-            num_device = get_device_count()
-            for i in range(num_device):
+            for i in range(torch.cuda.device_count()):
                max_memory[i] = gpu_memory_limit
            max_memory["cpu"] = "256GiB"  # something sufficiently large to fit anything

@@ -609,11 +595,8 @@ class ModelLoader:
        self.model_kwargs["device_map"] = device_map
        self.model_kwargs["torch_dtype"] = self.cfg.torch_dtype

-        cur_device = get_device_type()
-        if "mps" in str(cur_device):
+        if torch.backends.mps.is_available():
            self.model_kwargs["device_map"] = "mps:0"
-        elif "npu" in str(cur_device):
-            self.model_kwargs["device_map"] = "npu:0"

        # TODO can we put the reference model on it's own gpu? I think we have to move logits around to calculate loss
        # if cfg.rl:
@@ -1058,9 +1041,7 @@ class ModelLoader:
            and self.model.get_input_embeddings().num_embeddings < embeddings_len
        ):
            resize_kwargs = {}
-            if self.cfg.mean_resizing_embeddings is not None and not (
-                self.model_config.model_type == "llava"
-            ):
+            if self.cfg.mean_resizing_embeddings is not None:
                resize_kwargs["mean_resizing"] = self.cfg.mean_resizing_embeddings
            self.model.resize_token_embeddings(embeddings_len, **resize_kwargs)
        else:
@@ -1069,11 +1050,7 @@ class ModelLoader:
        self.ajust_model_config()

        # log device memory usage
-        if hasattr(self.model, "device") and self.model.device.type in (
-            "cuda",
-            "mps",
-            "npu",
-        ):
+        if hasattr(self.model, "device") and self.model.device.type in ("cuda", "mps"):
            log_gpu_memory_usage(LOG, "after model load", self.model.device)

        # make sure these are fp32 per Ramesh et al. (2021)
@@ -1099,17 +1076,14 @@ class ModelLoader:

        self.prepare_model(qlora_fsdp)

-        should_convert = (
-            # LlamaRMSNorm layers are in fp32 after kbit_training or full finetune, so we need to
-            # convert them back to fp16/bf16 for flash-attn compatibility.
-            ((needs_fa2_dtype or self.cfg.flash_attention) and not qlora_fsdp)
-            or self.cfg.cut_cross_entropy  # Cut cross entropy requires embedding layers to be in fp16/bf16 for backward pass
-        )
-
-        if should_convert:
-            LOG.info("Converting modules to %s", self.cfg.torch_dtype)
+        # LlamaRMSNorm layers are in fp32 after kbit_training or full finetune, so we need to
+        # convert them back to fp16/bf16 for flash-attn compatibility.
+        if (needs_fa2_dtype or self.cfg.flash_attention) and not qlora_fsdp:
+            LOG.info(
+                "converting modules to %s for flash attention", self.cfg.torch_dtype
+            )
            self.convert_embedding_modules_dtype(
-                embedding_modules=embedding_modules,
+                embedding_modules,
                dist_dtype=self.cfg.torch_dtype,
                before_kbit_train_or_finetune=False,
            )
@@ -1144,9 +1118,9 @@ class ModelLoader:
            and not skip_move_to_device
        ):
            # TODO revaldate this conditional
-            self.model.to(f"{str(get_device_type())}:{self.cfg.local_rank}")
+            self.model.to(f"cuda:{self.cfg.local_rank}")

-        if get_device_count() > 1 and int(os.getenv("WORLD_SIZE", "1")) == 1:
+        if torch.cuda.device_count() > 1 and int(os.getenv("WORLD_SIZE", "1")) == 1:
            setattr(self.model, "is_parallelizable", True)
            setattr(self.model, "model_parallel", True)

--- a/src/axolotl/utils/optimizers/adopt.py
+++ b/src/axolotl/utils/optimizers/adopt.py
@@ -6,29 +6,21 @@ Taniguchi, Shohei and Harada, Keno and Minegishi, Gouki and Oshima, Yuta and Jeo
 """
 # mypy: ignore-errors
 # pylint: skip-file
-# flake8: noqa
 # mypy: allow-untyped-decorators
 # mypy: allow-untyped-defs
-from typing import Callable, List, Optional, Tuple, Union, cast
+from typing import List, Optional, Tuple, Union, cast

 import torch
 from torch import Tensor
-from torch.optim.optimizer import (  # DeviceDict,; _capturable_doc,; _differentiable_doc,; _foreach_doc,; _fused_doc,; _maximize_doc,; _stack_if_compiling,
-    DeviceDict,
+from torch.optim.optimizer import (
    Optimizer,
    ParamsT,
-    _capturable_doc,
    _default_to_fused_or_foreach,
    _device_dtype_check_for_fused,
-    _differentiable_doc,
    _disable_dynamo_if_unsupported,
-    _foreach_doc,
-    _fused_doc,
    _get_capturable_supported_devices,
    _get_scalar_dtype,
    _get_value,
-    _maximize_doc,
-    _stack_if_compiling,
    _use_grad_for_differentiable,
    _view_as_real,
 )
@@ -43,9 +35,8 @@ class ADOPT(Optimizer):
        lr: Union[float, Tensor] = 1e-3,
        betas: Tuple[float, float] = (0.9, 0.9999),
        eps: float = 1e-6,
-        clip_lambda: Optional[Callable[[int], float]] = lambda step: step**0.25,
        weight_decay: float = 0.0,
-        decouple: bool = False,
+        decoupled: bool = False,
        *,
        foreach: Optional[bool] = None,
        maximize: bool = False,
@@ -71,14 +62,12 @@ class ADOPT(Optimizer):
        if not 0.0 <= weight_decay:
            raise ValueError(f"Invalid weight_decay value: {weight_decay}")

-        self.clip_lambda = clip_lambda
-
        defaults = dict(
            lr=lr,
            betas=betas,
            eps=eps,
            weight_decay=weight_decay,
-            decouple=decouple,
+            decoupled=decoupled,
            maximize=maximize,
            foreach=foreach,
            capturable=capturable,
@@ -230,9 +219,8 @@ class ADOPT(Optimizer):
                beta1=beta1,
                beta2=beta2,
                lr=group["lr"],
-                clip_lambda=self.clip_lambda,
                weight_decay=group["weight_decay"],
-                decouple=group["decouple"],
+                decoupled=group["decoupled"],
                eps=group["eps"],
                maximize=group["maximize"],
                foreach=group["foreach"],
@@ -259,9 +247,8 @@ def _single_tensor_adopt(
    beta1: float,
    beta2: float,
    lr: Union[float, Tensor],
-    clip_lambda: Optional[Callable[[int], float]],
    weight_decay: float,
-    decouple: bool,
+    decoupled: bool,
    eps: float,
    maximize: bool,
    capturable: bool,
@@ -289,10 +276,14 @@ def _single_tensor_adopt(
                and param.device.type in capturable_supported_devices
            ), f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}."

-        step = step_t if capturable or differentiable else _get_value(step_t)
+        # update step
+        step_t += 1

-        if weight_decay != 0 and not decouple:
-            grad = grad.add(param, alpha=weight_decay)
+        if weight_decay != 0:
+            if decoupled:
+                param.add_(param, alpha=-lr * weight_decay)
+            else:
+                grad = grad.add(param, alpha=weight_decay)

        if torch.is_complex(param):
            grad = torch.view_as_real(grad)
@@ -302,29 +293,20 @@ def _single_tensor_adopt(
                exp_avg_sq = torch.view_as_real(exp_avg_sq)
            param = torch.view_as_real(param)

-        if step == 0:
+        step = step_t if capturable or differentiable else _get_value(step_t)
+        if step == 1:
            exp_avg_sq.addcmul_(grad, grad.conj())
-            # update step
-            step_t += 1
            continue

-        if weight_decay != 0 and decouple:
-            param.add_(param, alpha=-lr * weight_decay)
-
        denom = torch.clamp(exp_avg_sq.sqrt(), eps)
-        normed_grad = grad.div(denom)
-        if clip_lambda is not None:
-            clip = clip_lambda(step)
-            normed_grad.clamp_(-clip, clip)
-
-        exp_avg.lerp_(normed_grad, 1 - beta1)
+        if step == 2:
+            exp_avg.addcdiv_(grad, denom)
+        else:
+            exp_avg.mul_(beta1).addcdiv_(grad, denom, value=1 - beta1)

        param.add_(exp_avg, alpha=-lr)
        exp_avg_sq.mul_(beta2).addcmul_(grad, grad.conj(), value=1 - beta2)

-        # update step
-        step_t += 1
-

 def _multi_tensor_adopt(
    params: List[Tensor],
@@ -339,9 +321,8 @@ def _multi_tensor_adopt(
    beta1: float,
    beta2: float,
    lr: Union[float, Tensor],
-    clip_lambda: Optional[Callable[[int], float]],
    weight_decay: float,
-    decouple: bool,
+    decoupled: bool,
    eps: float,
    maximize: bool,
    capturable: bool,
@@ -395,51 +376,6 @@ def _multi_tensor_adopt(
        if maximize:
            device_grads = torch._foreach_neg(device_grads)  # type: ignore[assignment]

-        if weight_decay != 0 and not decouple:
-            # Re-use the intermediate memory (device_grads) already allocated for maximize
-            if maximize:
-                torch._foreach_add_(device_grads, device_params, alpha=weight_decay)
-            else:
-                device_grads = torch._foreach_add(  # type: ignore[assignment]
-                    device_grads, device_params, alpha=weight_decay
-                )
-
-        if device_state_steps[0] == 0:
-            torch._foreach_addcmul_(device_exp_avg_sqs, device_grads, device_grads)
-
-            # Update steps
-            # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over
-            # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just
-            # wrapped it once now. The alpha is required to assure we go to the right overload.
-            if not torch._utils.is_compiling() and device_state_steps[0].is_cpu:
-                torch._foreach_add_(
-                    device_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0
-                )
-            else:
-                torch._foreach_add_(device_state_steps, 1)
-
-            continue
-
-        if weight_decay != 0 and decouple:
-            torch._foreach_add_(device_params, device_params, alpha=-lr * weight_decay)
-
-        exp_avg_sq_sqrt = torch._foreach_sqrt(device_exp_avg_sqs)
-        torch._foreach_maximum_(exp_avg_sq_sqrt, eps)
-
-        normed_grad = torch._foreach_div(device_grads, exp_avg_sq_sqrt)
-        if clip_lambda is not None:
-            clip = clip_lambda(device_state_steps[0])
-            torch._foreach_maximum_(normed_grad, -clip)
-            torch._foreach_minimum_(normed_grad, clip)
-
-        torch._foreach_lerp_(device_exp_avgs, normed_grad, 1 - beta1)
-
-        torch._foreach_add_(device_params, device_exp_avgs, alpha=-lr)
-        torch._foreach_mul_(device_exp_avg_sqs, beta2)
-        torch._foreach_addcmul_(
-            device_exp_avg_sqs, device_grads, device_grads, value=1 - beta2
-        )
-
        # Update steps
        # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over
        # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just
@@ -451,6 +387,41 @@ def _multi_tensor_adopt(
        else:
            torch._foreach_add_(device_state_steps, 1)

+        if weight_decay != 0:
+            if decoupled:
+                torch._foreach_add_(
+                    device_params, device_params, alpha=-lr * weight_decay
+                )
+            else:
+                # Re-use the intermediate memory (device_grads) already allocated for maximize
+                if maximize:
+                    torch._foreach_add_(device_grads, device_params, alpha=weight_decay)
+                else:
+                    device_grads = torch._foreach_add(  # type: ignore[assignment]
+                        device_grads, device_params, alpha=weight_decay
+                    )
+
+        if device_state_steps[0] == 1:
+            torch._foreach_addcmul_(device_exp_avg_sqs, device_grads, device_grads)
+            continue
+
+        exp_avg_sq_sqrt = torch._foreach_sqrt(device_exp_avg_sqs)
+        exp_avg_sq_sqrt = torch._foreach_maximum(exp_avg_sq_sqrt, eps)
+
+        if device_state_steps[0] == 2:
+            torch._foreach_addcdiv_(device_exp_avgs, device_grads, exp_avg_sq_sqrt)
+        else:
+            torch._foreach_mul_(device_exp_avgs, beta1)
+            torch._foreach_addcdiv_(
+                device_exp_avgs, device_grads, exp_avg_sq_sqrt, value=1 - beta1
+            )
+
+        torch._foreach_add_(device_params, device_exp_avgs, alpha=-lr)
+        torch._foreach_mul_(device_exp_avg_sqs, beta2)
+        torch._foreach_addcmul_(
+            device_exp_avg_sqs, device_grads, device_grads, value=1 - beta2
+        )
+

@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_adopt)
 def adopt(
@@ -472,9 +443,8 @@ def adopt(
    beta1: float,
    beta2: float,
    lr: Union[float, Tensor],
-    clip_lambda: Optional[Callable[[int], float]],
    weight_decay: float,
-    decouple: bool,
+    decoupled: bool,
    eps: float,
    maximize: bool,
 ):
@@ -527,9 +497,8 @@ def adopt(
        beta1=beta1,
        beta2=beta2,
        lr=lr,
-        clip_lambda=clip_lambda,
        weight_decay=weight_decay,
-        decouple=decouple,
+        decoupled=decoupled,
        eps=eps,
        maximize=maximize,
        capturable=capturable,
--- a/src/axolotl/utils/tokenization.py
+++ b/src/axolotl/utils/tokenization.py
@@ -66,47 +66,28 @@ def process_tokens_for_rl_debug(tokens, color, tokenizer, text_only):


 def check_rl_example_labels(example, tokenizer, text_only=False):
-    field_prompt, field_chosen, field_rejected, field_completion = (
-        "prompt",
-        "chosen",
-        "rejected",
-        "completion",
-    )
+    field_prompt, field_chosen, field_rejected = "prompt", "chosen", "rejected"

    input_tokens = example[field_prompt]
-
-    labels_chosen = example.get(field_chosen)
-    labels_rejected = example.get(field_rejected)
-    labels_completion = example.get(field_completion)
-
-    # Create a delimiter based on text_only flag
-    delimiter = "" if text_only else " "
+    labels_chosen, labels_rejected = example[field_chosen], example[field_rejected]

    # Process and color each type of token
    colored_tokens = process_tokens_for_rl_debug(
        input_tokens, "yellow", tokenizer, text_only
    )
+    colored_chosens = process_tokens_for_rl_debug(
+        labels_chosen, "green", tokenizer, text_only
+    )
+    colored_rejecteds = process_tokens_for_rl_debug(
+        labels_rejected, "red", tokenizer, text_only
+    )

-    # Process tokens
-    if labels_completion is None:
-        colored_chosens = process_tokens_for_rl_debug(
-            labels_chosen, "green", tokenizer, text_only
-        )
-        colored_rejecteds = process_tokens_for_rl_debug(
-            labels_rejected, "red", tokenizer, text_only
-        )
-    else:
-        colored_completion = process_tokens_for_rl_debug(
-            labels_completion, "green", tokenizer, text_only
-        )
+    # Create a delimiter based on text_only flag
+    delimiter = "" if text_only else " "

    # Logging information
    LOG.info(f"INPUT PROMPT: {delimiter.join(colored_tokens)}\n\n")
-
-    if labels_completion is None:
-        LOG.info(f"CHOSEN RESPONSE: {delimiter.join(colored_chosens)}\n\n")
-        LOG.info(f"REJECTED RESPONSE: {delimiter.join(colored_rejecteds)}\n\n\n")
-    else:
-        LOG.info(f"COMPLETION RESPONSE: {delimiter.join(colored_completion)}\n\n\n")
+    LOG.info(f"CHOSEN RESPONSE: {delimiter.join(colored_chosens)}\n\n")
+    LOG.info(f"REJECTED RESPONSE: {delimiter.join(colored_rejecteds)}\n\n\n")

    return delimiter.join(colored_tokens)
--- a/src/axolotl/utils/trainer.py
+++ b/src/axolotl/utils/trainer.py
@@ -16,6 +16,9 @@ from torch.utils.data import DataLoader, RandomSampler
 from transformers.utils import is_torch_bf16_gpu_available

 from axolotl.core.trainer_builder import HFCausalTrainerBuilder, HFRLTrainerBuilder
+from axolotl.monkeypatch.trainer_fsdp_grad_accum import (
+    patch_training_loop_for_fsdp_grad_accum,
+)
 from axolotl.utils.distributed import reduce_and_broadcast
 from axolotl.utils.environment import check_cuda_p2p_ib_support
 from axolotl.utils.samplers import MultipackBatchSampler, get_dataset_lengths
@@ -203,59 +206,37 @@ def process_datasets_for_packing(cfg, train_dataset, eval_dataset):
        if eval_dataset and "token_type_ids" in eval_dataset.column_names:
            eval_dataset = eval_dataset.remove_columns("token_type_ids")

-    prior_len = len(train_dataset)
    train_dataset = train_dataset.filter(
        drop_long,
        num_proc=cfg.dataset_processes,
        load_from_cache_file=not cfg.is_preprocess,
        desc="Dropping Long Sequences",
    )
-    dropped = prior_len - len(train_dataset)
-    if dropped:
-        LOG.warning(f"Dropped {dropped} long samples from train dataset")
-
    if eval_dataset:
-        prior_len = len(eval_dataset)
        eval_dataset = eval_dataset.filter(
            drop_long,
            num_proc=cfg.dataset_processes,
            load_from_cache_file=not cfg.is_preprocess,
            desc="Dropping Long Sequences",
        )
-        dropped = prior_len - len(eval_dataset)
-        if dropped:
-            LOG.warning(f"Dropped {dropped} long samples from eval dataset")

    # drop samples with where the number of elements with labels not equal to -100 is zero
    def drop_no_trainable_tokens(sample):
        return np.sum(np.array(sample["labels"]) != -100) > 0

-    prior_len = len(train_dataset)
    train_dataset = train_dataset.filter(
        drop_no_trainable_tokens,
        num_proc=cfg.dataset_processes,
        load_from_cache_file=not cfg.is_preprocess,
        desc="Drop Samples with Zero Trainable Tokens",
    )
-    dropped = prior_len - len(train_dataset)
-    if dropped:
-        LOG.warning(
-            f"Dropped {dropped} samples with no trainable tokens from train dataset"
-        )
-
    if eval_dataset:
-        prior_len = len(eval_dataset)
        eval_dataset = eval_dataset.filter(
            drop_no_trainable_tokens,
            num_proc=cfg.dataset_processes,
            load_from_cache_file=not cfg.is_preprocess,
            desc="Drop Samples with Zero Trainable Tokens",
        )
-        dropped = prior_len - len(eval_dataset)
-        if dropped:
-            LOG.warning(
-                f"Dropped {dropped} samples with no trainable tokens from eval dataset"
-            )

    if cfg.group_by_length:
        train_dataset = train_dataset.map(
@@ -515,7 +496,12 @@ def prepare_opinionated_env(cfg):
 def setup_trainer(
    cfg, train_dataset, eval_dataset, model, tokenizer, processor, total_num_steps
 ):
-    if cfg.rl in ("dpo", "ipo", "orpo", "kto", "simpo"):
+    if cfg.fsdp:
+        try:
+            patch_training_loop_for_fsdp_grad_accum()
+        except AssertionError:
+            pass
+    if cfg.rl in ["dpo", "ipo", "orpo", "kto", "simpo"]:
        trainer_builder = HFRLTrainerBuilder(cfg, model[0], tokenizer, processor)
        trainer_builder.model_ref = model[1]
        trainer_builder.peft_config = model[2]
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,76 +0,0 @@
-"""
-shared pytest fixtures
-"""
-import shutil
-import tempfile
-
-import pytest
-from huggingface_hub import snapshot_download
-
-
-@pytest.fixture(scope="session", autouse=True)
-def download_smollm2_135m_model():
-    # download the model
-    snapshot_download("HuggingFaceTB/SmolLM2-135M")
-
-
-@pytest.fixture(scope="session", autouse=True)
-def download_llama_68m_random_model():
-    # download the model
-    snapshot_download("JackFram/llama-68m")
-
-
-@pytest.fixture(scope="session", autouse=True)
-def download_qwen_2_5_half_billion_model():
-    # download the model
-    snapshot_download("Qwen/Qwen2.5-0.5B")
-
-
-@pytest.fixture(scope="session", autouse=True)
-def download_tatsu_lab_alpaca_dataset():
-    # download the dataset
-    snapshot_download("tatsu-lab/alpaca", repo_type="dataset")
-
-
-@pytest.fixture(scope="session", autouse=True)
-def download_mhenrichsen_alpaca_2k_dataset():
-    # download the dataset
-    snapshot_download("mhenrichsen/alpaca_2k_test", repo_type="dataset")
-
-
-@pytest.fixture(scope="session", autouse=True)
-def download_mhenrichsen_alpaca_2k_w_revision_dataset():
-    # download the dataset
-    snapshot_download(
-        "mhenrichsen/alpaca_2k_test", repo_type="dataset", revision="d05c1cb"
-    )
-
-
-def download_mlabonne_finetome_100k_dataset():
-    # download the dataset
-    snapshot_download("mlabonne/FineTome-100k", repo_type="dataset")
-
-
-@pytest.fixture
-def download_argilla_distilabel_capybara_dpo_7k_binarized_dataset():
-    # download the dataset
-    snapshot_download(
-        "argilla/distilabel-capybara-dpo-7k-binarized", repo_type="dataset"
-    )
-
-
-@pytest.fixture
-def download_arcee_ai_distilabel_intel_orca_dpo_pairs_dataset():
-    # download the dataset
-    snapshot_download(
-        "arcee-ai/distilabel-intel-orca-dpo-pairs-binarized", repo_type="dataset"
-    )
-
-
-@pytest.fixture
-def temp_dir():
-    # Create a temporary directory
-    _temp_dir = tempfile.mkdtemp()
-    yield _temp_dir
-    # Clean up the directory after the test
-    shutil.rmtree(_temp_dir)
--- a/tests/constants.py
+++ b/tests/constants.py
@@ -1,32 +0,0 @@
-# constants.py
-"""
-This module contains constants and configuration dictionaries used for
-datasets and other utilities in the Axolotl project, specifically for testing.
-"""
-# Configuration for Alpaca Messages Dataset
-ALPACA_MESSAGES_CONFIG_OG = {
-    "path": "fozziethebeat/alpaca_messages_2k_dpo_test",
-    "type": "chat_template.default",
-    "chat_template": "llama3",
-    "field_messages": "conversation",
-    "field_chosen": "chosen",
-    "field_rejected": "rejected",
-    "message_field_role": "role",
-    "message_field_content": "content",
-    "roles": {
-        "system": ["system"],
-        "user": ["user"],
-        "assistant": ["assistant"],
-    },
-}
-
-# Revision configuration extending the original
-ALPACA_MESSAGES_CONFIG_REVISION = ALPACA_MESSAGES_CONFIG_OG.copy()
-ALPACA_MESSAGES_CONFIG_REVISION["revision"] = "ea82cff"
-
-
-SPECIAL_TOKENS = {
-    "bos_token": "<s>",
-    "eos_token": "</s>",
-    "unk_token": "<unk>",
-}
--- a/tests/core/test_trainer_builder.py
+++ b/tests/core/test_trainer_builder.py
@@ -14,7 +14,9 @@ from axolotl.utils.models import load_model, load_tokenizer
 def fixture_cfg():
    cfg = DictDefault(
        {
-            "base_model": "HuggingFaceTB/SmolLM2-135M",
+            "base_model": "TinyLlama/TinyLlama-1.1B-Chat-v0.6",
+            "model_type": "AutoModelForCausalLM",
+            "tokenizer_type": "LlamaTokenizer",
            "micro_batch_size": 1,
            "gradient_accumulation_steps": 1,
            "learning_rate": 0.00005,
@@ -31,9 +33,6 @@ def fixture_cfg():
            "dataloader_num_workers": 1,
            "dataloader_pin_memory": True,
            "model_config_type": "llama",
-            "special_tokens": {
-                "pad_token": "<|endoftext|>",
-            },
        }
    )

--- a/tests/e2e/conftest.py
+++ b/tests/e2e/conftest.py
@@ -0,0 +1,16 @@
+"""
+shared pytest fixtures
+"""
+import shutil
+import tempfile
+
+import pytest
+
+
+@pytest.fixture
+def temp_dir():
+    # Create a temporary directory
+    _temp_dir = tempfile.mkdtemp()
+    yield _temp_dir
+    # Clean up the directory after the test
+    shutil.rmtree(_temp_dir)
--- a/tests/e2e/integrations/liger.py
+++ b/tests/e2e/integrations/liger.py
@@ -7,7 +7,7 @@ from pathlib import Path
 from axolotl.cli import load_datasets
 from axolotl.common.cli import TrainerCliArgs
 from axolotl.train import train
-from axolotl.utils.config import normalize_config, prepare_plugins
+from axolotl.utils.config import normalize_config
 from axolotl.utils.dict import DictDefault

 from ..utils import with_temp_dir
@@ -54,10 +54,8 @@ class LigerIntegrationTestCase(unittest.TestCase):
                "lr_scheduler": "cosine",
                "save_safetensors": True,
                "bf16": "auto",
-                "max_steps": 10,
            }
        )
-        prepare_plugins(cfg)
        normalize_config(cfg)
        cli_args = TrainerCliArgs()
        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
@@ -101,10 +99,8 @@ class LigerIntegrationTestCase(unittest.TestCase):
                "lr_scheduler": "cosine",
                "save_safetensors": True,
                "bf16": "auto",
-                "max_steps": 10,
            }
        )
-        prepare_plugins(cfg)
        normalize_config(cfg)
        cli_args = TrainerCliArgs()
        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
--- a/tests/e2e/integrations/test_cut_cross_entropy.py
+++ b/tests/e2e/integrations/test_cut_cross_entropy.py
@@ -1,94 +0,0 @@
-"""
-Simple end-to-end test for Cut Cross Entropy integration
-"""
-
-from pathlib import Path
-
-import pytest
-
-from axolotl.cli import load_datasets
-from axolotl.common.cli import TrainerCliArgs
-from axolotl.train import train
-from axolotl.utils import get_pytorch_version
-from axolotl.utils.config import normalize_config, prepare_plugins
-from axolotl.utils.dict import DictDefault
-
-# pylint: disable=duplicate-code
-
-
-@pytest.fixture()
-def min_cfg(temp_dir):
-    return {
-        "base_model": "HuggingFaceTB/SmolLM2-135M",
-        "plugins": [
-            "axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin",
-        ],
-        "cut_cross_entropy": True,
-        "sequence_len": 1024,
-        "val_set_size": 0.1,
-        "special_tokens": {
-            "pad_token": "<|endoftext|>",
-        },
-        "datasets": [
-            {
-                "path": "mhenrichsen/alpaca_2k_test",
-                "type": "alpaca",
-            },
-        ],
-        "num_epochs": 1,
-        "micro_batch_size": 8,
-        "gradient_accumulation_steps": 1,
-        "learning_rate": 0.00001,
-        "optimizer": "adamw_torch",
-        "output_dir": temp_dir,
-        "lr_scheduler": "cosine",
-        "save_safetensors": True,
-        "max_steps": 10,
-        "bf16": "auto",
-    }
-
-
-class TestCutCrossEntropyIntegration:
-    """
-    e2e tests for cut_cross_entropy integration with Axolotl
-    """
-
-    # pylint: disable=redefined-outer-name
-    def test_llama_w_cce(self, min_cfg, temp_dir):
-        cfg = DictDefault(min_cfg)
-        prepare_plugins(cfg)
-        normalize_config(cfg)
-        cli_args = TrainerCliArgs()
-        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
-
-        major, minor, _ = get_pytorch_version()
-        if (major, minor) < (2, 4):
-            with pytest.raises(ImportError):
-                train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        else:
-            train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-            assert (Path(temp_dir) / "model.safetensors").exists()
-
-    @pytest.mark.parametrize(
-        "attention_type",
-        ["flash_attention", "sdp_attention", "xformers_attention"],
-    )
-    def test_llama_w_cce_and_attention(self, min_cfg, temp_dir, attention_type):
-        cfg = DictDefault(
-            min_cfg
-            | {
-                attention_type: True,
-            }
-        )
-        prepare_plugins(cfg)
-        normalize_config(cfg)
-        cli_args = TrainerCliArgs()
-        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
-
-        major, minor, _ = get_pytorch_version()
-        if (major, minor) < (2, 4):
-            with pytest.raises(ImportError):
-                train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        else:
-            train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-            assert (Path(temp_dir) / "model.safetensors").exists()
--- a/tests/e2e/multigpu/test_eval.py
+++ b/tests/e2e/multigpu/test_eval.py
@@ -7,13 +7,10 @@ from pathlib import Path

 import yaml
 from accelerate.test_utils import execute_subprocess_async
-from tbparse import SummaryReader
 from transformers.testing_utils import get_torch_dist_unique_port

 from axolotl.utils.dict import DictDefault

-from ..utils import most_recent_subdir
-
 LOG = logging.getLogger("axolotl.tests.e2e.multigpu")
 os.environ["WANDB_DISABLED"] = "true"

@@ -29,7 +26,7 @@ class TestMultiGPUEval:
        # pylint: disable=duplicate-code
        cfg = DictDefault(
            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "base_model": "JackFram/llama-68m",
                "load_in_8bit": False,
                "load_in_4bit": True,
                "strict": False,
@@ -43,8 +40,8 @@ class TestMultiGPUEval:
                "lora_dropout": 0.05,
                "lora_target_linear": True,
                "lora_modules_to_save": ["embed_tokens", "lm_head"],
-                "val_set_size": 0.004,
-                "special_tokens": {"pad_token": "<|endoftext|>"},
+                "val_set_size": 0.1,
+                "special_tokens": {"pad_token": "<|end_of_text|>"},
                "datasets": [
                    {
                        "path": "teknium/GPT4-LLM-Cleaned",
@@ -69,7 +66,6 @@ class TestMultiGPUEval:
                "saves_per_epoch": 1,
                "logging_steps": 1,
                "weight_decay": 0.0,
-                "use_tensorboard": True,
            }
        )

@@ -91,18 +87,12 @@ class TestMultiGPUEval:
                str(Path(temp_dir) / "config.yaml"),
            ]
        )
-        tb_log_path = most_recent_subdir(temp_dir + "/runs")
-        event_file = os.path.join(tb_log_path, sorted(os.listdir(tb_log_path))[0])
-        reader = SummaryReader(event_file)
-        df = reader.scalars  # pylint: disable=invalid-name
-        df = df[(df.tag == "eval/loss")]  # pylint: disable=invalid-name
-        assert df.value.values[-1] < 2.5, "Loss is too high"

    def test_eval(self, temp_dir):
        # pylint: disable=duplicate-code
        cfg = DictDefault(
            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "base_model": "JackFram/llama-68m",
                "load_in_8bit": False,
                "load_in_4bit": True,
                "strict": False,
@@ -116,8 +106,8 @@ class TestMultiGPUEval:
                "lora_dropout": 0.05,
                "lora_target_linear": True,
                "lora_modules_to_save": ["embed_tokens", "lm_head"],
-                "val_set_size": 0.0004,
-                "special_tokens": {"pad_token": "<|endoftext|>"},
+                "val_set_size": 0.1,
+                "special_tokens": {"pad_token": "<|end_of_text|>"},
                "datasets": [
                    {
                        "path": "teknium/GPT4-LLM-Cleaned",
@@ -142,7 +132,6 @@ class TestMultiGPUEval:
                "saves_per_epoch": 1,
                "logging_steps": 1,
                "weight_decay": 0.0,
-                "use_tensorboard": True,
            }
        )

@@ -164,9 +153,3 @@ class TestMultiGPUEval:
                str(Path(temp_dir) / "config.yaml"),
            ]
        )
-        tb_log_path = most_recent_subdir(temp_dir + "/runs")
-        event_file = os.path.join(tb_log_path, sorted(os.listdir(tb_log_path))[0])
-        reader = SummaryReader(event_file)
-        df = reader.scalars  # pylint: disable=invalid-name
-        df = df[(df.tag == "eval/loss")]  # pylint: disable=invalid-name
-        assert df.value.values[-1] < 2.9, "Loss is too high"
--- a/tests/e2e/multigpu/test_llama.py
+++ b/tests/e2e/multigpu/test_llama.py
@@ -25,7 +25,7 @@ AXOLOTL_ROOT = Path(__file__).parent.parent.parent.parent
@pytest.fixture(scope="session", autouse=True)
 def download_model():
    # download the model
-    snapshot_download("HuggingFaceTB/SmolLM2-135M")
+    snapshot_download("TinyLlama/TinyLlama_v1.1")


 class TestMultiGPULlama:
@@ -37,7 +37,7 @@ class TestMultiGPULlama:
        # pylint: disable=duplicate-code
        cfg = DictDefault(
            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "base_model": "HuggingFaceTB/SmolLM-135M",
                "sequence_len": 2048,
                "adapter": "lora",
                "lora_r": 8,
@@ -93,7 +93,7 @@ class TestMultiGPULlama:
        # pylint: disable=duplicate-code
        cfg = DictDefault(
            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "base_model": "HuggingFaceTB/SmolLM-135M",
                "sequence_len": 2048,
                "sample_packing": True,
                "eval_sample_packing": False,
@@ -149,7 +149,8 @@ class TestMultiGPULlama:
        # pylint: disable=duplicate-code
        cfg = DictDefault(
            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "base_model": "TinyLlama/TinyLlama_v1.1",
+                "tokenizer_type": "LlamaTokenizer",
                "sequence_len": 2048,
                "sample_packing": False,
                "eval_sample_packing": False,
@@ -162,10 +163,12 @@ class TestMultiGPULlama:
                "lora_target_linear": True,
                "val_set_size": 0.05,
                "special_tokens": {
-                    "pad_token": "<|endoftext|>",
+                    "unk_token": "<unk>",
+                    "bos_token": "<s>",
+                    "eos_token": "</s>",
                },
                "rl": "dpo",
-                "chat_template": "chatml",
+                "chat_template": "llama3",
                "datasets": [
                    {
                        "path": "fozziethebeat/alpaca_messages_2k_dpo_test",
@@ -218,7 +221,7 @@ class TestMultiGPULlama:
        # pylint: disable=duplicate-code
        cfg = DictDefault(
            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "base_model": "HuggingFaceTB/SmolLM-135M",
                "sequence_len": 2048,
                "sample_packing": False,
                "eval_sample_packing": False,
@@ -291,7 +294,7 @@ class TestMultiGPULlama:
        # pylint: disable=duplicate-code
        cfg = DictDefault(
            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "base_model": "HuggingFaceTB/SmolLM-135M",
                "sequence_len": 2048,
                "val_set_size": 0.01,
                "special_tokens": {
@@ -356,7 +359,7 @@ class TestMultiGPULlama:
        # pylint: disable=duplicate-code
        cfg = DictDefault(
            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "base_model": "HuggingFaceTB/SmolLM-135M",
                "sample_packing": True,
                "pad_to_sequence_len": True,
                "sequence_len": 2048,
@@ -419,7 +422,8 @@ class TestMultiGPULlama:
        # pylint: disable=duplicate-code
        cfg = DictDefault(
            {
-                "base_model": "axolotl-ai-co/SmolLM2-135M-bnb-nf4-bf16",
+                "base_model": "axolotl-ai-co/TinyLlama_v1.1-bnb-nf4-bf16",
+                "tokenizer_type": "AutoTokenizer",
                "adapter": "qlora",
                "mean_resizing_embeddings": True,
                "load_in_4bit": True,
@@ -427,17 +431,17 @@ class TestMultiGPULlama:
                "lora_alpha": 16,
                "lora_dropout": 0.05,
                "lora_target_linear": True,
-                # "lora_modules_to_save": [
-                #     "embed_tokens",
-                #     "lm_head",
-                # ],
+                "lora_modules_to_save": [
+                    "embed_tokens",
+                    "lm_head",
+                ],
                "sample_packing": True,
                "eval_sample_packing": False,
                "pad_to_sequence_len": True,
                "sequence_len": 2048,
                "val_set_size": 0.05,
                "special_tokens": {
-                    "pad_token": "<|endoftext|>",
+                    "pad_token": "</s>",
                },
                "datasets": [
                    {
@@ -499,7 +503,7 @@ class TestMultiGPULlama:
        # pylint: disable=duplicate-code
        cfg = DictDefault(
            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "base_model": "HuggingFaceTB/SmolLM-135M",
                "sample_packing": True,
                "pad_to_sequence_len": True,
                "sequence_len": 2048,
@@ -549,7 +553,7 @@ class TestMultiGPULlama:
        # pylint: disable=duplicate-code
        cfg = DictDefault(
            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "base_model": "HuggingFaceTB/SmolLM-135M",
                "load_in_4bit": True,
                "adapter": "qlora",
                "lora_r": 8,
--- a/tests/e2e/patched/test_fa_xentropy.py
+++ b/tests/e2e/patched/test_fa_xentropy.py
@@ -4,11 +4,11 @@ E2E tests for lora llama

 import logging
 import os
+import unittest
 from importlib import reload
 from pathlib import Path

 import pytest
-from tbparse import SummaryReader
 from transformers.utils import is_torch_bf16_gpu_available

 from axolotl.cli import load_datasets
@@ -17,7 +17,7 @@ from axolotl.train import train
 from axolotl.utils.config import normalize_config
 from axolotl.utils.dict import DictDefault

-from ..utils import most_recent_subdir
+from ..utils import with_temp_dir

 LOG = logging.getLogger("axolotl.tests.e2e")
 os.environ["WANDB_DISABLED"] = "true"
@@ -31,55 +31,47 @@ def reload_transformers():
    reload(transformers.models.llama.modeling_llama)


-class TestFAXentropyLlama:
+class TestFAXentropyLlama(unittest.TestCase):
    """
    Test case for Llama models using LoRA w multipack
    """

-    @pytest.mark.parametrize(
-        "gradient_accumulation_steps",
-        [1, 4],
-    )
-    def test_lora_packing_fa_cross_entropy(self, temp_dir, gradient_accumulation_steps):
+    @with_temp_dir
+    def test_lora_packing_fa_cross_entropy(self, temp_dir):
        # pylint: disable=duplicate-code
        cfg = DictDefault(
            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "base_model": "JackFram/llama-68m",
+                "tokenizer_type": "LlamaTokenizer",
                "sequence_len": 1024,
                "sample_packing": True,
                "flash_attention": True,
                "flash_attn_cross_entropy": True,
                "load_in_8bit": True,
                "adapter": "lora",
-                "lora_r": 8,
-                "lora_alpha": 16,
+                "lora_r": 32,
+                "lora_alpha": 64,
                "lora_dropout": 0.05,
                "lora_target_linear": True,
-                "val_set_size": 0.05,
+                "val_set_size": 0.2,
                "special_tokens": {
-                    "pad_token": "<|endoftext|>",
+                    "unk_token": "<unk>",
+                    "bos_token": "<s>",
+                    "eos_token": "</s>",
                },
-                "chat_template": "chatml",
                "datasets": [
                    {
-                        "path": "mlabonne/FineTome-100k",
-                        "field_messages": "conversations",
-                        "message_field_content": "value",
-                        "message_field_role": "from",
-                        "type": "chat_template",
-                        "split": "train[:2%]",
+                        "path": "mhenrichsen/alpaca_2k_test",
+                        "type": "alpaca",
                    },
                ],
                "num_epochs": 1,
-                "max_steps": 5,
-                "save_steps": 5,
-                "micro_batch_size": 2,
-                "gradient_accumulation_steps": gradient_accumulation_steps,
+                "micro_batch_size": 8,
+                "gradient_accumulation_steps": 1,
                "output_dir": temp_dir,
                "learning_rate": 0.00001,
-                "optimizer": "adamw_8bit",
+                "optimizer": "adamw_torch",
                "lr_scheduler": "cosine",
-                "use_tensorboard": True,
            }
        )
        if is_torch_bf16_gpu_available():
@@ -93,10 +85,3 @@ class TestFAXentropyLlama:

        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
        assert (Path(temp_dir) / "adapter_model.bin").exists()
-
-        tb_log_path = most_recent_subdir(temp_dir + "/runs")
-        event_file = os.path.join(tb_log_path, sorted(os.listdir(tb_log_path))[0])
-        reader = SummaryReader(event_file)
-        df = reader.scalars  # pylint: disable=invalid-name
-        df = df[(df.tag == "train/train_loss")]  # pylint: disable=invalid-name
-        assert df.value.values[-1] < 1.5, "Loss is too high"
--- a/tests/e2e/patched/test_lora_llama_multipack.py
+++ b/tests/e2e/patched/test_lora_llama_multipack.py
@@ -56,8 +56,6 @@ class TestLoraLlama(unittest.TestCase):
                    },
                ],
                "num_epochs": 2,
-                "max_steps": 20,
-                "save_steps": 10,
                "micro_batch_size": 8,
                "gradient_accumulation_steps": 1,
                "output_dir": temp_dir,
@@ -111,7 +109,6 @@ class TestLoraLlama(unittest.TestCase):
                    },
                ],
                "num_epochs": 2,
-                "max_steps": 20,
                "save_steps": 0.5,
                "micro_batch_size": 8,
                "gradient_accumulation_steps": 1,
--- a/tests/e2e/patched/test_resume.py
+++ b/tests/e2e/patched/test_resume.py
@@ -6,6 +6,7 @@ import logging
 import os
 import re
 import subprocess
+import unittest
 from pathlib import Path

 from transformers.utils import is_torch_bf16_gpu_available
@@ -16,35 +17,35 @@ from axolotl.train import train
 from axolotl.utils.config import normalize_config
 from axolotl.utils.dict import DictDefault

-from ..utils import most_recent_subdir
+from ..utils import most_recent_subdir, with_temp_dir

 LOG = logging.getLogger("axolotl.tests.e2e")
 os.environ["WANDB_DISABLED"] = "true"


-class TestResumeLlama:
+class TestResumeLlama(unittest.TestCase):
    """
    Test case for resuming training of llama models
    """

-    def test_resume_lora_packed(self, temp_dir):
+    @with_temp_dir
+    def test_resume_qlora_packed(self, temp_dir):
        # pylint: disable=duplicate-code
        cfg = DictDefault(
            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "base_model": "JackFram/llama-68m",
+                "tokenizer_type": "LlamaTokenizer",
                "sequence_len": 1024,
                "sample_packing": True,
                "flash_attention": True,
-                "load_in_8bit": True,
-                "adapter": "lora",
-                "lora_r": 8,
-                "lora_alpha": 16,
+                "load_in_4bit": True,
+                "adapter": "qlora",
+                "lora_r": 32,
+                "lora_alpha": 64,
                "lora_dropout": 0.05,
                "lora_target_linear": True,
-                "val_set_size": 0.001,
-                "special_tokens": {
-                    "pad_token": "<|endoftext|>",
-                },
+                "val_set_size": 0.1,
+                "special_tokens": {},
                "datasets": [
                    {
                        "path": "vicgalle/alpaca-gpt4",
@@ -56,11 +57,11 @@ class TestResumeLlama:
                "gradient_accumulation_steps": 1,
                "output_dir": temp_dir,
                "learning_rate": 0.00001,
-                "optimizer": "adamw_8bit",
+                "optimizer": "adamw_torch",
                "lr_scheduler": "cosine",
-                "save_steps": 3,
+                "save_steps": 10,
                "save_total_limit": 5,
-                "max_steps": 15,
+                "max_steps": 40,
                "use_tensorboard": True,
            }
        )
@@ -76,7 +77,7 @@ class TestResumeLlama:

        resume_cfg = cfg | DictDefault(
            {
-                "resume_from_checkpoint": f"{temp_dir}/checkpoint-9/",
+                "resume_from_checkpoint": f"{temp_dir}/checkpoint-30/",
            }
        )
        normalize_config(resume_cfg)
@@ -92,4 +93,4 @@ class TestResumeLlama:
        )
        pattern = r"first_step\s+(\d+)"
        first_steps = int(re.findall(pattern, res.stdout)[0])
-        assert first_steps == 10
+        assert first_steps == 31
--- a/tests/e2e/patched/test_trainer_fsdp.py
+++ b/tests/e2e/patched/test_trainer_fsdp.py
@@ -0,0 +1,15 @@
+"""Test module for checking whether the integration of Unsloth with Hugging Face Transformers is working as expected."""
+import unittest
+
+from axolotl.monkeypatch.trainer_fsdp_grad_accum import check_training_loop_is_patchable
+
+
+class TestTrainerFSDPIntegration(unittest.TestCase):
+    """Unsloth monkeypatch integration tests."""
+
+    def test_train_loop_patchable(self):
+        # ensures the current version of transformers has loss code that matches our patching code
+        self.assertTrue(
+            check_training_loop_is_patchable(),
+            "HF transformers _inner_training_loop has changed and isn't patchable",
+        )
--- a/tests/e2e/patched/test_unsloth_qlora.py
+++ b/tests/e2e/patched/test_unsloth_qlora.py
@@ -1,186 +0,0 @@
-"""
-e2e tests for unsloth qlora
-"""
-import logging
-import os
-from pathlib import Path
-
-import pytest
-from e2e.utils import most_recent_subdir
-from tbparse import SummaryReader
-
-from axolotl.cli import load_datasets
-from axolotl.common.cli import TrainerCliArgs
-from axolotl.train import train
-from axolotl.utils.config import normalize_config
-from axolotl.utils.dict import DictDefault
-
-LOG = logging.getLogger("axolotl.tests.e2e")
-os.environ["WANDB_DISABLED"] = "true"
-
-
-# pylint: disable=duplicate-code
-class TestUnslothQLoRA:
-    """
-    Test class for Unsloth QLoRA Llama models
-    """
-
-    @pytest.mark.parametrize(
-        "sample_packing",
-        [True, False],
-    )
-    def test_unsloth_llama_qlora_fa2(self, temp_dir, sample_packing):
-        cfg = DictDefault(
-            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
-                "sequence_len": 1024,
-                "sample_packing": sample_packing,
-                "flash_attention": True,
-                "load_in_4bit": True,
-                "adapter": "qlora",
-                "lora_r": 16,
-                "lora_alpha": 16,
-                "lora_dropout": 0.05,
-                "lora_target_linear": True,
-                "val_set_size": 0.05,
-                "special_tokens": {
-                    "pad_token": "<|endoftext|>",
-                },
-                "datasets": [
-                    {
-                        "path": "mhenrichsen/alpaca_2k_test",
-                        "type": "alpaca",
-                    },
-                ],
-                "num_epochs": 1,
-                "max_steps": 5,
-                "save_steps": 10,
-                "micro_batch_size": 4,
-                "gradient_accumulation_steps": 2,
-                "output_dir": temp_dir,
-                "learning_rate": 0.00001,
-                "optimizer": "adamw_8bit",
-                "lr_scheduler": "cosine",
-                "use_tensorboard": True,
-                "bf16": "auto",
-            }
-        )
-
-        normalize_config(cfg)
-        cli_args = TrainerCliArgs()
-        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
-
-        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        assert (Path(temp_dir) / "adapter_model.bin").exists()
-
-        tb_log_path = most_recent_subdir(temp_dir + "/runs")
-        event_file = os.path.join(tb_log_path, sorted(os.listdir(tb_log_path))[0])
-        reader = SummaryReader(event_file)
-        df = reader.scalars  # pylint: disable=invalid-name
-        df = df[(df.tag == "train/train_loss")]  # pylint: disable=invalid-name
-        assert df.value.values[-1] < 2.0, "Loss is too high"
-
-    def test_unsloth_llama_qlora_unpacked(self, temp_dir):
-        cfg = DictDefault(
-            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
-                "sequence_len": 1024,
-                "sample_packing": False,
-                "load_in_4bit": True,
-                "adapter": "qlora",
-                "lora_r": 16,
-                "lora_alpha": 16,
-                "lora_dropout": 0.05,
-                "lora_target_linear": True,
-                "val_set_size": 0.05,
-                "special_tokens": {
-                    "pad_token": "<|endoftext|>",
-                },
-                "datasets": [
-                    {
-                        "path": "mhenrichsen/alpaca_2k_test",
-                        "type": "alpaca",
-                    },
-                ],
-                "num_epochs": 1,
-                "max_steps": 5,
-                "save_steps": 10,
-                "micro_batch_size": 4,
-                "gradient_accumulation_steps": 2,
-                "output_dir": temp_dir,
-                "learning_rate": 0.00001,
-                "optimizer": "adamw_8bit",
-                "lr_scheduler": "cosine",
-                "use_tensorboard": True,
-                "bf16": "auto",
-            }
-        )
-
-        normalize_config(cfg)
-        cli_args = TrainerCliArgs()
-        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
-
-        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        assert (Path(temp_dir) / "adapter_model.bin").exists()
-
-        tb_log_path = most_recent_subdir(temp_dir + "/runs")
-        event_file = os.path.join(tb_log_path, sorted(os.listdir(tb_log_path))[0])
-        reader = SummaryReader(event_file)
-        df = reader.scalars  # pylint: disable=invalid-name
-        df = df[(df.tag == "train/train_loss")]  # pylint: disable=invalid-name
-        assert df.value.values[-1] < 2.0, "Loss is too high"
-
-    @pytest.mark.parametrize(
-        "sdp_attention",
-        [True, False],
-    )
-    def test_unsloth_llama_qlora_unpacked_no_fa2_fp16(self, temp_dir, sdp_attention):
-        cfg = DictDefault(
-            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
-                "sequence_len": 1024,
-                "sample_packing": False,
-                "load_in_4bit": True,
-                "adapter": "qlora",
-                "lora_r": 16,
-                "lora_alpha": 16,
-                "lora_dropout": 0.05,
-                "lora_target_linear": True,
-                "val_set_size": 0.05,
-                "special_tokens": {
-                    "pad_token": "<|endoftext|>",
-                },
-                "datasets": [
-                    {
-                        "path": "mhenrichsen/alpaca_2k_test",
-                        "type": "alpaca",
-                    },
-                ],
-                "num_epochs": 1,
-                "max_steps": 5,
-                "save_steps": 10,
-                "micro_batch_size": 4,
-                "gradient_accumulation_steps": 2,
-                "sdp_attention": sdp_attention,
-                "output_dir": temp_dir,
-                "learning_rate": 0.00001,
-                "optimizer": "adamw_8bit",
-                "lr_scheduler": "cosine",
-                "use_tensorboard": True,
-                "fp16": True,
-            }
-        )
-
-        normalize_config(cfg)
-        cli_args = TrainerCliArgs()
-        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
-
-        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        assert (Path(temp_dir) / "adapter_model.bin").exists()
-
-        tb_log_path = most_recent_subdir(temp_dir + "/runs")
-        event_file = os.path.join(tb_log_path, sorted(os.listdir(tb_log_path))[0])
-        reader = SummaryReader(event_file)
-        df = reader.scalars  # pylint: disable=invalid-name
-        df = df[(df.tag == "train/train_loss")]  # pylint: disable=invalid-name
-        assert df.value.values[-1] < 2.0, "Loss is too high"
--- a/tests/e2e/test_embeddings_lr.py
+++ b/tests/e2e/test_embeddings_lr.py
@@ -1,121 +0,0 @@
-"""
-E2E tests for llama pretrain
-"""
-
-import logging
-import os
-import unittest
-from pathlib import Path
-
-from tbparse import SummaryReader
-
-from axolotl.cli import load_datasets
-from axolotl.common.cli import TrainerCliArgs
-from axolotl.train import train
-from axolotl.utils.config import normalize_config
-from axolotl.utils.dict import DictDefault
-
-from .utils import most_recent_subdir, with_temp_dir
-
-LOG = logging.getLogger("axolotl.tests.e2e")
-os.environ["WANDB_DISABLED"] = "true"
-
-
-class TestEmbeddingsLrScale(unittest.TestCase):
-    """
-    Test case for embedding_lr*
-    """
-
-    @with_temp_dir
-    def test_train_w_embedding_lr_scale(self, temp_dir):
-        # pylint: disable=duplicate-code
-        cfg = DictDefault(
-            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
-                "flash_attention": True,
-                "sequence_len": 1024,
-                "sample_packing": True,
-                "special_tokens": {
-                    "pad_token": "<|endoftext|>",
-                },
-                "datasets": [
-                    {
-                        "path": "mhenrichsen/alpaca_2k_test",
-                        "type": "alpaca",
-                    },
-                ],
-                "max_steps": 5,
-                "num_epochs": 1,
-                "micro_batch_size": 1,
-                "gradient_accumulation_steps": 1,
-                "val_set_size": 0.0,
-                "output_dir": temp_dir,
-                "learning_rate": 0.00001,
-                "optimizer": "adamw_torch",
-                "embedding_lr_scale": 0.5,
-                "lr_scheduler": "cosine",
-                "save_safetensors": True,
-                "bf16": "auto",
-                "use_tensorboard": True,
-            }
-        )
-        normalize_config(cfg)
-        cli_args = TrainerCliArgs()
-        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
-
-        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        assert (Path(temp_dir) / "model.safetensors").exists()
-
-        tb_log_path = most_recent_subdir(temp_dir + "/runs")
-        event_file = os.path.join(tb_log_path, sorted(os.listdir(tb_log_path))[0])
-        reader = SummaryReader(event_file)
-        df = reader.scalars  # pylint: disable=invalid-name
-        df = df[(df.tag == "train/train_loss")]  # pylint: disable=invalid-name
-        assert df.value.values[-1] < 2.0, "Loss is too high"
-
-    @with_temp_dir
-    def test_train_w_embedding_lr(self, temp_dir):
-        # pylint: disable=duplicate-code
-        cfg = DictDefault(
-            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
-                "flash_attention": True,
-                "sequence_len": 1024,
-                "sample_packing": True,
-                "special_tokens": {
-                    "pad_token": "<|endoftext|>",
-                },
-                "datasets": [
-                    {
-                        "path": "mhenrichsen/alpaca_2k_test",
-                        "type": "alpaca",
-                    },
-                ],
-                "max_steps": 5,
-                "num_epochs": 1,
-                "micro_batch_size": 1,
-                "gradient_accumulation_steps": 1,
-                "val_set_size": 0.0,
-                "output_dir": temp_dir,
-                "learning_rate": 0.00001,
-                "optimizer": "adamw_torch",
-                "embedding_lr": 0.000005,
-                "lr_scheduler": "cosine",
-                "save_safetensors": True,
-                "bf16": "auto",
-                "use_tensorboard": True,
-            }
-        )
-        normalize_config(cfg)
-        cli_args = TrainerCliArgs()
-        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
-
-        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        assert (Path(temp_dir) / "model.safetensors").exists()
-
-        tb_log_path = most_recent_subdir(temp_dir + "/runs")
-        event_file = os.path.join(tb_log_path, sorted(os.listdir(tb_log_path))[0])
-        reader = SummaryReader(event_file)
-        df = reader.scalars  # pylint: disable=invalid-name
-        df = df[(df.tag == "train/train_loss")]  # pylint: disable=invalid-name
-        assert df.value.values[-1] < 2.0, "Loss is too high"
--- a/tests/e2e/test_llama_vision.py
+++ b/tests/e2e/test_llama_vision.py
@@ -1,116 +0,0 @@
-"""
-E2E tests for lora llama
-"""
-
-import logging
-import os
-import unittest
-from pathlib import Path
-
-from axolotl.cli import load_datasets
-from axolotl.common.cli import TrainerCliArgs
-from axolotl.train import train
-from axolotl.utils.config import normalize_config
-from axolotl.utils.dict import DictDefault
-
-from .utils import with_temp_dir
-
-LOG = logging.getLogger("axolotl.tests.e2e")
-os.environ["WANDB_DISABLED"] = "true"
-
-
-class TestLlamaVision(unittest.TestCase):
-    """
-    Test case for Llama Vision models
-    """
-
-    @with_temp_dir
-    def test_lora_llama_vision_text_only_dataset(self, temp_dir):
-        # pylint: disable=duplicate-code
-        cfg = DictDefault(
-            {
-                "base_model": "axolotl-ai-co/Llama-3.2-39M-Vision",
-                "processor_type": "AutoProcessor",
-                "skip_prepare_dataset": True,
-                "remove_unused_columns": False,
-                "sample_packing": False,
-                "sequence_len": 1024,
-                "adapter": "lora",
-                "lora_r": 8,
-                "lora_alpha": 16,
-                "lora_dropout": 0.05,
-                "lora_target_modules": r"language_model.model.layers.[\d]+.(mlp|cross_attn|self_attn).(up|down|gate|q|k|v|o)_proj",
-                "val_set_size": 0,
-                "chat_template": "llama3_2_vision",
-                "datasets": [
-                    {
-                        "path": "LDJnr/Puffin",
-                        "type": "chat_template",
-                        "field_messages": "conversations",
-                        "message_field_role": "from",
-                        "message_field_content": "value",
-                    },
-                ],
-                "num_epochs": 1,
-                "micro_batch_size": 1,
-                "gradient_accumulation_steps": 4,
-                "output_dir": temp_dir,
-                "learning_rate": 0.00001,
-                "optimizer": "adamw_bnb_8bit",
-                "lr_scheduler": "cosine",
-                "max_steps": 5,
-                "save_safetensors": True,
-                "bf16": True,
-            }
-        )
-        normalize_config(cfg)
-        cli_args = TrainerCliArgs()
-        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
-
-        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        assert (Path(temp_dir) / "adapter_model.safetensors").exists()
-
-    @with_temp_dir
-    def test_lora_llama_vision_multimodal_dataset(self, temp_dir):
-        # pylint: disable=duplicate-code
-        cfg = DictDefault(
-            {
-                "base_model": "axolotl-ai-co/Llama-3.2-39M-Vision",
-                "processor_type": "AutoProcessor",
-                "skip_prepare_dataset": True,
-                "remove_unused_columns": False,
-                "sample_packing": False,
-                "sequence_len": 1024,
-                "adapter": "lora",
-                "lora_r": 8,
-                "lora_alpha": 16,
-                "lora_dropout": 0.05,
-                "lora_target_modules": r"language_model.model.layers.[\d]+.(mlp|cross_attn|self_attn).(up|down|gate|q|k|v|o)_proj",
-                "val_set_size": 0,
-                "chat_template": "llama3_2_vision",
-                "datasets": [
-                    {
-                        "path": "axolotl-ai-co/llava-instruct-mix-vsft-small",
-                        "type": "chat_template",
-                        "split": "train",
-                        "field_messages": "messages",
-                    },
-                ],
-                "num_epochs": 1,
-                "micro_batch_size": 1,
-                "gradient_accumulation_steps": 4,
-                "output_dir": temp_dir,
-                "learning_rate": 0.00001,
-                "optimizer": "adamw_bnb_8bit",
-                "lr_scheduler": "cosine",
-                "max_steps": 5,
-                "save_safetensors": True,
-                "bf16": True,
-            }
-        )
-        normalize_config(cfg)
-        cli_args = TrainerCliArgs()
-        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
-
-        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        assert (Path(temp_dir) / "adapter_model.safetensors").exists()
--- a/tests/e2e/test_lora_llama.py
+++ b/tests/e2e/test_lora_llama.py
@@ -57,7 +57,6 @@ class TestLoraLlama(unittest.TestCase):
                "learning_rate": 0.00001,
                "optimizer": "adamw_torch",
                "lr_scheduler": "cosine",
-                "max_steps": 20,
            }
        )
        normalize_config(cfg)
--- a/tests/e2e/test_optimizers.py
+++ b/tests/e2e/test_optimizers.py
@@ -56,7 +56,6 @@ class TestCustomOptimizers(unittest.TestCase):
                "output_dir": temp_dir,
                "learning_rate": 0.00001,
                "optimizer": "optimi_adamw",
-                "max_steps": 5,
                "lr_scheduler": "cosine",
            }
        )
@@ -95,7 +94,6 @@ class TestCustomOptimizers(unittest.TestCase):
                    },
                ],
                "num_epochs": 1,
-                "max_steps": 5,
                "micro_batch_size": 8,
                "gradient_accumulation_steps": 1,
                "output_dir": temp_dir,
@@ -115,9 +113,9 @@ class TestCustomOptimizers(unittest.TestCase):
    def test_fft_schedule_free_adamw(self, temp_dir):
        cfg = DictDefault(
            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "base_model": "HuggingFaceTB/SmolLM-135M",
                "sequence_len": 1024,
-                "val_set_size": 0.01,
+                "val_set_size": 0.1,
                "special_tokens": {
                    "pad_token": "<|endoftext|>",
                },
@@ -128,14 +126,13 @@ class TestCustomOptimizers(unittest.TestCase):
                    },
                ],
                "num_epochs": 1,
-                "micro_batch_size": 2,
+                "micro_batch_size": 4,
                "gradient_accumulation_steps": 2,
                "output_dir": temp_dir,
                "learning_rate": 0.00001,
                "optimizer": "schedule_free_adamw",
                "lr_scheduler": "constant",
                "save_safetensors": True,
-                "max_steps": 10,
            }
        )
        # pylint: disable=duplicate-code
--- a/tests/e2e/test_packing_loss.py
+++ b/tests/e2e/test_packing_loss.py
@@ -31,7 +31,7 @@ class TestPackedLlama(unittest.TestCase):
        # pylint: disable=duplicate-code
        cfg = DictDefault(
            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
+                "base_model": "HuggingFaceTB/SmolLM-135M",
                "sequence_len": 1024,
                "sample_packing": True,
                "flash_attention": True,
--- a/tests/e2e/test_relora_llama.py
+++ b/tests/e2e/test_relora_llama.py
@@ -7,15 +7,13 @@ import os
 import unittest
 from pathlib import Path

-from tbparse import SummaryReader
-
 from axolotl.cli import load_datasets
 from axolotl.common.cli import TrainerCliArgs
 from axolotl.train import train
 from axolotl.utils.config import normalize_config
 from axolotl.utils.dict import DictDefault

-from .utils import most_recent_subdir, with_temp_dir
+from .utils import with_temp_dir

 LOG = logging.getLogger("axolotl.tests.e2e")
 os.environ["WANDB_DISABLED"] = "true"
@@ -31,48 +29,35 @@ class TestReLoraLlama(unittest.TestCase):
        # pylint: disable=duplicate-code
        cfg = DictDefault(
            {
-                "base_model": "HuggingFaceTB/SmolLM2-135M",
-                "sequence_len": 2048,
-                "sample_packing": True,
-                "pad_to_sequence_len": True,
-                "flash_attention": True,
+                "base_model": "JackFram/llama-68m",
+                "tokenizer_type": "LlamaTokenizer",
+                "sequence_len": 1024,
                "load_in_8bit": True,
                "adapter": "lora",
-                "lora_r": 8,
+                "lora_r": 32,
                "lora_alpha": 16,
                "lora_dropout": 0.05,
                "lora_target_modules": ["q_proj", "v_proj"],
-                "relora_steps": 100,
-                "relora_warmup_steps": 20,
-                "relora_anneal_steps": 10,
-                "relora_prune_ratio": 0.9,
+                "relora_steps": 25,
+                "relora_warmup_steps": 5,
+                "relora_anneal_steps": 5,
                "relora_cpu_offload": True,
                "val_set_size": 0.0,
-                "special_tokens": {
-                    "pad_token": "<|endoftext|>",
-                },
-                "chat_template": "chatml",
+                "special_tokens": {},
                "datasets": [
                    {
-                        "path": "mlabonne/FineTome-100k",
-                        "type": "chat_template",
-                        "split": "train[:10%]",
-                        "field_messages": "conversations",
-                        "message_field_role": "from",
-                        "message_field_content": "value",
+                        "path": "mhenrichsen/alpaca_2k_test",
+                        "type": "alpaca",
                    },
                ],
-                "warmup_steps": 20,
+                "warmup_steps": 15,
                "num_epochs": 2,
-                "max_steps": 205,  # at least 2x relora_steps
-                "micro_batch_size": 2,
+                "micro_batch_size": 4,
                "gradient_accumulation_steps": 1,
                "output_dir": temp_dir,
                "learning_rate": 0.00001,
-                "optimizer": "adamw_8bit",
+                "optimizer": "adamw_torch",
                "lr_scheduler": "cosine",
-                "save_safetensors": True,
-                "use_tensorboard": True,
            }
        )
        normalize_config(cfg)
@@ -80,14 +65,4 @@ class TestReLoraLlama(unittest.TestCase):
        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)

        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        assert (
-            Path(temp_dir) / "checkpoint-100/adapter/adapter_model.safetensors"
-        ).exists()
-        assert (Path(temp_dir) / "checkpoint-100/relora/model.safetensors").exists()
-
-        tb_log_path = most_recent_subdir(temp_dir + "/runs")
-        event_file = os.path.join(tb_log_path, sorted(os.listdir(tb_log_path))[0])
-        reader = SummaryReader(event_file)
-        df = reader.scalars  # pylint: disable=invalid-name
-        df = df[(df.tag == "train/grad_norm")]  # pylint: disable=invalid-name
-        assert df.value.values[-1] < 0.2, "grad_norm is too high"
+        assert (Path(temp_dir) / "model.safetensors").exists()
--- a/tests/e2e/utils.py
+++ b/tests/e2e/utils.py
@@ -53,7 +53,7 @@ def require_torch_2_3_1(test_case):

 def require_torch_2_5_1(test_case):
    """
-    Decorator marking a test that requires torch >= 2.5.1
+    Decorator marking a test that requires torch >= 2.3.1
    """

    def is_min_2_5_1():
--- a/tests/prompt_strategies/init.py
+++ b/tests/prompt_strategies/init.py
--- a/tests/test_datasets.py
+++ b/tests/test_datasets.py
@@ -7,11 +7,6 @@ import tempfile
 import unittest
 from pathlib import Path

-from constants import (
-    ALPACA_MESSAGES_CONFIG_OG,
-    ALPACA_MESSAGES_CONFIG_REVISION,
-    SPECIAL_TOKENS,
-)
 from datasets import Dataset
 from huggingface_hub import snapshot_download
 from transformers import AutoTokenizer
@@ -26,7 +21,13 @@ class TestDatasetPreparation(unittest.TestCase):

    def setUp(self) -> None:
        self.tokenizer = AutoTokenizer.from_pretrained("huggyllama/llama-7b")
-        self.tokenizer.add_special_tokens(SPECIAL_TOKENS)
+        self.tokenizer.add_special_tokens(
+            {
+                "bos_token": "<s>",
+                "eos_token": "</s>",
+                "unk_token": "<unk>",
+            }
+        )
        # Alpaca dataset.
        self.dataset = Dataset.from_list(
            [
@@ -67,7 +68,7 @@ class TestDatasetPreparation(unittest.TestCase):
    def test_load_local_hub(self):
        """Niche use case.  Verify that a local copy of a hub dataset can be loaded"""
        with tempfile.TemporaryDirectory() as tmp_dir:
-            tmp_ds_path = Path(tmp_dir) / "mhenrichsen/alpaca_2k_test"
+            tmp_ds_path = Path("mhenrichsen/alpaca_2k_test")
            tmp_ds_path.mkdir(parents=True, exist_ok=True)
            snapshot_download(
                repo_id="mhenrichsen/alpaca_2k_test",
@@ -89,7 +90,7 @@ class TestDatasetPreparation(unittest.TestCase):
                            "ds_type": "parquet",
                            "type": "alpaca",
                            "data_files": [
-                                f"{tmp_ds_path}/alpaca_2000.parquet",
+                                "mhenrichsen/alpaca_2k_test/alpaca_2000.parquet",
                            ],
                        },
                    ],
@@ -276,7 +277,23 @@ class TestDatasetPreparation(unittest.TestCase):
                "sequence_len": 1024,
                "rl": "dpo",
                "chat_template": "llama3",
-                "datasets": [ALPACA_MESSAGES_CONFIG_OG],
+                "datasets": [
+                    {
+                        "path": "fozziethebeat/alpaca_messages_2k_dpo_test",
+                        "type": "chat_template.default",
+                        "chat_template": "llama3",
+                        "field_messages": "conversation",
+                        "field_chosen": "chosen",
+                        "field_rejected": "rejected",
+                        "message_field_role": "role",
+                        "message_field_content": "content",
+                        "roles": {
+                            "system": ["system"],
+                            "user": ["user"],
+                            "assistant": ["assistant"],
+                        },
+                    }
+                ],
            }
        )

@@ -325,7 +342,24 @@ class TestDatasetPreparation(unittest.TestCase):
                "sequence_len": 1024,
                "rl": "dpo",
                "chat_template": "llama3",
-                "datasets": [ALPACA_MESSAGES_CONFIG_REVISION],
+                "datasets": [
+                    {
+                        "path": "fozziethebeat/alpaca_messages_2k_dpo_test",
+                        "type": "chat_template.default",
+                        "chat_template": "llama3",
+                        "revision": "ea82cff",
+                        "field_messages": "conversation",
+                        "field_chosen": "chosen",
+                        "field_rejected": "rejected",
+                        "message_field_role": "role",
+                        "message_field_content": "content",
+                        "roles": {
+                            "system": ["system"],
+                            "user": ["user"],
+                            "assistant": ["assistant"],
+                        },
+                    }
+                ],
            }
        )

--- a/tests/test_exact_deduplication.py
+++ b/tests/test_exact_deduplication.py
@@ -1,433 +0,0 @@
-"""
-Test suite for functions in the axolotl.utils.data.utils module, focusing on the deduplicate_and_log_datasets function.
-
-Additionally, this test suite includes tests for functions that indirectly call deduplicate_and_log_datasets during the execution of the preprocess command.
-"""
-import hashlib
-import unittest
-from unittest.mock import patch
-
-from constants import ALPACA_MESSAGES_CONFIG_REVISION, SPECIAL_TOKENS
-from datasets import Dataset
-from transformers import AutoTokenizer
-
-from axolotl.utils.data import prepare_dataset
-from axolotl.utils.data.rl import load_prepare_dpo_datasets
-from axolotl.utils.data.utils import deduplicate_and_log_datasets
-from axolotl.utils.dict import DictDefault
-from axolotl.utils.models import load_processor, load_tokenizer
-
-
-def verify_deduplication(actual_dataset, expected_dataset, dataset_name):
-    """
-    Validates deduplication results and size consistency.
-
-    Parameters:
-    - actual_dataset: Deduplicated dataset.
-    - expected_dataset: Expected dataset.
-    - dataset_name: Name of the dataset (e.g., 'train' or 'eval').
-
-    Asserts:
-    - Datasets match in content.
-    - Dataset size matches unique row count.
-    """
-    # Convert datasets to sets of tuples for unordered comparison
-    actual_rows = set(tuple(row.values()) for row in actual_dataset)
-    expected_rows = set(tuple(row.values()) for row in expected_dataset)
-
-    # Verify deduplication correctness
-    assert actual_rows == expected_rows, f"Mismatch in {dataset_name} dataset"
-
-    # Verify size consistency
-    assert len(actual_rows) == len(
-        actual_dataset
-    ), f"Size mismatch in {dataset_name} dataset after deduplication"
-
-
-class TestDeduplicateIndividualFunctions(unittest.TestCase):
-    """
-    test class for deduplication function in data utils
-    """
-
-    def setUp(self):
-        # Sample data with duplicates
-        self.data = {
-            "column1": ["apple", "banana", "apple", "orange", "banana"],
-            "column2": [1, 2, 1, 3, 2],
-            "column3": ["red", "yellow", "red", "orange", "yellow"],
-        }
-
-        # Expected result after deduplication
-        self.expected_data = {
-            "column1": ["apple", "banana", "orange"],
-            "column2": [1, 2, 3],
-            "column3": ["red", "yellow", "orange"],
-        }
-
-        # Convert to Dataset format
-        self.dataset = Dataset.from_dict(self.data)
-        self.expected_dataset = Dataset.from_dict(self.expected_data)
-
-    def test_deduplication(self):
-        train_dataset, _, _ = deduplicate_and_log_datasets(train_dataset=self.dataset)
-        _, eval_dataset, _ = deduplicate_and_log_datasets(eval_dataset=self.dataset)
-
-        verify_deduplication(train_dataset, self.expected_dataset, "train_dataset")
-        verify_deduplication(eval_dataset, self.expected_dataset, "eval_dataset")
-
-    def test_datasets_are_none(self):
-        # Test when both datasets are None
-        train_dataset, eval_dataset, _ = deduplicate_and_log_datasets(
-            train_dataset=None, eval_dataset=None
-        )
-        self.assertIsNone(train_dataset, "Expected train_dataset to be None")
-        self.assertIsNone(eval_dataset, "Expected eval_dataset to be None")
-
-    def test_only_train_is_none(self):
-        # Test when only train_dataset is None
-        train_dataset, eval_dataset, _ = deduplicate_and_log_datasets(
-            train_dataset=None, eval_dataset=self.dataset
-        )
-        self.assertIsNone(train_dataset, "Expected train_dataset to be None")
-        verify_deduplication(eval_dataset, self.expected_dataset, "eval_dataset")
-
-    def test_only_eval_is_none(self):
-        # Test when only eval_dataset is None
-        train_dataset, eval_dataset, _ = deduplicate_and_log_datasets(
-            train_dataset=self.dataset, eval_dataset=None
-        )
-        self.assertIsNone(eval_dataset, "Expected eval_dataset to be None")
-        verify_deduplication(train_dataset, self.expected_dataset, "train_dataset")
-
-    def test_exact_duplicates(self):
-        # Test when datasets are exact duplicates
-        duplicate_data = {
-            "column1": ["apple", "apple", "apple"],
-            "column2": [1, 1, 1],
-            "column3": ["red", "red", "red"],
-        }
-        expected_data = {"column1": ["apple"], "column2": [1], "column3": ["red"]}
-
-        # Convert to Dataset format
-        dataset = Dataset.from_dict(duplicate_data)
-        expected_dataset = Dataset.from_dict(expected_data)
-
-        # Run deduplication
-        train_dataset, _, _ = deduplicate_and_log_datasets(train_dataset=dataset)
-        _, eval_dataset, _ = deduplicate_and_log_datasets(eval_dataset=dataset)
-
-        verify_deduplication(train_dataset, expected_dataset, "train_dataset")
-        verify_deduplication(eval_dataset, expected_dataset, "eval_dataset")
-
-    def test_partial_duplicates(self):
-        # Test when only part of the dataset is a duplicate
-        partial_duplicate_data = {
-            "column1": ["apple", "banana", "apple"],
-            "column2": [1, 2, 1],
-            "column3": ["red", "yellow", "red"],
-        }
-        expected_data = {
-            "column1": ["apple", "banana"],
-            "column2": [1, 2],
-            "column3": ["red", "yellow"],
-        }
-
-        # Convert to Dataset format
-        dataset = Dataset.from_dict(partial_duplicate_data)
-        expected_dataset = Dataset.from_dict(expected_data)
-
-        # Run deduplication
-        train_dataset, _, _ = deduplicate_and_log_datasets(train_dataset=dataset)
-        _, eval_dataset, _ = deduplicate_and_log_datasets(eval_dataset=dataset)
-
-        verify_deduplication(train_dataset, expected_dataset, "train_dataset")
-        verify_deduplication(eval_dataset, expected_dataset, "eval_dataset")
-
-    def test_combined_duplicates_empty(self):
-        # Test when only part of the dataset is a duplicate
-        partial_duplicate_data = {
-            "column1": ["apple", "banana", "apple"],
-            "column2": [1, 2, 1],
-            "column3": ["red", "yellow", "red"],
-        }
-        expected_data_train = {
-            "column1": ["apple", "banana"],
-            "column2": [1, 2],
-            "column3": ["red", "yellow"],
-        }
-        expected_data_eval = {
-            "column1": [],
-            "column2": [],
-            "column3": [],
-        }
-
-        # Convert to Dataset format
-        dataset = Dataset.from_dict(partial_duplicate_data)
-        expected_dataset_train = Dataset.from_dict(expected_data_train)
-        expected_dataset_eval = Dataset.from_dict(expected_data_eval)
-
-        # Run deduplication
-        train_dataset, eval_dataset, _ = deduplicate_and_log_datasets(
-            train_dataset=dataset, eval_dataset=dataset
-        )
-
-        verify_deduplication(train_dataset, expected_dataset_train, "train_dataset")
-        verify_deduplication(eval_dataset, expected_dataset_eval, "eval_dataset")
-
-    def test_combined_duplicates_one(self):
-        # Test when only part of the dataset is a duplicate
-        partial_duplicate_data_train = {
-            "column1": ["apple", "banana", "apple"],
-            "column2": [1, 2, 1],
-            "column3": ["red", "yellow", "red"],
-        }
-        partial_duplicate_data_eval = {
-            "column1": ["apple", "orange", "apple"],
-            "column2": [1, 2, 1],
-            "column3": ["red", "orange", "red"],
-        }
-        expected_data_train = {
-            "column1": ["apple", "banana"],
-            "column2": [1, 2],
-            "column3": ["red", "yellow"],
-        }
-        expected_data_eval = {
-            "column1": ["orange"],
-            "column2": [2],
-            "column3": ["orange"],
-        }
-
-        # Convert to Dataset format
-        dataset_train = Dataset.from_dict(partial_duplicate_data_train)
-        dataset_eval = Dataset.from_dict(partial_duplicate_data_eval)
-        expected_dataset_train = Dataset.from_dict(expected_data_train)
-        expected_dataset_eval = Dataset.from_dict(expected_data_eval)
-
-        # Run deduplication
-        train_dataset, eval_dataset, _ = deduplicate_and_log_datasets(
-            train_dataset=dataset_train, eval_dataset=dataset_eval
-        )
-
-        verify_deduplication(train_dataset, expected_dataset_train, "train_dataset")
-        verify_deduplication(eval_dataset, expected_dataset_eval, "eval_dataset")
-
-
-class TestDeduplicateRLDataset(unittest.TestCase):
-    """Test a configured dataloader with deduplication."""
-
-    def setUp(self) -> None:
-        self.tokenizer = AutoTokenizer.from_pretrained("huggyllama/llama-7b")
-        self.tokenizer.add_special_tokens(SPECIAL_TOKENS)
-        self.cfg = DictDefault(
-            {
-                "tokenizer_config": "huggyllama/llama-7b",
-                "sequence_len": 1024,
-                "rl": "dpo",
-                "chat_template": "llama3",
-                "dataset_exact_deduplication": True,
-                "datasets": [
-                    ALPACA_MESSAGES_CONFIG_REVISION,
-                    ALPACA_MESSAGES_CONFIG_REVISION,
-                ],
-            }
-        )
-
-    def test_load_with_deduplication(self):
-        """Verify that loading with deduplication removes duplicates."""
-
-        # Load the dataset using the deduplication setting
-        train_dataset, _ = load_prepare_dpo_datasets(self.cfg)
-
-        # Verify that the dataset has been deduplicated
-        assert len(train_dataset) == 1800, "Dataset was not properly deduplicated"
-
-    def test_load_without_deduplication(self):
-        """Verify that loading without deduplication retains duplicates."""
-        self.cfg.dataset_exact_deduplication = False
-        # Load the dataset without deduplication
-        train_dataset, _ = load_prepare_dpo_datasets(self.cfg)
-
-        # Verify that the dataset retains duplicates
-        assert (
-            len(train_dataset) == 1800 * 2
-        ), "Dataset deduplication occurred when it should not have"
-
-
-class TestDeduplicateNonRL(unittest.TestCase):
-    """Test prepare_dataset function with different configurations."""
-
-    def setUp(self) -> None:
-        self.tokenizer = AutoTokenizer.from_pretrained("huggyllama/llama-7b")
-        self.tokenizer.add_special_tokens(SPECIAL_TOKENS)
-        self.cfg_1 = DictDefault(
-            {
-                "tokenizer_config": "huggyllama/llama-7b",
-                "sequence_len": 1024,
-                "dataset_exact_deduplication": True,
-                "datasets": [
-                    {
-                        "path": "mhenrichsen/alpaca_2k_test",
-                        "type": "alpaca",
-                    },
-                    {
-                        "path": "mhenrichsen/alpaca_2k_test",
-                        "type": "alpaca",
-                    },
-                ],
-                "val_set_size": 0.0,
-                "gradient_accumulation_steps": 4,
-                "batch_size": 10,
-                "micro_batch_size": 10,
-                "num_epochs": 1,
-            }
-        )
-
-    def test_prepare_dataset_with_deduplication_train(self):
-        """Verify that prepare_dataset function processes the dataset correctly with deduplication."""
-        self.cfg_1.dataset_exact_deduplication = True
-
-        # Load tokenizer and processor
-        tokenizer = load_tokenizer(self.cfg_1)
-        processor = (
-            load_processor(self.cfg_1, tokenizer=tokenizer)
-            if self.cfg_1.processor_type
-            else None
-        )
-
-        # Prepare dataset using the prepare_dataset function
-        train_dataset, _, _, _ = prepare_dataset(
-            self.cfg_1,
-            tokenizer,
-            processor=processor,
-        )
-
-        self.assertEqual(
-            len(train_dataset),
-            2000,
-            "Train dataset should have 2000 samples after deduplication.",
-        )
-
-    def test_prepare_dataset_with_deduplication_eval(self):
-        """Verify that prepare_dataset function processes the dataset correctly with deduplication."""
-        self.cfg_1.dataset_exact_deduplication = True
-        self.cfg_1.val_set_size = 0.5
-        # Load tokenizer and processor
-        tokenizer = load_tokenizer(self.cfg_1)
-        processor = (
-            load_processor(self.cfg_1, tokenizer=tokenizer)
-            if self.cfg_1.processor_type
-            else None
-        )
-
-        # Prepare dataset using the prepare_dataset function
-        _, eval_dataset, _, _ = prepare_dataset(
-            self.cfg_1,
-            tokenizer,
-            processor=processor,
-        )
-
-        self.assertEqual(
-            len(eval_dataset),
-            1000,
-            "Eval dataset should have 2000 samples after deduplication.",
-        )
-
-    def test_prepare_dataset_without_deduplication(self):
-        """Verify that prepare_dataset function processes the dataset correctly without deduplication."""
-        self.cfg_1.dataset_exact_deduplication = False
-        self.cfg_1.val_set_size = 0.1
-        # Load tokenizer and processor
-        tokenizer = load_tokenizer(self.cfg_1)
-        processor = (
-            load_processor(self.cfg_1, tokenizer=tokenizer)
-            if self.cfg_1.processor_type
-            else None
-        )
-
-        # Prepare dataset using the prepare_dataset function
-        train_dataset, eval_dataset, _, _ = prepare_dataset(
-            self.cfg_1,
-            tokenizer,
-            processor=processor,
-        )
-
-        # Verify that the dataset has been prepared correctly
-        self.assertEqual(
-            len(train_dataset),
-            1800 * 2,
-            "Train dataset should have 3600 samples without deduplication.",
-        )
-        self.assertEqual(
-            len(eval_dataset),
-            200 * 2,
-            "Train dataset should have 400 samples after deduplication.",
-        )
-
-
-class TestWrongCollisions(unittest.TestCase):
-    """Creating mock datasets for testing wrong collisions"""
-
-    def setUp(self):
-        self.train_data = {"text": ["sample 5", "sample 6"], "label": [1, 2]}
-        self.eval_data = {
-            "text": [
-                "sample 5",
-                "sample 7",
-            ],  # Different label but same text as in train_data
-            "label": [2, 3],
-        }
-        self.dataset_data = {
-            "text": ["sample 5", "sample 9", "sample 5"],
-            "label": [1, 2, 8],
-        }
-        self.train_dataset = Dataset.from_dict(self.train_data)
-        self.eval_dataset = Dataset.from_dict(self.eval_data)
-        self.dataset = Dataset.from_dict(self.dataset_data)
-
-    @patch(
-        "axolotl.utils.data.utils.sha256",
-        side_effect=lambda x: hashlib.sha256(
-            "forced_collision_hash".encode("utf-8")
-        ).hexdigest()
-        if "sample 5" in x
-        else hashlib.sha256(x.encode("utf-8")).hexdigest(),
-    )
-    def test_deduplication_wrong_collision_train_eval(self, _mock_sha256):
-        dedup_train, dedup_eval, _ = deduplicate_and_log_datasets(
-            train_dataset=self.train_dataset, eval_dataset=self.eval_dataset
-        )
-        self.assertEqual(
-            len(dedup_train),
-            2,
-            "train dataset should not deduplicate rows with forced hash collisions but different labels.",
-        )
-        self.assertEqual(
-            len(dedup_eval),
-            2,
-            "Eval dataset should not deduplicate rows with forced hash collisions but different labels.",
-        )
-        self.assertEqual(
-            len(dedup_eval),
-            len(self.eval_dataset),
-            "The output eval dataset should have the same number of rows as the input eval dataset.",
-        )
-        self.assertEqual(
-            str(dedup_eval),
-            str(self.eval_dataset),
-            "The string representation of the output eval dataset should be identical to the input eval dataset.",
-        )
-
-    def test_deduplication_dataset_only(self):
-        _, _, dedup_dataset = deduplicate_and_log_datasets(dataset=self.dataset)
-        self.assertEqual(
-            len(dedup_dataset), 3, "Dataset should have all original values"
-        )
-        self.assertEqual(
-            str(dedup_dataset),
-            str(self.dataset),
-            "The string representation of the output dataset should not differ.",
-        )
-
-
-if __name__ == "__main__":
-    unittest.main()
--- a/tests/test_perplexity.py
+++ b/tests/test_perplexity.py
@@ -7,40 +7,35 @@ from transformers.models.auto.tokenization_auto import AutoTokenizer

 from axolotl.utils.callbacks.perplexity import Perplexity

-MODEL_NAME = "HuggingFaceTB/SmolLM2-135M"
+MODEL_NAME = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"


@fixture()
 def metric(tokenizer):
-    return Perplexity(tokenizer=tokenizer, max_seq_len=512)
+    model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, trust_remote_code=True)

-
-@fixture()
-def model():
-    return AutoModelForCausalLM.from_pretrained(MODEL_NAME, trust_remote_code=True)
+    return Perplexity(model, tokenizer, 512)


@fixture()
 def tokenizer():
-    tokenizer_ = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
-    tokenizer_.add_special_tokens({"pad_token": "<|endoftext|>"})
-    return tokenizer_
+    return AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)


-def test_perplexity_longer_than_stride(model, metric):
+def test_perplexity_longer_than_stride(metric):
    # taken from https://huggingface.co/datasets/roneneldan/TinyStories
    sample_text = """
 Once upon a time, there was a little car named Beep. Beep loved to go fast and play in the sun. Beep was a healthy car because he always had good fuel. Good fuel made Beep happy and strong. One day, Beep was driving in the park when he saw a big tree. The tree had many leaves that were falling. Beep liked how the leaves fall and wanted to play with them. Beep drove under the tree and watched the leaves fall on him. He laughed and beeped his horn. Beep played with the falling leaves all day. When it was time to go home, Beep knew he needed more fuel. He went to the fuel place and got more healthy fuel. Now, Beep was ready to go fast and play again the next day. And Beep lived happily ever after.
 One day, a little fish named Fin was swimming near the shore. He saw a big crab and wanted to be friends. "Hi, I am Fin. Do you want to play?" asked the little fish. The crab looked at Fin and said, "No, I don't want to play. I am cold and I don't feel fine." Fin felt sad but wanted to help the crab feel better. He swam away and thought of a plan. He remembered that the sun could make things warm. So, Fin swam to the top of the water and called to the sun, "Please, sun, help my new friend feel fine and not freeze!" The sun heard Fin's call and shone its warm light on the shore. The crab started to feel better and not so cold. He saw Fin and said, "Thank you, little fish, for making me feel fine. I don't feel like I will freeze now. Let's play together!" And so, Fin and the crab played and became good friends.
 """
-    result = metric.compute(model, [sample_text])
+    result = metric.compute([sample_text])
    ppl = result["score"]
-    assert round(ppl, 2) == 7.41
+    assert round(ppl, 2) == 5.37


-def test_perplexity_short(model, metric):
+def test_perplexity_short(metric):
    # taken from https://huggingface.co/datasets/roneneldan/TinyStories
    sample_text = "Once upon a time, there was a little car named Beep. Beep loved to go fast and play in the sun."
-    result = metric.compute(model, [sample_text])
+    result = metric.compute([sample_text])
    ppl = result["score"]
-    assert round(ppl, 2) == 10.33
+    assert round(ppl, 2) == 10.02
--- a/tests/test_schedulers.py
+++ b/tests/test_schedulers.py
@@ -32,19 +32,16 @@ class TestCosineConstantLr(unittest.TestCase):
    def test_schedulers(self):
        self.assertEqual(self.lr_scheduler.get_last_lr()[0], 0)
        for _ in range(self.warmup_steps):
-            self.optimizer.step()
            self.lr_scheduler.step()
        self.assertEqual(self.lr_scheduler.get_last_lr()[0], self._lr)
        constant_step = int(self.train_steps * self.constant_lr_ratio)
        remaining_step = self.train_steps - constant_step
        for _ in range(constant_step):
-            self.optimizer.step()
            self.lr_scheduler.step()
        self.assertEqual(
            self.lr_scheduler.get_last_lr()[0], self._lr * self.min_lr_ratio
        )
        for _ in range(remaining_step):
-            self.optimizer.step()
            self.lr_scheduler.step()
        self.assertEqual(
            self.lr_scheduler.get_last_lr()[0], self._lr * self.min_lr_ratio
--- a/tests/test_validation.py
+++ b/tests/test_validation.py
@@ -68,53 +68,6 @@ class TestValidation(BaseValidation):
        assert cfg.train_on_inputs is False
        assert cfg.weight_decay is None

-    def test_zero3_qlora_use_reentrant_false(self, minimal_cfg):
-        test_cfg = DictDefault(
-            {
-                "deepspeed": "deepspeed_configs/zero3_bf16.json",
-                "gradient_checkpointing": True,
-                "gradient_checkpointing_kwargs": {"use_reentrant": False},
-                "load_in_4bit": True,
-                "adapter": "qlora",
-            }
-            | minimal_cfg
-        )
-
-        with self._caplog.at_level(logging.WARNING):
-            validate_config(test_cfg)
-            assert (
-                "qlora + zero3 with use_reentrant: false may result in a CheckpointError about recomputed values"
-                in self._caplog.records[0].message
-            )
-
-    def test_deepspeed_empty(self, minimal_cfg):
-        test_cfg = DictDefault(
-            {
-                "deepspeed": "",
-                "gradient_checkpointing": True,
-                "gradient_checkpointing_kwargs": {"use_reentrant": False},
-                "load_in_4bit": True,
-                "adapter": "qlora",
-            }
-            | minimal_cfg
-        )
-
-        _ = validate_config(test_cfg)
-
-    def test_deepspeed_not_set(self, minimal_cfg):
-        test_cfg = DictDefault(
-            {
-                "deepspeed": None,
-                "gradient_checkpointing": True,
-                "gradient_checkpointing_kwargs": {"use_reentrant": False},
-                "load_in_4bit": True,
-                "adapter": "qlora",
-            }
-            | minimal_cfg
-        )
-
-        _ = validate_config(test_cfg)
-
    def test_datasets_min_length(self):
        cfg = DictDefault(
            {
@@ -672,9 +625,6 @@ class TestValidation(BaseValidation):
                {
                    "bf16": True,
                    "capabilities": {"bf16": False},
-                    "env_capabilities": {
-                        "torch_version": "2.5.1",
-                    },
                }
            )
            | minimal_cfg
@@ -1163,38 +1113,6 @@ class TestValidation(BaseValidation):
                in self._caplog.records[0].message
            )

-    def test_torch_version_adopt_req(self, minimal_cfg):
-        cfg = (
-            DictDefault(
-                {
-                    "optimizer": "adopt_adamw",
-                }
-            )
-            | minimal_cfg
-        )
-
-        with pytest.raises(
-            ValueError,
-            match=r".*ADOPT optimizer is incompatible with torch version*",
-        ):
-            env_capabilities = {"torch_version": "2.3.0"}
-            capabilities = {"bf16": False}
-            _ = validate_config(
-                cfg, capabilities=capabilities, env_capabilities=env_capabilities
-            )
-
-        env_capabilities = {"torch_version": "2.5.1"}
-        capabilities = {"bf16": False}
-        _ = validate_config(
-            cfg, capabilities=capabilities, env_capabilities=env_capabilities
-        )
-
-        env_capabilities = {"torch_version": "2.5.2"}
-        capabilities = {"bf16": False}
-        _ = validate_config(
-            cfg, capabilities=capabilities, env_capabilities=env_capabilities
-        )
-

 class TestValidationCheckModelConfig(BaseValidation):
    """
--- a/tests/test_validation_dataset.py
+++ b/tests/test_validation_dataset.py
@@ -72,9 +72,6 @@ class TestValidationCheckDatasetConfig(BaseValidation):
                "n_gpu": 1,
                "compute_capability": "8.0",
            },
-            env_capabilities={
-                "torch_version": "2.5.1",
-            },
        )

        _check_config()
@@ -127,9 +124,6 @@ class TestValidationCheckDatasetConfig(BaseValidation):
                "n_gpu": 1,
                "compute_capability": "8.0",
            },
-            env_capabilities={
-                "torch_version": "2.5.1",
-            },
        )

        _check_config()
@@ -183,9 +177,6 @@ class TestValidationCheckDatasetConfig(BaseValidation):
                "n_gpu": 1,
                "compute_capability": "8.0",
            },
-            env_capabilities={
-                "torch_version": "2.5.1",
-            },
        )

        _check_config()
@@ -240,9 +231,6 @@ class TestValidationCheckDatasetConfig(BaseValidation):
                "n_gpu": 1,
                "compute_capability": "8.0",
            },
-            env_capabilities={
-                "torch_version": "2.5.1",
-            },
        )

        _check_config()