Fix(debug): Use space delimiter for debug_text_only also

.github/FUNDING.yml

@@ -3,11 +3,11 @@
 github: OpenAccess-AI-Collective # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
 patreon: # Replace with a single Patreon username
 open_collective: # Replace with a single Open Collective username
-ko_fi: axolotl_ai # Replace with a single Ko-fi username
+ko_fi: # Replace with a single Ko-fi username
 tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
 community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
 liberapay: # Replace with a single Liberapay username
 issuehunt: # Replace with a single IssueHunt username
 otechie: # Replace with a single Otechie username
 lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
-custom: ['https://quickchart.io/qr?text=bitcoin%3Abc1qxlgwlqwfea5s2cxm42xqsfmwjct0rj8w8ea5np&size=480&centerImageUrl=https%3A%2F%2Fupload.wikimedia.org%2Fwikipedia%2Fcommons%2Fthumb%2F4%2F46%2FBitcoin.svg%2F64px-Bitcoin.svg.png'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/PULL_REQUEST_TEMPLATE/pull_request_template_simple.md

@@ -20,8 +20,3 @@
 ## Types of changes
 
 <!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
-
-## Social Handles (Optional)
-
-<!-- Thanks for submitting a bugfix or enhancement. -->
-<!-- We'd love to show our thanks to you on Twitter & Discord if you provide your handle -->

.github/workflows/lint.yml

@@ -1,22 +0,0 @@
-name: lint
-on:
-  # check on PRs, and manual triggers
-  pull_request:
-      paths:
-       - '**.py'
-       - 'requirements.txt'
-       - '.github/workflows/*.yml'
-       - "*.md"
-  workflow_dispatch:
-
-jobs:
-  pre-commit:
-    name: pre-commit
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.9"
-          cache: 'pip' # caching pip dependencies
-      - uses: pre-commit/action@v3.0.0

.github/workflows/pypi.yml

@@ -34,11 +34,11 @@ jobs:
         run: echo ::set-output name=TAG_NAME::$(echo $GITHUB_REF | cut -d / -f 3)
 
       - name: Update version in setup.py
-        run: |
+        run: >-
           sed -i -E 's/version="([0-9.]+)",/version="${{ steps.tag.outputs.TAG_NAME }}",/g' setup.py
 
       - name: Build a binary wheel
-        run: |
+        run: >-
           python setup.py sdist bdist_wheel
 
       - name: Publish package distributions to PyPI

.github/workflows/tests-docker.yml

@@ -0,0 +1,46 @@
+name: e2e-docker-tests
+
+on:
+  pull_request:
+    paths:
+      - '**.py'
+      - 'requirements.txt'
+      - '.github/workflows/*.yml'
+  workflow_dispatch:
+
+jobs:
+  build-axolotl:
+    if: github.repository_owner == 'OpenAccess-AI-Collective'
+    # this job needs to be run on self-hosted GPU runners...
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - cuda: 118
+            cuda_version: 11.8.0
+            python_version: "3.10"
+            pytorch: 2.0.1
+          - cuda: 121
+            cuda_version: 12.1.0
+            python_version: "3.10"
+            pytorch: 2.1.1
+    runs-on: [self-hosted, gpu, docker]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Build Docker image
+        run: |
+          # Set up build arguments
+          BASE_TAG="main-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}"
+          CUDA="${{ matrix.cuda }}"
+          PYTORCH_VERSION="${{ matrix.pytorch }}"
+          # Build the Docker image
+          docker build . \
+            --file ./docker/Dockerfile \
+            --build-arg BASE_TAG=$BASE_TAG \
+            --build-arg CUDA=$CUDA \
+            --build-arg PYTORCH_VERSION=$PYTORCH_VERSION \
+            --tag test-axolotl
+      - name: Unit Tests w docker image
+        run: |
+          docker run --rm test-axolotl pytest --ignore=tests/e2e/ /workspace/axolotl/tests/

.github/workflows/tests.yml

@@ -7,12 +7,10 @@ on:
     paths:
       - '**.py'
       - 'requirements.txt'
-      - '.github/workflows/*.yml'
   pull_request:
       paths:
        - '**.py'
        - 'requirements.txt'
-       - '.github/workflows/*.yml'
   workflow_dispatch:
 
 jobs:
@@ -33,7 +31,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.9", "3.10", "3.11"]
+        python_version: ["3.9", "3.10"]
     timeout-minutes: 10
 
     steps:
@@ -55,54 +53,29 @@ jobs:
         run: |
           pytest --ignore=tests/e2e/ tests/
 
-  docker-e2e-tests:
-    if: github.repository_owner == 'OpenAccess-AI-Collective'
-    # this job needs to be run on self-hosted GPU runners...
-    runs-on: [self-hosted, gpu, docker]
-    timeout-minutes: 30
+  e2e-test:
+    name: E2E Tests
+    runs-on: [self-hosted, gpu]
+    timeout-minutes: 20
     needs: [pre-commit, pytest]
 
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - cuda: 118
-            cuda_version: 11.8.0
-            python_version: "3.10"
-            pytorch: 2.0.1
-          - cuda: 121
-            cuda_version: 12.1.0
-            python_version: "3.10"
-            pytorch: 2.1.1
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Docker metadata
-        id: metadata
-        uses: docker/metadata-action@v5
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
         with:
-          images: winglian/axolotl-tests
-      - name: Build Docker image
+          python-version: "3.10"
+#          cache: 'pip' # caching pip dependencies
+
+      - name: Install dependencies
         run: |
-          # Set up build arguments
-          BASE_TAG="main-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}"
-          CUDA="${{ matrix.cuda }}"
-          PYTORCH_VERSION="${{ matrix.pytorch }}"
-          # Build the Docker image
-          docker build . \
-            --file ./docker/Dockerfile-tests \
-            --build-arg BASE_TAG=$BASE_TAG \
-            --build-arg CUDA=$CUDA \
-            --build-arg GITHUB_REF=$GITHUB_REF \
-            --build-arg PYTORCH_VERSION=$PYTORCH_VERSION \
-            --tag ${{ steps.metadata.outputs.tags }}-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }} \
-            --no-cache
-      - name: Unit Tests w docker image
+          pip3 install --extra-index-url https://download.pytorch.org/whl/cu118 -U torch==2.0.1
+          pip3 uninstall -y transformers accelerate
+          pip3 install -U -e .[flash-attn,mamba-ssm]
+          pip3 install -r requirements-tests.txt
+
+      - name: Run e2e tests
         run: |
-          docker run --rm ${{ steps.metadata.outputs.tags }}-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }} pytest --ignore=tests/e2e/ /workspace/axolotl/tests/
-      - name: GPU Unit Tests w docker image
-        run: |
-          docker run --privileged --gpus "all" --env WANDB_DISABLED=true --rm ${{ steps.metadata.outputs.tags }}-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }} pytest --ignore=tests/e2e/patched/ /workspace/axolotl/tests/e2e/
-      - name: GPU Unit Tests monkeypatched w docker image
-        run: |
-          docker run --privileged --gpus "all" --env WANDB_DISABLED=true --rm ${{ steps.metadata.outputs.tags }}-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }} pytest /workspace/axolotl/tests/e2e/patched/
+          pytest tests/e2e/

.gitignore

@@ -1,7 +1,5 @@
 **/axolotl.egg-info
 configs
-last_run_prepared/
-.vscode
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.vscode/README.md

@@ -1 +0,0 @@
-See [docs/debugging.md](../docs/debugging.md) for guidance on how to modify these files to debug axolotl with VSCode.

.vscode/launch.json

@@ -1,34 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Debug axolotl prompt - sharegpt",
-            "type": "python",
-            "module": "accelerate.commands.launch",
-            "request": "launch",
-            "args": [
-                "-m", "axolotl.cli.train", "dev_sharegpt.yml",
-                // The flags below simplify debugging by overriding the axolotl config
-                // with the debugging tips above.  Modify as needed.
-                "--dataset_processes=1",      // limits data preprocessing to one process
-                "--max_steps=1",              // limits training to just one step
-                "--batch_size=1",             // minimizes batch size
-                "--micro_batch_size=1",       // minimizes batch size
-                "--val_set_size=0",           // disables validation
-                "--sample_packing=False",     // disables sample packing which is necessary for small datasets
-                "--eval_sample_packing=False",// disables sample packing on eval set
-                "--dataset_prepared_path=temp_debug/axolotl_outputs/data", // send data outputs to a temp folder
-                "--output_dir=temp_debug/axolotl_outputs/model" // send model outputs to a temp folder
-                ],
-            "console": "integratedTerminal",      // show output in the integrated terminal
-            "cwd": "${workspaceFolder}/devtools", // set working directory to devtools from the root of the project
-            "justMyCode": true,                   // step through only axolotl code
-            "env": {"CUDA_VISIBLE_DEVICES": "0",  // Since we aren't doing distributed training, we need to limit to one GPU
-                    "HF_HOME": "${workspaceFolder}/devtools/temp_debug/.hf-cache"}, // send HF cache to a temp folder
-            "preLaunchTask": "cleanup-for-dataprep", // delete temp folders (see below)
-        }
-    ]
-}

.vscode/tasks.json

@@ -1,27 +0,0 @@
-//this file is used by launch.json
-{
-  "version": "2.0.0",
-  "tasks": [
-    // this task changes into the devtools directory and deletes the temp_debug/axolotl_outputs folder
-    {
-      "label": "delete-outputs",
-      "type": "shell",
-      "command": "rm -rf temp_debug/axolotl_outputs",
-      "options":{ "cwd": "${workspaceFolder}/devtools"},
-      "problemMatcher": []
-    },
-    // this task changes into the devtools directory and deletes the `temp_debug/.hf-cache/datasets` folder
-    {
-      "label": "delete-temp-hf-dataset-cache",
-      "type": "shell",
-      "command": "rm -rf temp_debug/.hf-cache/datasets",
-      "options":{ "cwd": "${workspaceFolder}/devtools"},
-      "problemMatcher": []
-    },
-      // this task combines the two tasks above
-    {
-     "label": "cleanup-for-dataprep",
-     "dependsOn": ["delete-outputs", "delete-temp-hf-dataset-cache"],
-    }
-  ]
-}

README.md

@@ -10,7 +10,7 @@ Features:
 - Integrated with xformer, flash attention, rope scaling, and multipacking
 - Works with single GPU or multiple GPUs via FSDP or Deepspeed
 - Easily run with Docker locally or on the cloud
-- Log results and optionally checkpoints to wandb or mlflow
+- Log results and optionally checkpoints to wandb
 - And more!
 
 
@@ -39,12 +39,10 @@ Features:
   - [Special Tokens](#special-tokens)
 - [Common Errors](#common-errors-)
   - [Tokenization Mismatch b/w Training & Inference](#tokenization-mismatch-bw-inference--training)
-- [Debugging Axolotl](#debugging-axolotl)
 - [Need Help?](#need-help-)
 - [Badge](#badge-)
 - [Community Showcase](#community-showcase)
 - [Contributing](#contributing-)
-- [Sponsors](#sponsors-)
 
 </td>
 <td>
@@ -132,9 +130,6 @@ accelerate launch -m axolotl.cli.inference examples/openllama-3b/lora.yml \
   docker compose up -d
   ```
 
->[!Tip]
-> If you want to debug axolotl or prefer to use Docker as your development environment, see the [debugging guide's section on Docker](docs/debugging.md#debugging-with-docker).
-
   <details>
 
   <summary>Docker advanced</summary>
@@ -142,7 +137,7 @@ accelerate launch -m axolotl.cli.inference examples/openllama-3b/lora.yml \
   A more powerful Docker command to run would be this:
 
   ```bash
-docker run --privileged --gpus '"all"' --shm-size 10g --rm -it --name axolotl --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 --mount type=bind,src="${PWD}",target=/workspace/axolotl -v ${HOME}/.cache/huggingface:/root/.cache/huggingface winglian/axolotl:main-py3.10-cu118-2.0.1
+  docker run --privileged --gpus '"all"' --shm-size 10g --rm -it --name axolotl --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 --mount type=volume,src=axolotl,target=/workspace/axolotl -v ${HOME}/.cache/huggingface:/root/.cache/huggingface winglian/axolotl:main-py3.10-cu118-2.0.1
   ```
 
   It additionally:
@@ -599,9 +594,6 @@ datasets:
       # For `completion` datsets only, uses the provided field instead of `text` column
       field:
 
-# use RL training: dpo, ipo, kto_pair
-rl:
-
 # Saves the desired chat template to the tokenizer_config.json for easier inferencing
 # Currently supports chatml and inst (mistral/mixtral)
 chat_template: chatml
@@ -702,10 +694,6 @@ wandb_name: # Set the name of your wandb run
 wandb_run_id: # Set the ID of your wandb run
 wandb_log_model: # "checkpoint" to log model to wandb Artifacts every `save_steps` or "end" to log only at the end of training
 
-# mlflow configuration if you're using it
-mlflow_tracking_uri: # URI to mlflow
-mlflow_experiment_name: # Your experiment name
-
 # Where to save the full-finetuned model to
 output_dir: ./completed-model
 
@@ -766,7 +754,6 @@ early_stopping_patience: 3
 # Specify a scheduler and kwargs to use with the optimizer
 lr_scheduler: # 'one_cycle' | 'log_sweep' | empty for cosine
 lr_scheduler_kwargs:
-cosine_min_lr_ratio: # decay lr to some percentage of the peak lr, e.g. cosine_min_lr_ratio=0.1 for 10% of peak lr
 
 # For one_cycle optim
 lr_div_factor: # Learning rate div factor
@@ -1070,7 +1057,7 @@ although this will be very slow, and using the config options above are recommen
 
 ## Common Errors 🧰
 
-See also the [FAQ's](./docs/faq.md) and [debugging guide](docs/debugging.md).
+See also the [FAQ's](./docs/faq.md).
 
 > If you encounter a 'Cuda out of memory' error, it means your GPU ran out of memory during the training process. Here's how to resolve it:
 
@@ -1120,10 +1107,6 @@ If you decode a prompt constructed by axolotl, you might see spaces between toke
 
 Having misalignment between your prompts during training and inference can cause models to perform very poorly, so it is worth checking this.  See [this blog post](https://hamel.dev/notes/llm/05_tokenizer_gotchas.html) for a concrete example.
 
-## Debugging Axolotl
-
-See [this debugging guide](docs/debugging.md) for tips on debugging Axolotl, along with an example configuration for debugging with VSCode.
-
 ## Need help? 🙋♂️
 
 Join our [Discord server](https://discord.gg/HhrNrHJPRb) where we can help you
@@ -1166,33 +1149,3 @@ pre-commit install
 # test
 pytest tests/
 ```
-
-## Sponsors 🤝❤
-
-OpenAccess AI Collective is run by volunteer contributors such as [winglian](https://github.com/winglian),
-[NanoCode012](https://github.com/NanoCode012), [tmm1](https://github.com/tmm1),
-[mhenrichsen](https://github.com/mhenrichsen), [casper-hansen](https://github.com/casper-hansen),
-[hamelsmu](https://github.com/hamelsmu) and many more who help us accelerate forward by fixing bugs, answering
-community questions and implementing new features. Axolotl needs donations from sponsors for the compute needed to
-run our unit & integration tests, troubleshooting community issues, and providing bounties. If you love axolotl,
-consider sponsoring the project via [GitHub Sponsors](https://github.com/sponsors/OpenAccess-AI-Collective),
-[Ko-fi](https://ko-fi.com/axolotl_ai) or reach out directly to
-[wing@openaccessaicollective.org](mailto:wing@openaccessaicollective.org).
-
----
-
-#### 💎 Diamond Sponsors - [Contact directly](mailto:wing@openaccessaicollective.org)
-
----
-
-#### 🥇 Gold Sponsors - $5000/mo
-
----
-
-#### 🥈 Silver Sponsors - $1000/mo
-
----
-
-#### 🥉 Bronze Sponsors - $500/mo
-
----

devtools/README.md

@@ -1 +0,0 @@
-This directory contains example config files that might be useful for debugging. Please see [docs/debugging.md](../docs/debugging.md) for more information.

devtools/dev_sharegpt.yml

@@ -1,49 +0,0 @@
-# Example config for debugging the sharegpt prompt format
-base_model: TinyLlama/TinyLlama-1.1B-Chat-v1.0
-model_type: LlamaForCausalLM
-tokenizer_type: LlamaTokenizer
-is_llama_derived_model: true
-
-load_in_8bit: true
-load_in_4bit: false
-
-datasets:
-  - path: philschmid/guanaco-sharegpt-style
-    type: sharegpt
-    shards: 10
-val_set_size: 0
-output_dir: temp_debug/axolotl_outputs/model
-dataset_prepared_path: temp_debug/axolotl_outputs/data
-dataset_processes: 1
-
-sequence_len: 4096
-sample_packing: false
-pad_to_sequence_len: true
-
-adapter: lora
-lora_model_dir:
-lora_r: 32
-lora_alpha: 16
-lora_dropout: 0.05
-lora_target_linear: true
-lora_fan_in_fan_out:
-
-micro_batch_size: 1
-num_epochs: 1
-max_steps: 10
-optimizer: adamw_bnb_8bit
-lr_scheduler: cosine
-learning_rate: 0.0002
-
-train_on_inputs: false
-group_by_length: false
-bf16: false
-fp16: true
-tf32: false
-
-gradient_checkpointing: true
-logging_steps: 1
-flash_attention: true
-
-warmup_steps: 10
-weight_decay: 0.0

docker/Dockerfile

@@ -20,9 +20,9 @@ WORKDIR /workspace/axolotl
 
 # If AXOLOTL_EXTRAS is set, append it in brackets
 RUN if [ "$AXOLOTL_EXTRAS" != "" ] ; then \
-        pip install -e .[deepspeed,flash-attn,mamba-ssm,$AXOLOTL_EXTRAS]; \
+        pip install -e .[deepspeed,flash-attn,$AXOLOTL_EXTRAS]; \
     else \
-        pip install -e .[deepspeed,flash-attn,mamba-ssm]; \
+        pip install -e .[deepspeed,flash-attn]; \
     fi
 
 # So we can test the Docker image

docker/Dockerfile-runpod

@@ -5,7 +5,6 @@ ENV HF_DATASETS_CACHE="/workspace/data/huggingface-cache/datasets"
 ENV HUGGINGFACE_HUB_CACHE="/workspace/data/huggingface-cache/hub"
 ENV TRANSFORMERS_CACHE="/workspace/data/huggingface-cache/hub"
 ENV HF_HOME="/workspace/data/huggingface-cache/hub"
-ENV HF_HUB_ENABLE_HF_TRANSFER="1"
 
 COPY scripts/runpod-entrypoint.sh /root/runpod-entrypoint.sh
 

docker/Dockerfile-tests

@@ -1,40 +0,0 @@
-ARG BASE_TAG=main-base
-FROM winglian/axolotl-base:$BASE_TAG
-
-ARG TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6+PTX"
-ARG AXOLOTL_EXTRAS=""
-ARG CUDA="118"
-ENV BNB_CUDA_VERSION=$CUDA
-ARG PYTORCH_VERSION="2.0.1"
-ARG GITHUB_REF="main"
-
-ENV PYTORCH_VERSION=$PYTORCH_VERSION
-
-RUN apt-get update && \
-    apt-get install -y --allow-change-held-packages vim curl nano libnccl2 libnccl-dev
-
-WORKDIR /workspace
-
-RUN git clone --depth=1 https://github.com/OpenAccess-AI-Collective/axolotl.git
-
-WORKDIR /workspace/axolotl
-
-RUN git fetch origin +$GITHUB_REF && \
-    git checkout FETCH_HEAD
-
-# If AXOLOTL_EXTRAS is set, append it in brackets
-RUN if [ "$AXOLOTL_EXTRAS" != "" ] ; then \
-        pip install -e .[deepspeed,flash-attn,mamba-ssm,$AXOLOTL_EXTRAS]; \
-    else \
-        pip install -e .[deepspeed,flash-attn,mamba-ssm]; \
-    fi
-
-# So we can test the Docker image
-RUN pip install pytest
-
-# fix so that git fetch/pull from remote works
-RUN git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && \
-    git config --get remote.origin.fetch
-
-# helper for huggingface-login cli
-RUN git config --global credential.helper store

docs/debugging.md

@@ -1,242 +0,0 @@
-# Debugging Axolotl
-
-This document provides some tips and tricks for debugging Axolotl.  It also provides an example configuration for debugging with VSCode.  A good debugging setup is essential to understanding how Axolotl code works behind the scenes.
-
-## Table of Contents
-
-- [General Tips](#general-tips)
-- [Debugging with VSCode](#debugging-with-vscode)
-    - [Background](#background)
-    - [Configuration](#configuration)
-    - [Customizing your debugger](#customizing-your-debugger)
-    - [Video Tutorial](#video-tutorial)
-- [Debugging With Docker](#debugging-with-docker)
-    - [Setup](#setup)
-    - [Attach To Container](#attach-to-container)
-    - [Video - Attaching To Docker On Remote Host](#video---attaching-to-docker-on-remote-host)
-
-## General Tips
-
-While debugging it's helpful to simplify your test scenario as much as possible.  Here are some tips for doing so:
-
-> [!Important]
-> All of these tips are incorporated into the [example configuration](#configuration) for debugging with VSCode below.
-
-1. **Make sure you are using the latest version of axolotl**:  This project changes often and bugs get fixed fast.  Check your git branch and make sure you have pulled the latest changes from `main`.
-1. **Eliminate concurrency**: Restrict the number of processes to 1 for both training and data preprocessing:
-    - Set `CUDA_VISIBLE_DEVICES` to a single GPU, ex: `export CUDA_VISIBLE_DEVICES=0`.
-    - Set `dataset_processes: 1` in your axolotl config or run the training command with `--dataset_processes=1`.
-2. **Use a small dataset**: Construct or use a small dataset from HF Hub. When using a small dataset, you will often have to make sure `sample_packing: False` and `eval_sample_packing: False` to avoid errors.  If you are in a pinch and don't have time to construct a small dataset but want to use from the HF Hub, you can shard the data (this will still tokenize the entire dataset, but will only use a fraction of the data for training.  For example, to shard the dataset into 20 pieces, add the following to your axolotl config):
-    ```yaml
-    dataset:
-        ...
-        shards: 20
-    ```
-3. **Use a small model**: A good example of a small model is [TinyLlama/TinyLlama-1.1B-Chat-v1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0).
-4. **Minimize iteration time**: Make sure the training loop finishes as fast as possible, with these settings.
-    - `micro_batch_size: 1`
-    - `max_steps: 1`
-    - `val_set_size: 0`
-5. **Clear Caches:** Axolotl caches certain steps and so does the underlying HuggingFace trainer.  You may want to clear some of these caches when debugging.
-    - Data preprocessing: When debugging data preprocessing, which includes prompt template formation, you may want to delete the directory set in `dataset_prepared_path:` in your axolotl config.  If you didn't set this value, the default is `last_run_prepared`.
-    - HF Hub: If you are debugging data preprocessing, you should clear the relevant HF cache [HuggingFace cache](https://huggingface.co/docs/datasets/cache), by deleting the appropriate `~/.cache/huggingface/datasets/...` folder(s).
-    - **The recommended approach is to redirect all outputs and caches to a temporary folder and delete selected subfolders before each run.  This is demonstrated in the example configuration below.**
-
-
-## Debugging with VSCode
-
-### Background
-
-The below example shows how to configure VSCode to debug data preprocessing of the `sharegpt` format.  This is the format used when you have the following in your axolotl config:
-
-```yaml
-datasets:
-  - path: <path to your sharegpt formatted dataset> # example on HF Hub: philschmid/guanaco-sharegpt-style
-    type: sharegpt
-```
-
->[!Important]
-> If you are already familiar with advanced VSCode debugging, you can skip the below explanation and look at the files [.vscode/launch.json](../.vscode/launch.json) and [.vscode/tasks.json](../.vscode/tasks.json) for an example configuration.
-
->[!Tip]
-> If you prefer to watch a video, rather than read, you can skip to the [video tutorial](#video-tutorial) below (but doing both is recommended).
-
-### Setup
-
-Make sure you have an [editable install](https://setuptools.pypa.io/en/latest/userguide/development_mode.html) of Axolotl, which ensures that changes you make to the code are reflected at runtime.  Run the following commands from the root of this project:
-
-```bash
-pip3 install packaging
-pip3 install -e '.[flash-attn,deepspeed]'
-```
-
-#### Remote Hosts
-
-If you developing on a remote host, you can easily use VSCode to debug remotely.  To do so, you will need to follow this [remote - SSH guide](https://code.visualstudio.com/docs/remote/ssh).  You can also see the video below on [Docker and Remote SSH debugging](#video---attaching-to-docker-on-remote-host).
-
-```bash
-
-### Configuration
-
-The easiest way to get started is to modify the [.vscode/launch.json](../.vscode/launch.json) file in this project.  This is just an example configuration, so you may need to modify or copy it to suit your needs.
-
-For example, to mimic the command `cd devtools && CUDA_VISIBLE_DEVICES=0 accelerate launch -m axolotl.cli.train dev_sharegpt.yml`, you would use the below configuration[^1].  Note that we add additional flags that override the axolotl config and incorporate the tips above (see the comments). We also set the working directory to `devtools` and set the `env` variable `HF_HOME` to a temporary folder that is later partially deleted.  This is because we want to delete the HF dataset cache before each run in order to ensure that the data preprocessing code is run from scratch.
-
-```jsonc
-// .vscode/launch.json
-{
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Debug axolotl prompt - sharegpt",
-            "type": "python",
-            "module": "accelerate.commands.launch",
-            "request": "launch",
-            "args": [
-                "-m", "axolotl.cli.train", "dev_sharegpt.yml",
-                // The flags below simplify debugging by overriding the axolotl config
-                // with the debugging tips above.  Modify as needed.
-                "--dataset_processes=1",      // limits data preprocessing to one process
-                "--max_steps=1",              // limits training to just one step
-                "--batch_size=1",             // minimizes batch size
-                "--micro_batch_size=1",       // minimizes batch size
-                "--val_set_size=0",           // disables validation
-                "--sample_packing=False",     // disables sample packing which is necessary for small datasets
-                "--eval_sample_packing=False",// disables sample packing on eval set
-                "--dataset_prepared_path=temp_debug/axolotl_outputs/data", // send data outputs to a temp folder
-                "--output_dir=temp_debug/axolotl_outputs/model" // send model outputs to a temp folder
-                ],
-            "console": "integratedTerminal",      // show output in the integrated terminal
-            "cwd": "${workspaceFolder}/devtools", // set working directory to devtools from the root of the project
-            "justMyCode": true,                   // step through only axolotl code
-            "env": {"CUDA_VISIBLE_DEVICES": "0",  // Since we aren't doing distributed training, we need to limit to one GPU
-                    "HF_HOME": "${workspaceFolder}/devtools/temp_debug/.hf-cache"}, // send HF cache to a temp folder
-            "preLaunchTask": "cleanup-for-dataprep", // delete temp folders (see below)
-        }
-    ]
-}
-```
-
-**Additional notes about this configuration:**
-
-- The argument `justMyCode` is set to `true` such that you step through only the axolotl code.  If you want to step into dependencies, set this to `false`.
-- The `preLaunchTask`: `cleanup-for-dataprep` is defined in [.vscode/tasks.json](../.vscode/tasks.json) and is used to delete the following folders before debugging, which is essential to ensure that the data pre-processing code is run from scratch:
-    -  `./devtools/temp_debug/axolotl_outputs`
-    - `./devtools/temp_debug/.hf-cache/datasets`
-
->[!Tip]
-> You may not want to delete these folders. For example, if you are debugging model training instead of data pre-processing, you may NOT want to delete the cache or output folders. You may also need to add additional tasks to the `tasks.json` file depending on your use case.
-
-Below is the [./vscode/tasks.json](../.vscode/tasks.json) file that defines the `cleanup-for-dataprep` task.  This task is run before each debugging session when you use the above configuration.  Note how there are two tasks that delete the two folders mentioned above.  The third task `cleanup-for-dataprep` is a composite task that combines the two tasks.  A composite task is necessary because VSCode does not allow you to specify multiple tasks in the `preLaunchTask` argument of the `launch.json` file.
-
-```jsonc
-// .vscode/tasks.json
-// this file is used by launch.json
-{
-    "version": "2.0.0",
-    "tasks": [
-      // this task changes into the devtools directory and deletes the temp_debug/axolotl_outputs folder
-      {
-        "label": "delete-outputs",
-        "type": "shell",
-        "command": "rm -rf temp_debug/axolotl_outputs",
-        "options":{ "cwd": "${workspaceFolder}/devtools"},
-        "problemMatcher": []
-      },
-      // this task changes into the devtools directory and deletes the `temp_debug/.hf-cache/datasets` folder
-      {
-        "label": "delete-temp-hf-dataset-cache",
-        "type": "shell",
-        "command": "rm -rf temp_debug/.hf-cache/datasets",
-        "options":{ "cwd": "${workspaceFolder}/devtools"},
-        "problemMatcher": []
-      },
-        // this task combines the two tasks above
-      {
-       "label": "cleanup-for-dataprep",
-       "dependsOn": ["delete-outputs", "delete-temp-hf-dataset-cache"],
-      }
-    ]
-}
-```
-
-### Customizing your debugger
-
-Your debugging use case may differ from the example above.  The easiest thing to do is to put your own axolotl config in the `devtools` folder and modify the `launch.json` file to use your config.  You may also want to modify the `preLaunchTask` to delete different folders or not delete anything at all.
-
-### Video Tutorial
-
-The following video tutorial walks through the above configuration and demonstrates how to debug with VSCode, (click the image below to watch):
-
-<div style="text-align: center; line-height: 0;">
-
-<a href="https://youtu.be/xUUB11yeMmc" target="_blank"
-title="How to debug Axolotl (for fine tuning LLMs)"><img
-src="https://i.ytimg.com/vi/xUUB11yeMmc/maxresdefault.jpg"
-style="border-radius: 10px; display: block; margin: auto;" width="560" height="315" /></a>
-
-<figcaption style="font-size: smaller;"><a href="https://hamel.dev">Hamel Husain's</a> tutorial: <a href="https://www.youtube.com/watch?v=xUUB11yeMmc">Debugging Axolotl w/VSCode</a></figcaption>
-
-</div>
-<br>
-
-## Debugging With Docker
-
-Using [official Axolotl Docker images](https://hub.docker.com/r/winglian/axolotl/tags) is a great way to debug your code, and is a very popular way to use Axolotl.  Attaching VSCode to Docker takes a few more steps.
-
-### Setup
-
-On the host that is running axolotl (ex: if you are using a remote host), clone the axolotl repo and change your current directory to the root:
-
-```bash
-git clone https://github.com/OpenAccess-AI-Collective/axolotl
-cd axolotl
-```
-
->[!Tip]
-> If you already have axolotl cloned on your host, make sure you have the latest changes and change into the root of the project.
-
-Next, run the desired docker image and mount the current directory. Below is a docker command you can run to do this:[^2]
-
-```bash
-docker run --privileged --gpus '"all"' --shm-size 10g --rm -it --name axolotl --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 --mount type=bind,src="${PWD}",target=/workspace/axolotl -v ${HOME}/.cache/huggingface:/root/.cache/huggingface winglian/axolotl:main-py3.10-cu118-2.0.1
-```
-
->[!Tip]
-> To understand which containers are available, see the [Docker section of the README](../README.md#docker) and the [DockerHub repo](https://hub.docker.com/r/winglian/axolotl/tags).  For details of how the Docker containers are built, see axolotl's [Docker CI builds](../.github/workflows/main.yml).
-
-You will now be in the container.  Next, perform an editable install of Axolotl:
-
-```bash
-pip3 install packaging
-pip3 install -e '.[flash-attn,deepspeed]'
-```
-
-### Attach To Container
-
-Next, if you are using a remote host, [Remote into this host with VSCode](https://code.visualstudio.com/docs/remote/ssh).  If you are using a local host, you can skip this step.
-
-Next, select `Dev Containers: Attach to Running Container...` using the command palette (`CMD + SHIFT + P`) in VSCode.  You will be prompted to select a container to attach to.  Select the container you just created.  You will now be in the container with a working directory that is at the root of the project.  Any changes you make to the code will be reflected both in the container and on the host.
-
-Now you are ready to debug as described above (see [Debugging with VSCode](#debugging-with-vscode)).
-
-### Video - Attaching To Docker On Remote Host
-
-Here is a short video that demonstrates how to attach to a Docker container on a remote host:
-
-<div style="text-align: center; line-height: 0;">
-
-<a href="https://youtu.be/0AuoR7QnHR0" target="_blank"
-title="Debugging Axolotl Part 2: Attaching to Docker on a Remote Host"><img
-src="https://i.ytimg.com/vi/0AuoR7QnHR0/hqdefault.jpg"
-style="border-radius: 10px; display: block; margin: auto;" width="560" height="315" /></a>
-
-<figcaption style="font-size: smaller;"><a href="https://hamel.dev">Hamel Husain's</a> tutorial: <a href="https://youtu.be/0AuoR7QnHR0">Debugging Axolotl Part 2: Attaching to Docker on a Remote Host
-</a></figcaption>
-
-</div>
-<br>
-
-[^1]: The config actually mimics the command `CUDA_VISIBLE_DEVICES=0 python -m accelerate.commands.launch -m axolotl.cli.train devtools/sharegpt.yml`, but this is the same thing.
-
-[^2]: Many of the below flags are recommended best practices by Nvidia when using nvidia-container-toolkit.  You can read more about these flags [here](https://docs.nvidia.com/deeplearning/frameworks/user-guide/index.html).

docs/rlhf.md

@@ -33,12 +33,3 @@ datasets:
 ```yaml
 rl: ipo
 ```
-
-#### Trl autounwrap for peft
-
-Trl supports autounwrapping peft models, so that a ref model does not need to be additionally loaded, leading to less VRAM needed. This is on by default. To turn it off, pass the following config.
-
-```yaml
-# load ref model when adapter training.
-rl_adapter_ref_model: true
-```

examples/phi/phi2-ft.yml

@@ -1,74 +0,0 @@
-base_model: microsoft/phi-2
-model_revision:  834565c  # pin model repo to the previous architecture
-model_type: AutoModelForCausalLM
-tokenizer_type: AutoTokenizer
-trust_remote_code: true
-
-load_in_8bit: false
-load_in_4bit: false
-strict: false
-
-datasets:
-  - path: garage-bAInd/Open-Platypus
-    type: alpaca
-
-dataset_prepared_path:
-val_set_size: 0.05
-output_dir: ./phi-sft-out
-
-sequence_len: 2048
-sample_packing: false  # currently unsupported
-pad_to_sequence_len:
-
-adapter:
-lora_model_dir:
-lora_r: 16
-lora_alpha: 32
-lora_dropout: 0.1
-lora_target_linear: true
-lora_fan_in_fan_out:
-lora_modules_to_save:
-  - embd
-  - lm_head
-
-wandb_project:
-wandb_entity:
-wandb_watch:
-wandb_name:
-wandb_log_model:
-
-gradient_accumulation_steps: 1
-micro_batch_size: 1
-num_epochs: 4
-optimizer: paged_adamw_8bit
-adam_beta2: 0.95
-adam_epsilon: 0.00001
-max_grad_norm: 1.0
-lr_scheduler: cosine
-learning_rate: 1e-5
-
-train_on_inputs: false
-group_by_length: false
-bf16: true
-fp16: false
-tf32: true
-
-gradient_checkpointing: true
-early_stopping_patience:
-resume_from_checkpoint:
-local_rank:
-logging_steps: 1
-xformers_attention:
-flash_attention: true
-
-warmup_steps: 100
-evals_per_epoch: 4
-saves_per_epoch: 1
-debug:
-deepspeed:
-weight_decay: 0.1
-fsdp:
-fsdp_config:
-resize_token_embeddings_to_32x: true
-special_tokens:
-  pad_token: "<|endoftext|>"

requirements.txt

@@ -1,10 +1,11 @@
 --extra-index-url https://huggingface.github.io/autogptq-index/whl/cu118/
-packaging==23.2
-peft==0.7.0
+auto-gptq==0.5.1
+packaging
+peft==0.6.0
 transformers @ git+https://github.com/huggingface/transformers.git@3cefac1d974db5e2825a0cb2b842883a628be7a0
 tokenizers==0.15.0
 bitsandbytes>=0.41.1
-accelerate @ git+https://github.com/huggingface/accelerate.git@0d2280dadc6a93413a5496613b7fdda3a4d2551b
+accelerate==0.24.1
 deepspeed
 addict
 fire
@@ -20,7 +21,6 @@ hf_transfer
 colorama
 numba
 numpy>=1.24.4
-mlflow
 # qlora things
 bert-score==0.3.13
 evaluate==0.4.0
@@ -33,11 +33,9 @@ fschat==0.2.34
 gradio==3.50.2
 tensorboard
 
-mamba-ssm==1.1.1
-
 # remote filesystems
 s3fs
 gcsfs
 # adlfs
 
-trl>=0.7.9
+trl @ git+https://github.com/huggingface/trl.git@main

setup.py

@@ -11,17 +11,16 @@ def parse_requirements():
     with open("./requirements.txt", encoding="utf-8") as requirements_file:
         lines = [r.strip() for r in requirements_file.readlines()]
         for line in lines:
-            is_extras = (
-                "flash-attn" in line
-                or "flash-attention" in line
-                or "deepspeed" in line
-                or "mamba-ssm" in line
-            )
             if line.startswith("--extra-index-url"):
                 # Handle custom index URLs
                 _, url = line.split()
                 _dependency_links.append(url)
-            elif not is_extras and line and line[0] != "#":
+            elif (
+                "flash-attn" not in line
+                and "deepspeed" not in line
+                and line
+                and line[0] != "#"
+            ):
                 # Handle standard packages
                 _install_requires.append(line)
 
@@ -52,17 +51,11 @@ setup(
         "flash-attn": [
             "flash-attn==2.3.3",
         ],
-        "fused-dense-lib": [
-            "fused-dense-lib  @ git+https://github.com/Dao-AILab/flash-attention@v2.3.3#subdirectory=csrc/fused_dense_lib",
-        ],
         "deepspeed": [
             "deepspeed",
         ],
         "mamba-ssm": [
             "mamba-ssm==1.0.1",
         ],
-        "auto-gptq": [
-            "auto-gptq==0.5.1",
-        ],
     },
 )

src/axolotl/cli/__init__.py

@@ -25,15 +25,10 @@ from transformers import GenerationConfig, TextIteratorStreamer, TextStreamer
 from axolotl.common.cli import TrainerCliArgs, load_model_and_tokenizer
 from axolotl.logging_config import configure_logging
 from axolotl.train import TrainDatasetMeta
-from axolotl.utils.config import (
-    normalize_cfg_datasets,
-    normalize_config,
-    validate_config,
-)
+from axolotl.utils.config import normalize_config, validate_config
 from axolotl.utils.data import prepare_dataset
 from axolotl.utils.dict import DictDefault
 from axolotl.utils.distributed import is_main_process
-from axolotl.utils.mlflow_ import setup_mlflow_env_vars
 from axolotl.utils.models import load_tokenizer
 from axolotl.utils.tokenization import check_dataset_labels
 from axolotl.utils.trainer import prepare_optim_env
@@ -293,12 +288,7 @@ def load_cfg(config: Path = Path("examples/"), **kwargs):
 
     normalize_config(cfg)
 
-    normalize_cfg_datasets(cfg)
-
     setup_wandb_env_vars(cfg)
-
-    setup_mlflow_env_vars(cfg)
-
     return cfg
 
 

src/axolotl/cli/merge_lora.py

@@ -35,10 +35,6 @@ def do_cli(config: Path = Path("examples/"), **kwargs):
             f"Target directory for merge: `{parsed_cfg.lora_model_dir}` does not exist."
         )
 
-    parsed_cfg.load_in_4bit = False
-    parsed_cfg.load_in_8bit = False
-    parsed_cfg.flash_attention = False
-
     do_merge_lora(cfg=parsed_cfg, cli_args=parsed_cli_args)
 
 

src/axolotl/cli/preprocess.py

@@ -31,7 +31,6 @@ def do_cli(config: Path = Path("examples/"), **kwargs):
     parsed_cli_args, _ = parser.parse_args_into_dataclasses(
         return_remaining_strings=True
     )
-
     if not parsed_cfg.dataset_prepared_path:
         msg = (
             Fore.RED

src/axolotl/cli/train.py

@@ -31,7 +31,6 @@ def do_cli(config: Path = Path("examples/"), **kwargs):
     parsed_cli_args, _ = parser.parse_args_into_dataclasses(
         return_remaining_strings=True
     )
-
     if parsed_cfg.rl:
         dataset_meta = load_rl_datasets(cfg=parsed_cfg, cli_args=parsed_cli_args)
     else:

src/axolotl/core/trainer_builder.py

@@ -1,4 +1,3 @@
-# pylint: disable=too-many-lines
 """
 Builder for the training args and trainer
 """
@@ -35,14 +34,10 @@ from axolotl.utils.callbacks import (
 )
 from axolotl.utils.collators import (
     BatchSamplerDataCollatorForSeq2Seq,
-    DataCollatorForSeq2Seq,
     MambaDataCollator,
 )
-from axolotl.utils.samplers import MultipackBatchSampler, get_dataset_lengths
-from axolotl.utils.schedulers import (
-    get_cosine_schedule_with_min_lr,
-    get_cosine_schedule_with_quadratic_warmup,
-)
+from axolotl.utils.samplers import MultipackBatchSampler
+from axolotl.utils.schedulers import get_cosine_schedule_with_quadratic_warmup
 
 try:
     import torch._dynamo  # pylint: disable=ungrouped-imports
@@ -124,10 +119,6 @@ class AxolotlTrainingArguments(TrainingArguments):
         default=None,
         metadata={"help": "prefetch_factor argument to the dataloader"},
     )
-    cosine_min_lr_ratio: Optional[float] = field(
-        default=None,
-        metadata={"help": "Minimum learning rate is min_lr_ratio * learning_rate"},
-    )
 
 
 class AxolotlTrainer(Trainer):
@@ -138,19 +129,10 @@ class AxolotlTrainer(Trainer):
     args = None  # type: AxolotlTrainingArguments
     tag_names = ["axolotl"]
 
-    def __init__(
-        self,
-        *_args,
-        num_epochs=1,
-        bench_data_collator=None,
-        eval_data_collator=None,
-        **kwargs
-    ):
+    def __init__(self, *args, num_epochs=1, bench_data_collator=None, **kwargs):
         self.num_epochs = num_epochs
         self.bench_data_collator = bench_data_collator
-        self.eval_data_collator = eval_data_collator
-        super().__init__(*_args, **kwargs)
-        self.train_data_collator = self.data_collator
+        super().__init__(*args, **kwargs)
 
     def create_scheduler(
         self, num_training_steps: int, optimizer: torch.optim.Optimizer = None
@@ -176,17 +158,6 @@ class AxolotlTrainer(Trainer):
                     num_warmup_steps=self.args.get_warmup_steps(num_training_steps),
                     num_training_steps=num_training_steps,
                 )
-            elif self.args.lr_scheduler_type == "cosine" and self.args.cosine_min_lr_ratio is not None:
-                assert 0 <= self.args.cosine_min_lr_ratio <= 1.0, "cosine_min_lr_ratio must be between 0.0 and 1.0"
-                if self.args.deepspeed:
-                    LOG.warning("Using cosine scheduler with deepspeed. This may be ignored if a scheduler is set \
-                                in the deepspeed JSON")
-                self.lr_scheduler = get_cosine_schedule_with_min_lr(  # pylint: disable=attribute-defined-outside-init
-                    optimizer,
-                    num_warmup_steps=self.args.get_warmup_steps(num_training_steps),
-                    num_training_steps=num_training_steps,
-                    min_lr_ratio=self.args.cosine_min_lr_ratio,
-                )
             else:
                 return super().create_scheduler(num_training_steps, optimizer)
         return self.lr_scheduler
@@ -198,7 +169,12 @@ class AxolotlTrainer(Trainer):
                 self.args.train_batch_size,
                 drop_last=True,
                 batch_max_len=self._train_batch_size * self.args.max_seq_length,
-                lengths=get_dataset_lengths(self.train_dataset),
+                lengths=(
+                    self.train_dataset.data.column("position_ids")
+                    .to_pandas()
+                    .apply(lambda x: x[-1] + 1)
+                    .values
+                ),
                 packing_efficiency_estimate=self.args.sample_packing_efficiency,
             )
         return super()._get_train_sampler()
@@ -212,7 +188,12 @@ class AxolotlTrainer(Trainer):
                 self.args.per_device_eval_batch_size,
                 drop_last=True,
                 batch_max_len=self.args.eval_batch_size * self.args.max_seq_length,
-                lengths=get_dataset_lengths(eval_dataset),
+                lengths=(
+                    eval_dataset.data.column("position_ids")
+                    .to_pandas()
+                    .apply(lambda x: x[-1] + 1)
+                    .values
+                ),
                 packing_efficiency_estimate=self.args.sample_packing_efficiency,
             )
         return super()._get_eval_sampler(eval_dataset)
@@ -249,16 +230,6 @@ class AxolotlTrainer(Trainer):
         return super().get_train_dataloader()
 
     def get_eval_dataloader(self, eval_dataset: Optional[Dataset] = None) -> DataLoader:
-        if self.args.sample_packing and self.args.eval_sample_packing is False:
-            self.data_collator = (  # pylint: disable=attribute-defined-outside-init
-                self.eval_data_collator
-            )
-            dataloader = super().get_eval_dataloader(eval_dataset)
-            self.data_collator = (  # pylint: disable=attribute-defined-outside-init
-                self.train_data_collator
-            )
-            return dataloader
-
         if self.args.sample_packing and self.args.eval_sample_packing is not False:
             eval_dataset = (
                 eval_dataset if eval_dataset is not None else self.eval_dataset
@@ -289,7 +260,6 @@ class AxolotlTrainer(Trainer):
             return self.accelerator.prepare_data_loader(
                 DataLoader(eval_dataset, **dataloader_params)
             )
-
         return super().get_eval_dataloader(eval_dataset)
 
     def _get_bench_sampler(
@@ -672,12 +642,6 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
             training_arguments_kwargs[
                 "dataloader_prefetch_factor"
             ] = self.cfg.dataloader_prefetch_factor
-        if self.cfg.dataloader_drop_last is not None:
-            training_arguments_kwargs[
-                "dataloader_drop_last"
-            ] = self.cfg.dataloader_drop_last
-        elif self.cfg.sample_packing and self.cfg.eval_sample_packing is False:
-            training_arguments_kwargs["dataloader_drop_last"] = True
 
         if self.cfg.val_set_size == 0:
             # no eval set, so don't eval
@@ -774,12 +738,7 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
             False if self.cfg.ddp else None
         )
         training_arguments_kwargs["group_by_length"] = self.cfg.group_by_length
-        report_to = None
-        if self.cfg.use_wandb:
-            report_to = "wandb"
-        if self.cfg.use_mlflow:
-            report_to = "mlflow"
-        training_arguments_kwargs["report_to"] = report_to
+        training_arguments_kwargs["report_to"] = "wandb" if self.cfg.use_wandb else None
         training_arguments_kwargs["run_name"] = (
             self.cfg.wandb_name if self.cfg.use_wandb else None
         )
@@ -795,7 +754,6 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
         training_arguments_kwargs["lr_scheduler_kwargs"] = (
             self.cfg.lr_scheduler_kwargs if self.cfg.lr_scheduler_kwargs else {}
         )
-        training_arguments_kwargs["cosine_min_lr_ratio"] = self.cfg.cosine_min_lr_ratio
         training_arguments_kwargs["weight_decay"] = (
             self.cfg.weight_decay if self.cfg.weight_decay is not None else 0.0
         )
@@ -858,9 +816,6 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
             eval_dataset=self.eval_dataset,
             args=training_args,
             data_collator=self.build_collator(training_args, **data_collator_kwargs),
-            eval_data_collator=self.build_collator(
-                training_args, is_eval=True, **data_collator_kwargs
-            ),
             bench_data_collator=transformers.DataCollatorForSeq2Seq(
                 self.tokenizer,
                 return_tensors="pt",
@@ -881,29 +836,14 @@ class HFCausalTrainerBuilder(TrainerBuilderBase):
 
         return trainer
 
-    def build_collator(
-        self, training_args: AxolotlTrainingArguments, is_eval=False, **kwargs
-    ):
+    def build_collator(self, training_args: AxolotlTrainingArguments, **kwargs):
         if training_args.pretraining:
             return None
 
         if self.cfg.model_config_type == "mamba":
             return MambaDataCollator(tokenizer=self.tokenizer)
 
-        use_batch_sampler_collator = False
-        if is_eval is False and training_args.sample_packing:
-            use_batch_sampler_collator = True
-        if is_eval and training_args.eval_sample_packing:
-            use_batch_sampler_collator = True
-
-        if use_batch_sampler_collator:
-            return BatchSamplerDataCollatorForSeq2Seq(
-                self.tokenizer,
-                return_tensors="pt",
-                **kwargs,
-            )
-
-        return DataCollatorForSeq2Seq(
+        return BatchSamplerDataCollatorForSeq2Seq(
             self.tokenizer,
             return_tensors="pt",
             **kwargs,
@@ -965,8 +905,6 @@ class HFDPOTrainerBuilder(TrainerBuilderBase):
             dpo_trainer_kwargs["loss_type"] = "ipo"
             if self.cfg.dpo_label_smoothing:
                 dpo_trainer_kwargs["label_smoothing"] = self.cfg.dpo_label_smoothing
-        elif self.cfg.rl == "kto_pair":
-            dpo_trainer_kwargs["loss_type"] = "kto_pair"
 
         dpo_trainer = DPOTrainer(
             self.model,

src/axolotl/models/mamba/__init__.py

@@ -2,20 +2,8 @@
 Modeling module for Mamba models
 """
 
-import importlib
-
-
-def check_mamba_ssm_installed():
-    mamba_ssm_spec = importlib.util.find_spec("mamba_ssm")
-    if mamba_ssm_spec is None:
-        raise ImportError(
-            "MambaLMHeadModel requires mamba_ssm. Please install it with `pip install -e .[mamba-ssm]`"
-        )
-
 
 def fix_mamba_attn_for_loss():
-    check_mamba_ssm_installed()
-
     from mamba_ssm.models import mixer_seq_simple
 
     from .modeling_mamba import MambaLMHeadModel as MambaLMHeadModelFixed

src/axolotl/models/phi/modeling_phi.py

@@ -9,32 +9,27 @@ from __future__ import annotations
 
 import math
 from dataclasses import dataclass, field
-from typing import Any, Callable, Dict, Optional, Tuple, Union
+from typing import Any, Dict, Optional, Tuple, Union
 
 import torch
 import torch.nn as nn
 from einops import rearrange, repeat
-from torch.utils.checkpoint import checkpoint
 from transformers import PretrainedConfig, PreTrainedModel
 from transformers.activations import ACT2FN
 from transformers.modeling_outputs import CausalLMOutputWithPast
 
+from ...monkeypatch.utils import get_cu_seqlens_from_pos_ids
 from .configuration_phi import PhiConfig
 
 try:
     from flash_attn.bert_padding import pad_input, unpad_input
     from flash_attn.layers.rotary import RotaryEmbedding as FlashRotaryEmbedding
     from flash_attn.modules.mha import FlashCrossAttention, FlashSelfAttention
-except ImportError:
+    from flash_attn.ops.fused_dense import FusedDense
+except:  # noqa: E722
     pad_input, unpad_input = None, None
     FlashRotaryEmbedding = None
     FlashSelfAttention, FlashCrossAttention = None, None
-
-# this is in a seperate try/except block since sometimes fused_dense isn't available
-# and it shouldn't completely disable flash attn when it isn't
-try:
-    from flash_attn.ops.fused_dense import FusedDense
-except ImportError:
     FusedDense = None
 
 
@@ -229,9 +224,7 @@ class RotaryEmbedding(nn.Module):
 
         # Initialize cached attributes since ONNX can't rely on dynamic initialization
         self._update_cos_sin_cache(
-            max_position_embeddings,
-            device=device,
-            dtype=torch.float32,
+            max_position_embeddings, device=device, dtype=torch.float32
         )
 
     def _compute_inv_freq(self, device: Optional[str] = None) -> torch.FloatTensor:
@@ -288,32 +281,34 @@ class RotaryEmbedding(nn.Module):
         seqlen_offset: int = 0,
         **kwargs,
     ) -> Tuple[torch.Tensor, torch.Tensor]:
+        seq_start = seqlen_offset
+        seq_end = seq_start + qkv.shape[1]
+
         if (
-            self._seq_len_cached < qkv.shape[1] + seqlen_offset
-            or self._cos_cached.device != qkv.device
+            self._cos_cached.device != qkv.device
             or self._cos_cached.dtype != qkv.dtype
             or (self.training and self._cos_cached.is_inference())
         ):
             self._update_cos_sin_cache(
-                qkv.shape[1] + seqlen_offset, device=qkv.device, dtype=qkv.dtype
+                self.max_position_embeddings, device=qkv.device, dtype=qkv.dtype
             )
 
         if kv is None:
             return _apply_rotary_emb_qkv(
                 qkv,
-                self._cos_cached[seqlen_offset:],
-                self._sin_cached[seqlen_offset:],
+                self._cos_cached[seq_start:seq_end],
+                self._sin_cached[seq_start:seq_end],
             )
         else:
             q = _apply_rotary_emb(
                 qkv,
-                self._cos_cached[seqlen_offset:],
-                self._sin_cached[seqlen_offset:],
+                self._cos_cached[seq_start:seq_end],
+                self._sin_cached[seq_start:seq_end],
             )
             kv = _apply_rotary_emb_kv(
                 kv,
-                self._cos_cached[seqlen_offset:],
-                self._sin_cached[seqlen_offset:],
+                self._cos_cached[seq_start:seq_end],
+                self._sin_cached[seq_start:seq_end],
             )
 
             return q, kv
@@ -516,7 +511,7 @@ def _update_kv_cache(
     num_heads, head_dim = kv.shape[-2:]
 
     if layer_idx not in inference_params.key_value_memory_dict:
-        inference_params.key_value_memory_dict[layer_idx] = torch.empty(
+        kv_cache = torch.empty(
             inference_params.max_batch_size,
             inference_params.max_seqlen,
             2,
@@ -525,6 +520,9 @@ def _update_kv_cache(
             dtype=kv.dtype,
             device=kv.device,
         )
+        inference_params.key_value_memory_dict[layer_idx] = kv_cache
+    else:
+        kv_cache = inference_params.key_value_memory_dict[layer_idx]
 
     batch_start = inference_params.batch_size_offset
     batch_end = batch_start + kv.shape[0]
@@ -532,19 +530,8 @@ def _update_kv_cache(
     sequence_start = inference_params.seqlen_offset
     sequence_end = sequence_start + kv.shape[1]
 
-    # When the current sequence length is equal to or larger than the maximum sequence length,
-    # we need to concatenate the current `kv` with the cached `kv` to expand its length
-    if sequence_end >= inference_params.max_seqlen:
-        inference_params.key_value_memory_dict[layer_idx] = torch.concatenate(
-            (inference_params.key_value_memory_dict[layer_idx], kv), dim=1
-        )
-
-    inference_params.key_value_memory_dict[layer_idx][
-        batch_start:batch_end, sequence_start:sequence_end, ...
-    ] = kv
-    kv = inference_params.key_value_memory_dict[layer_idx][
-        batch_start:batch_end, :sequence_end, ...
-    ]
+    kv_cache[batch_start:batch_end, sequence_start:sequence_end, ...] = kv
+    kv = kv_cache[batch_start:batch_end, :sequence_end, ...]
 
     return kv
 
@@ -637,10 +624,13 @@ class MHA(nn.Module):
         self.layer_idx = layer_idx
         self.return_residual = return_residual
         self.checkpointing = checkpointing
-        self._gradient_checkpointing_func = None
 
     def _forward_self_attn(
-        self, x: torch.FloatTensor, key_padding_mask: Optional[torch.BoolTensor]
+        self,
+        x: torch.FloatTensor,
+        key_padding_mask: Optional[torch.BoolTensor],
+        cu_seqlens: Optional[torch.LongTensor] = None,
+        max_seqlen: Optional[int] = None,
     ) -> torch.FloatTensor:
         qkv = self.Wqkv(x)
         qkv = rearrange(
@@ -653,21 +643,20 @@ class MHA(nn.Module):
         if self.flash_attn:
             batch_size, seqlen = qkv.shape[0], qkv.shape[1]
 
-            cu_seqlens, max_seqlen = None, None
-            if key_padding_mask is not None:
+            if (
+                key_padding_mask is not None
+                and cu_seqlens is None
+                and max_seqlen is None
+            ):
                 # If `key_padding_mask` is supplied, we need to unpad the input and retrieve
                 # the `cu_seqlens` and `max_seqlen` to be used by `flash-attn`
                 qkv, indices, cu_seqlens, max_seqlen = unpad_input(
                     qkv, key_padding_mask
                 )
 
-            if self.checkpointing and self.training:
-                attn_output = self._gradient_checkpointing_func(
-                    self.inner_attn,
-                    qkv,
-                    cu_seqlens=cu_seqlens,
-                    max_seqlen=max_seqlen,
-                    use_reentrant=False,
+            if self.checkpointing:
+                attn_output = torch.utils.checkpoint.checkpoint(
+                    self.inner_attn, qkv, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen
                 )
             else:
                 attn_output = self.inner_attn(
@@ -681,12 +670,9 @@ class MHA(nn.Module):
                 else attn_output
             )
 
-        if self.checkpointing and self.training:
-            return self._gradient_checkpointing_func(
-                self.inner_attn,
-                qkv,
-                key_padding_mask=key_padding_mask,
-                use_reentrant=False,
+        if self.checkpointing:
+            return torch.utils.checkpoint.checkpoint(
+                self.inner_attn, qkv, key_padding_mask=key_padding_mask
             )
 
         return self.inner_attn(qkv, key_padding_mask=key_padding_mask)
@@ -739,8 +725,8 @@ class MHA(nn.Module):
                     q, key_padding_mask
                 )
 
-            if self.checkpointing and self.training:
-                attn_output = self._gradient_checkpointing_func(
+            if self.checkpointing:
+                attn_output = torch.utils.checkpoint.checkpoint(
                     self.inner_cross_attn,
                     q,
                     kv,
@@ -749,7 +735,6 @@ class MHA(nn.Module):
                     max_seqlen=max_seqlen_q,
                     cu_seqlens_k=cu_seqlens_k,
                     max_seqlen_k=max_seqlen_k,
-                    use_reentrant=False,
                 )
             else:
                 attn_output = self.inner_cross_attn(
@@ -768,14 +753,13 @@ class MHA(nn.Module):
                 else attn_output
             )
 
-        if self.checkpointing and self.training:
-            return self._gradient_checkpointing_func(
+        if self.checkpointing:
+            return torch.utils.checkpoint.checkpoint(
                 self.inner_cross_attn,
                 q,
                 kv,
                 key_padding_mask=key_padding_mask,
                 causal=causal,
-                use_reentrant=False,
             )
 
         return self.inner_cross_attn(
@@ -787,8 +771,11 @@ class MHA(nn.Module):
         x: torch.FloatTensor,
         past_key_values: Optional[InferenceParams] = None,
         attention_mask: Optional[Union[torch.LongTensor, torch.BoolTensor]] = None,
+        cu_seqlens: Optional[torch.LongTensor] = None,
+        max_seqlen: Optional[int] = None,
         **kwargs,
     ) -> Tuple[torch.FloatTensor, torch.FloatTensor]:
+        # TODO: Need an alternative way for dynamic control flow: torch.any(~attention_mask.bool())
         if attention_mask is not None:
             attention_mask = attention_mask.bool()
         else:
@@ -798,12 +785,18 @@ class MHA(nn.Module):
         if self.n_head == self.n_head_kv:
             if past_key_values is None:
                 # If `past_key_values` are not supplied, we run self-attention
-                attn_output = self._forward_self_attn(x, attention_mask)
+                attn_output = self._forward_self_attn(
+                    x, attention_mask, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen
+                )
             else:
                 # If `past_key_values` are supplied, it means that we might have cached values and
                 # could take advantage of cross-attention
                 attn_output = self._forward_cross_attn(
-                    x, past_key_values, attention_mask
+                    x,
+                    past_key_values,
+                    attention_mask,
+                    cu_seqlens=cu_seqlens,
+                    max_seqlen=max_seqlen,
                 )
         # MQA / GQA
         else:
@@ -837,8 +830,6 @@ class ParallelBlock(nn.Module):
 
         self.mixer = MHA(config, layer_idx=block_idx)
         self.mlp = MLP(config)
-        self.checkpointing = False
-        self._gradient_checkpointing_func = None
 
     def forward(
         self,
@@ -847,52 +838,23 @@ class ParallelBlock(nn.Module):
         attention_mask: Optional[torch.BoolTensor] = None,
         **kwargs,
     ) -> torch.FloatTensor:
-        def _forward(
-            mixer,
-            resid_dropout,
-            mlp,
-            ln,
+        residual = hidden_states
+        hidden_states = self.ln(hidden_states)
+
+        attn_outputs = self.mixer(
             hidden_states,
-            past_key_values,
-            attention_mask,
-        ):
-            residual = hidden_states
-            hidden_states = ln(hidden_states)
-
-            attn_outputs = mixer(
-                hidden_states,
-                past_key_values=past_key_values,
-                attention_mask=attention_mask,
-            )
-            if isinstance(attn_outputs, tuple):
-                attn_outputs = attn_outputs[0]
-
-            attn_outputs = resid_dropout(attn_outputs)
-            feed_forward_hidden_states = resid_dropout(mlp(hidden_states))
-
-            return attn_outputs + feed_forward_hidden_states + residual
-
-        if self.training and self.checkpointing:
-            return self._gradient_checkpointing_func(
-                _forward,
-                self.mixer,
-                self.resid_dropout,
-                self.mlp,
-                self.ln,
-                hidden_states,
-                past_key_values,
-                attention_mask,
-            )
-
-        return _forward(
-            self.mixer,
-            self.resid_dropout,
-            self.mlp,
-            self.ln,
-            hidden_states,
-            past_key_values,
-            attention_mask,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
         )
+        if isinstance(attn_outputs, tuple):
+            attn_outputs = attn_outputs[0]
+
+        attn_outputs = self.resid_dropout(attn_outputs)
+        feed_forward_hidden_states = self.resid_dropout(self.mlp(hidden_states))
+
+        hidden_states = attn_outputs + feed_forward_hidden_states + residual
+
+        return hidden_states
 
 
 class CausalLMHead(nn.Module):
@@ -949,7 +911,7 @@ class PhiPreTrainedModel(PreTrainedModel):
 
     config_class = PhiConfig
     base_model_prefix = "transformer"
-    supports_gradient_checkpointing = True
+    supports_gradient_checkpointing = False
     _no_split_modules = ["ParallelBlock"]
 
     def __init__(self, *inputs, **kwargs) -> None:
@@ -969,14 +931,6 @@ class PhiPreTrainedModel(PreTrainedModel):
                 module.bias.data.zero_()
             module.weight.data.fill_(1.0)
 
-    def _set_gradient_checkpointing(
-        self, enable: bool = True, gradient_checkpointing_func: Callable = checkpoint
-    ):
-        for module in self.modules():
-            if hasattr(module, "checkpointing"):
-                module._gradient_checkpointing_func = gradient_checkpointing_func
-                module.checkpointing = enable
-
     def prepare_inputs_for_generation(
         self,
         input_ids: torch.LongTensor,
@@ -997,7 +951,7 @@ class PhiPreTrainedModel(PreTrainedModel):
             )
         else:
             # Assume that `past_key_values` has cached all tokens up to the last token in `input_ids`
-            past_key_values.seqlen_offset = input_ids.shape[1] - 1
+            past_key_values.seqlen_offset = len(input_ids[0]) - 1
             input_ids = input_ids[:, -1].unsqueeze(-1)
 
         return {
@@ -1034,6 +988,8 @@ class PhiModel(PhiPreTrainedModel):
         input_ids: torch.LongTensor,
         past_key_values: Optional[Union[torch.FloatTensor, InferenceParams]] = None,
         attention_mask: Optional[torch.BoolTensor] = None,
+        cu_seqlens: Optional[torch.LongTensor] = None,
+        max_seqlen: Optional[int] = None,
     ) -> torch.FloatTensor:
         hidden_states = self.embd(input_ids)
 
@@ -1042,6 +998,8 @@ class PhiModel(PhiPreTrainedModel):
                 hidden_states,
                 past_key_values=past_key_values,
                 attention_mask=attention_mask,
+                cu_seqlens=cu_seqlens,
+                max_seqlen=max_seqlen,
             )
 
         return hidden_states
@@ -1076,10 +1034,23 @@ class PhiForCausalLM(PhiPreTrainedModel):
         past_key_values: Optional[Union[torch.FloatTensor, InferenceParams]] = None,
         attention_mask: Optional[torch.BoolTensor] = None,
         labels: Optional[torch.LongTensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
         **kwargs,
     ) -> CausalLMOutputWithPast:
+        cu_seqlens: Optional[torch.LongTensor] = None
+        max_seqlen: Optional[int] = None
+        if position_ids is not None:
+            batch_size, seq_length = input_ids.shape
+            position_ids = position_ids.view(-1, seq_length).long()
+            cu_seqlens, max_seqlen = get_cu_seqlens_from_pos_ids(position_ids)
+            cu_seqlens = cu_seqlens.squeeze()
+
         hidden_states = self.transformer(
-            input_ids, past_key_values=past_key_values, attention_mask=attention_mask
+            input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            cu_seqlens=cu_seqlens,
+            max_seqlen=max_seqlen,
         )
         lm_logits = self.lm_head(hidden_states)
 

src/axolotl/monkeypatch/utils.py

@@ -55,7 +55,6 @@ def get_cu_seqlens(attn_mask):
     return torch.stack(results).to(dtype=torch.int32), torch.stack(max_seq_lens)
 
 
-@torch.jit.script
 def get_cu_seqlens_from_pos_ids(position_ids):
     """generate a cumulative sequence length mask for flash attention using pos ids"""
     if len(position_ids.shape) == 1:
@@ -82,7 +81,7 @@ def get_cu_seqlens_from_pos_ids(position_ids):
         # Get the indices where the sequence starts
         start_indices = torch.cat(
             [
-                torch.nonzero(seq_starts).unbind(dim=1)[0],
+                (seq_starts).nonzero(as_tuple=True)[0],
                 torch.tensor([len(adjusted_row)], dtype=torch.int32, device=device),
             ]
         )

src/axolotl/prompt_tokenizers.py

@@ -379,12 +379,10 @@ class ShareGPTPromptTokenizingStrategy(PromptTokenizingStrategy):
                         add_eos_token=False,
                         strip_bos_token=True,
                     )
-                    if self.train_on_inputs:
-                        labels = copy.deepcopy(res["input_ids"])
-                    else:
-                        # everything from this is masked out from the labels
-                        labels = [IGNORE_TOKEN_ID] * len(res["input_ids"])
+                    # everything from this is masked out from the labels
+                    labels = [IGNORE_TOKEN_ID] * len(res["input_ids"])
                 elif assistant in role:
+                    # TODO label assistant token/tokens w/ IGNORE_TOKEN_ID
                     role = (
                         role.replace(role_remap[1]["from"], role_remap[1]["to"])
                         if role_remap
@@ -394,13 +392,9 @@ class ShareGPTPromptTokenizingStrategy(PromptTokenizingStrategy):
                     # this should be the assistant response, should end with an eos token
                     if not content.strip():
                         LOG.warning(f"assistant turn has empty text: {prompt}")
-                    add_eos_token = not (
-                        conversation.name == "chatml"
-                        and conversation.sep == self.tokenizer.eos_token
-                    )
                     res = self._tokenize(
                         turn,
-                        add_eos_token=add_eos_token,
+                        add_eos_token=True,
                         strip_bos_token=True,
                     )
                     role_res = self._tokenize(
@@ -408,24 +402,18 @@ class ShareGPTPromptTokenizingStrategy(PromptTokenizingStrategy):
                         add_eos_token=False,
                         strip_bos_token=True,
                     )
+                    # not masked out from labels
                     labels = copy.deepcopy(res["input_ids"])
-                    if not self.train_on_inputs:
-                        # mask out role tokens from the labels
-                        len_role = len(role_res["input_ids"])
-                        labels[:len_role] = [IGNORE_TOKEN_ID] * min(
-                            len_role, len(labels)
-                        )
+                    len_role = len(role_res["input_ids"])
+                    labels[:len_role] = [IGNORE_TOKEN_ID] * min(len_role, len(labels))
                 elif role == "":
                     turn = content
                     # this is only ever the first part, should include the bos token and the user query
                     res = self._tokenize(
                         turn, add_eos_token=False, strip_bos_token=False
                     )
-                    if self.train_on_inputs:
-                        labels = copy.deepcopy(res["input_ids"])
-                    else:
-                        # everything from this is masked out from the labels
-                        labels = [IGNORE_TOKEN_ID] * len(res["input_ids"])
+                    # everything from this is masked out from the labels
+                    labels = [IGNORE_TOKEN_ID] * len(res["input_ids"])
                 else:
                     LOG.warning(f"unhandled role: {role}")
                     continue

src/axolotl/train.py

@@ -5,16 +5,14 @@ import signal
 import sys
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Optional, Tuple, Union
+from typing import Optional
 
 import torch
 import transformers.modelcard
 from accelerate.logging import get_logger
 from datasets import Dataset
 from optimum.bettertransformer import BetterTransformer
-from peft import PeftModel
 from pkg_resources import get_distribution  # type: ignore
-from transformers import PreTrainedModel, PreTrainedTokenizer
 from transformers.deepspeed import is_deepspeed_zero3_enabled
 
 from axolotl.common.cli import TrainerCliArgs
@@ -45,7 +43,7 @@ class TrainDatasetMeta:
 
 def train(
     *, cfg: DictDefault, cli_args: TrainerCliArgs, dataset_meta: TrainDatasetMeta
-) -> Tuple[Union[PeftModel, PreTrainedModel], PreTrainedTokenizer]:
+):
     # load the tokenizer first
     LOG.debug(
         f"loading tokenizer... {cfg.tokenizer_config or cfg.base_model_config}",
@@ -65,15 +63,10 @@ def train(
     model, peft_config = load_model(cfg, tokenizer, inference=cli_args.inference)
     model_ref = None
     if cfg.rl:
-        if cfg.adapter and not cfg.rl_adapter_ref_model:
-            # use built-in trl autounwrap
-            LOG.debug("Passing model_ref: None to RL trainer")
-            model_ref = None  # explicit setting to None
-        else:
-            # load the model again for model_ref/baseline
-            model_ref, _ = load_model(
-                cfg, tokenizer, inference=cli_args.inference, reference_model=True
-            )
+        # load the model again for model_ref/baseline
+        model_ref, _ = load_model(
+            cfg, tokenizer, inference=cli_args.inference, reference_model=True
+        )
 
     safe_serialization = cfg.save_safetensors is True
 

src/axolotl/utils/config.py

@@ -61,14 +61,6 @@ def normalize_config(cfg):
         cfg.device_map = {"": int(os.environ.get("LOCAL_RANK", 0))}
         cfg.batch_size = cfg.batch_size * cfg.world_size
 
-    if cfg.bf16 == "auto":
-        if is_torch_bf16_gpu_available():
-            LOG.debug("bf16 support detected, enabling for this configuration.")
-            cfg.bf16 = True
-        else:
-            LOG.debug("bf16 support not detected, disabling for this configuration.")
-            cfg.bf16 = False
-
     if cfg.device == "mps":
         cfg.load_in_8bit = False
         cfg.tf32 = False
@@ -158,26 +150,7 @@ def normalize_config(cfg):
     log_gpu_memory_usage(LOG, "baseline", cfg.device)
 
 
-def normalize_cfg_datasets(cfg):
-    """
-    helpers for mapping chat_template to various dataset configurations as necessary
-    """
-
-    if cfg.chat_template and cfg.chat_template == "chatml":
-        if cfg.datasets:
-            for idx, ds_cfg in enumerate(cfg.datasets):
-                if ds_cfg.type == "sharegpt" and not ds_cfg.conversation:
-                    LOG.info(
-                        f"updating dataset {ds_cfg.path} with `conversation: chatml` to match your chat_template"
-                    )
-                    cfg.datasets[idx].conversation = "chatml"
-
-
 def validate_config(cfg):
-    """
-    This is a "pre-validation" step that handles the yaml configuration before we have any
-    information about the model architecture
-    """
     if is_torch_bf16_gpu_available():
         if not cfg.bf16 and not cfg.bfloat16:
             LOG.info("bf16 support detected, but not enabled for this configuration.")
@@ -470,6 +443,20 @@ def validate_config(cfg):
     if cfg.neftune_noise_alpha is not None and cfg.neftune_noise_alpha <= 0.0:
         raise ValueError("neftune_noise_alpha must be > 0.0")
 
+    if (
+        cfg.adapter
+        and cfg.tokens
+        and (
+            not cfg.lora_modules_to_save
+            or not all(
+                x in cfg.lora_modules_to_save for x in ["embed_tokens", "lm_head"]
+            )
+        )
+    ):
+        raise ValueError(
+            "lora_modules_to_save not properly set yet adding new tokens. Please add `embed_tokens` and `lm_head` to `lora_modules_to_save`."
+        )
+
     if cfg.max_memory is not None and cfg.gpu_memory_limit is not None:
         raise ValueError(
             "max_memory and gpu_memory_limit are mutually exclusive and cannot be used together."

src/axolotl/utils/data.py

@@ -44,7 +44,7 @@ from axolotl.prompters import (
 from axolotl.utils.collators import PretrainingBatchSamplerDataCollatorForSeq2Seq
 from axolotl.utils.dict import DictDefault
 from axolotl.utils.distributed import is_main_process, zero_first
-from axolotl.utils.samplers import MultipackBatchSampler, get_dataset_lengths
+from axolotl.utils.samplers.multipack import MultipackBatchSampler
 from axolotl.utils.trainer import (
     calculate_total_num_steps,
     process_datasets_for_packing,
@@ -889,7 +889,12 @@ def encode_packed_pretraining(
         batch_size=batch_size,
         drop_last=True,
         batch_max_len=batch_size * max_seq_length,
-        lengths=get_dataset_lengths(train_dataset),
+        lengths=(
+            train_dataset.data.column("position_ids")
+            .to_pandas()
+            .apply(lambda x: x[-1] + 1)
+            .values
+        ),
     )
 
     chunked_data = defaultdict(list)

src/axolotl/utils/lora_embeddings.py

@@ -1,14 +0,0 @@
-"""
-helpers for lora embeddings
-"""
-
-
-def get_linear_embedding_layers(model_type):
-    """
-    returns the linear embedding layers needed for loras, dependent on the model arch
-    """
-    if model_type == "phi-msft":
-        return ["embd.wte", "lm_head.linear"]
-    if model_type == "gpt_neox":
-        return ["embed_in", "embed_out"]
-    return ["embed_tokens", "lm_head"]

src/axolotl/utils/mlflow_.py

@@ -1,18 +0,0 @@
-"""Module for mlflow utilities"""
-
-import os
-
-from axolotl.utils.dict import DictDefault
-
-
-def setup_mlflow_env_vars(cfg: DictDefault):
-    for key in cfg.keys():
-        if key.startswith("mlflow_"):
-            value = cfg.get(key, "")
-
-            if value and isinstance(value, str) and len(value) > 0:
-                os.environ[key.upper()] = value
-
-    # Enable mlflow if experiment name is present
-    if cfg.mlflow_experiment_name and len(cfg.mlflow_experiment_name) > 0:
-        cfg.use_mlflow = True

src/axolotl/utils/models.py

@@ -2,7 +2,7 @@
 import logging
 import math
 import os
-from typing import Any, Optional, Tuple, Union  # noqa: F401
+from typing import Any, Optional, Tuple  # noqa: F401
 
 import addict
 import bitsandbytes as bnb
@@ -28,16 +28,12 @@ from axolotl.prompt_tokenizers import LLAMA_DEFAULT_EOS_TOKEN
 from axolotl.utils.bench import log_gpu_memory_usage
 from axolotl.utils.chat_templates import chat_templates
 from axolotl.utils.dict import DictDefault
-from axolotl.utils.lora_embeddings import get_linear_embedding_layers
 
 LOG = logging.getLogger("axolotl")
 
 
-def check_model_config(cfg: DictDefault, model_config: Union[AutoConfig, DictDefault]):
-    quant_config_exists = (
-        hasattr(model_config, "quantization_config")
-        and model_config.quantization_config
-    )
+def check_model_config(cfg: DictDefault, model_config: AutoConfig):
+    quant_config_exists = hasattr(model_config, "quantization_config")
     quant_config_method_is_gptq = (
         quant_config_exists
         and "quant_method" in model_config.quantization_config
@@ -56,25 +52,9 @@ def check_model_config(cfg: DictDefault, model_config: Union[AutoConfig, DictDef
             "Please use the `gptq` flag to train quantized model or point to a non-quantized model."
         )
 
-    lora_modules_to_save = get_linear_embedding_layers(model_config.model_type)
-    if (
-        cfg.adapter
-        and cfg.tokens
-        and (
-            not cfg.lora_modules_to_save
-            or not all(x in cfg.lora_modules_to_save for x in lora_modules_to_save)
-        )
-    ):
-        lora_modules_to_save = ", ".join(map(lambda x: f"`{x}`", lora_modules_to_save))
-        raise ValueError(
-            f"`lora_modules_to_save` not properly set when adding new tokens. Please include {lora_modules_to_save} in `lora_modules_to_save`."
-        )
-
 
 def load_model_config(cfg):
     model_config_name = cfg.base_model_config or cfg.base_model
-    if not model_config_name and cfg.tokenizer_config:
-        model_config_name = cfg.tokenizer_config
     trust_remote_code = cfg.trust_remote_code is True
 
     try:
@@ -100,7 +80,6 @@ def load_model_config(cfg):
 
 
 def load_tokenizer(cfg):
-    model_config = load_model_config(cfg)
     tokenizer_kwargs = {}
     use_fast = True  # this is the default
 
@@ -157,26 +136,22 @@ def load_tokenizer(cfg):
                 setattr(tokenizer, attr_name, "<|endoftext|>")
 
     if cfg.special_tokens:
-        lora_modules_to_save = get_linear_embedding_layers(model_config.model_type)
         for k, val in cfg.special_tokens.items():
             # check if new special token is not already in tokenizer and
             # is adapter training to make sure lora_modules_to_save is set
-            # pylint: disable=too-many-boolean-expressions
             if (
                 (getattr(tokenizer, k) is None or getattr(tokenizer, k) != val)
                 and cfg.adapter
                 and (
                     not cfg.lora_modules_to_save
                     or not all(
-                        x in cfg.lora_modules_to_save for x in lora_modules_to_save
+                        x in cfg.lora_modules_to_save
+                        for x in ["embed_tokens", "lm_head"]
                     )
                 )
             ):
-                lora_modules_to_save = ", ".join(
-                    [f"`{x}`" for x in lora_modules_to_save]
-                )
                 raise ValueError(
-                    f"Please set lora_modules_to_save to {lora_modules_to_save} when using an adapter and changing the special tokens."
+                    "Please set lora_modules_to_save to ['embed_tokens', 'lm_head'] when using an adapter and changing the special tokens."
                 )
 
             tokenizer.add_special_tokens(
@@ -411,10 +386,6 @@ def load_model(
                 model_config._attn_implementation = (  # pylint: disable=protected-access
                     "eager"
                 )
-        if model_config.model_type == "phi-msft":
-            model_config.flash_attn = True
-            model_config.flash_rotary = True
-            model_config.fused_dense = True
 
     try:
         if cfg.is_llama_derived_model and not cfg.trust_remote_code and not cfg.gptq:
@@ -467,12 +438,11 @@ def load_model(
         #         device=cfg.device,
         #     )
         #     model.train() # sets to train instead of eval mode
-        elif model_type == "PhiForCausalLM" or model_config.model_type == "phi-msft":
+        elif model_type == "PhiForCausalLM":
             from axolotl.models.phi import PhiForCausalLM
 
             model = PhiForCausalLM.from_pretrained(
                 base_model,
-                config=model_config,
                 load_in_8bit=cfg.load_in_8bit and cfg.adapter is not None,
                 load_in_4bit=cfg.load_in_4bit and cfg.adapter is not None,
                 **model_kwargs,
@@ -588,14 +558,13 @@ def load_model(
         log_gpu_memory_usage(LOG, "after model load", model.device)
 
     # make sure these are fp32 per Ramesh et al. (2021)
-    embedding_modules = get_linear_embedding_layers(cfg.model_config_type)
     for name, module in model.named_modules():
-        if any(m in name for m in ["norm", "gate"]):
+        if "norm" in name:
             module.to(torch.float32)
         if model_config.model_type == "btlm":
             # don't upcast lm_head for btlm
             continue
-        if any(m in name for m in embedding_modules):
+        if "lm_head" in name or "embed_tokens" in name:
             if hasattr(module, "weight"):
                 module.to(torch.float32)
 
@@ -620,12 +589,12 @@ def load_model(
 
     # LlamaRMSNorm layers are in fp32 after kbit_training or full finetune, so we need to
     # convert them back to fp16/bf16 for flash-attn compatibility.
-    if needs_fa2_dtype or cfg.flash_attention:
+    if needs_fa2_dtype or (cfg.flash_attention and cfg.is_llama_derived_model):
         LOG.info("converting modules to %s for flash attention", cfg.torch_dtype)
         for name, module in model.named_modules():
             if "norm" in name:
                 module.to(cfg.torch_dtype)
-            if any(m in name for m in embedding_modules):
+            if "lm_head" in name or "embed_tokens" in name:
                 if hasattr(module, "weight"):
                     module.to(cfg.torch_dtype)
 

src/axolotl/utils/samplers/__init__.py

@@ -2,4 +2,3 @@
 axolotl samplers module
 """
 from .multipack import MultipackBatchSampler  # noqa: F401
-from .utils import get_dataset_lengths  # noqa: F401

src/axolotl/utils/samplers/utils.py

@@ -1,17 +0,0 @@
-"""
-helper util to calculate dataset lengths
-"""
-import numpy as np
-
-
-def get_dataset_lengths(dataset):
-    if "length" in dataset.data.column_names:
-        lengths = np.array(dataset.data.column("length"))
-    else:
-        lengths = (
-            dataset.data.column("position_ids")
-            .to_pandas()
-            .apply(lambda x: x[-1] + 1)
-            .values
-        )
-    return lengths

src/axolotl/utils/schedulers.py

@@ -100,43 +100,3 @@ def get_cosine_schedule_with_quadratic_warmup(
         num_cycles=num_cycles,
     )
     return LambdaLR(optimizer, lr_lambda, last_epoch)
-
-
-def _get_cosine_schedule_with_min_lr_lambda(
-    current_step: int,
-    *,
-    num_warmup_steps: int,
-    num_training_steps: int,
-    min_lr_ratio: float
-):
-    # Warm up
-    if current_step < num_warmup_steps:
-        return float(current_step) / float(max(1, num_warmup_steps))
-
-    # Cosine learning rate decay
-    progress = float(current_step - num_warmup_steps) / float(
-        max(1, num_training_steps - num_warmup_steps)
-    )
-    scaling = 0.5 * (1.0 + math.cos(math.pi * progress))
-    return (1 - min_lr_ratio) * scaling + min_lr_ratio
-
-
-def get_cosine_schedule_with_min_lr(
-    optimizer: Optimizer,
-    num_warmup_steps: int,
-    num_training_steps: int,
-    min_lr_ratio: float = 0.0,
-):
-    """
-    Create a learning rate schedule which has:
-        - linear warmup from 0 -> `max_lr` over `num_warmup_steps`
-        - cosine learning rate annealing from `max_lr` -> `min_lr` over `num_training_steps`
-    """
-
-    lr_lambda = partial(
-        _get_cosine_schedule_with_min_lr_lambda,
-        num_warmup_steps=num_warmup_steps,
-        num_training_steps=num_training_steps,
-        min_lr_ratio=min_lr_ratio,
-    )
-    return LambdaLR(optimizer, lr_lambda)

src/axolotl/utils/tokenization.py

@@ -31,8 +31,8 @@ def check_example_labels(example, tokenizer, text_only=False):
         )
         colored_tokens.append(colored_token)
 
-    delimiter = "" if text_only else " "
-    LOG.info(delimiter.join(colored_tokens))
+    output = " ".join(colored_tokens)
+    LOG.info(output)
     LOG.info("\n\n\n")
 
-    return " ".join(colored_tokens)
+    return output

src/axolotl/utils/trainer.py

@@ -14,7 +14,7 @@ from torch.utils.data import DataLoader, RandomSampler
 
 from axolotl.core.trainer_builder import HFCausalTrainerBuilder, HFDPOTrainerBuilder
 from axolotl.utils.distributed import is_main_process, reduce_and_broadcast, zero_first
-from axolotl.utils.samplers import MultipackBatchSampler, get_dataset_lengths
+from axolotl.utils.samplers import MultipackBatchSampler
 
 LOG = get_logger("axolotl")
 
@@ -108,7 +108,13 @@ def disable_datasets_caching():
 
 def process_datasets_for_packing(cfg, train_dataset, eval_dataset, tokenizer):
     drop_long = partial(drop_long_seq, sequence_len=cfg.sequence_len)
-    with zero_first(is_main_process()), disable_datasets_caching():
+    with zero_first(is_main_process()):
+        train_dataset = train_dataset.filter(drop_long, num_proc=cfg.dataset_processes)
+        if eval_dataset:
+            eval_dataset = eval_dataset.filter(
+                drop_long, num_proc=cfg.dataset_processes
+            )
+
         if cfg.group_by_length:
             train_dataset = train_dataset.map(
                 add_length, num_proc=cfg.dataset_processes
@@ -124,16 +130,6 @@ def process_datasets_for_packing(cfg, train_dataset, eval_dataset, tokenizer):
                         add_position_ids, num_proc=cfg.dataset_processes
                     )
 
-        if cfg.group_by_length or cfg.sample_packing:
-            max_input_len = np.max(get_dataset_lengths(train_dataset))
-            LOG.debug(f"max_input_len: {max_input_len}", main_process_only=True)
-
-        train_dataset = train_dataset.filter(drop_long, num_proc=cfg.dataset_processes)
-        if eval_dataset:
-            eval_dataset = eval_dataset.filter(
-                drop_long, num_proc=cfg.dataset_processes
-            )
-
         # Phi doesn't want the attention_mask feature when training
         if (
             "CodeGenTokenizer" in tokenizer.__class__.__name__
@@ -216,7 +212,12 @@ def calculate_total_num_steps(cfg, train_dataset, update=True):
                 drop_last=True,
                 batch_max_len=cfg.micro_batch_size
                 * (cfg.max_packed_sequence_len or cfg.sequence_len),
-                lengths=get_dataset_lengths(train_dataset),
+                lengths=(
+                    train_dataset.data.column("position_ids")
+                    .to_pandas()
+                    .apply(lambda x: x[-1] + 1)
+                    .values
+                ),
             )
 
             data_loader = DataLoader(

tests/core/test_trainer_builder.py

@@ -30,7 +30,6 @@ def fixture_cfg():
             "adam_epsilon": 0.00001,
             "dataloader_num_workers": 1,
             "dataloader_pin_memory": True,
-            "model_config_type": "llama",
         }
     )
 

tests/e2e/patched/test_lora_llama_multipack.py

@@ -1,126 +0,0 @@
-"""
-E2E tests for lora llama
-"""
-
-import logging
-import os
-import unittest
-from pathlib import Path
-
-import pytest
-from transformers.utils import is_auto_gptq_available, is_torch_bf16_gpu_available
-
-from axolotl.cli import load_datasets
-from axolotl.common.cli import TrainerCliArgs
-from axolotl.train import train
-from axolotl.utils.config import normalize_config
-from axolotl.utils.dict import DictDefault
-
-from ..utils import with_temp_dir
-
-LOG = logging.getLogger("axolotl.tests.e2e")
-os.environ["WANDB_DISABLED"] = "true"
-
-
-class TestLoraLlama(unittest.TestCase):
-    """
-    Test case for Llama models using LoRA w multipack
-    """
-
-    @with_temp_dir
-    def test_lora_packing(self, temp_dir):
-        # pylint: disable=duplicate-code
-        cfg = DictDefault(
-            {
-                "base_model": "JackFram/llama-68m",
-                "tokenizer_type": "LlamaTokenizer",
-                "sequence_len": 1024,
-                "sample_packing": True,
-                "flash_attention": True,
-                "load_in_8bit": True,
-                "adapter": "lora",
-                "lora_r": 32,
-                "lora_alpha": 64,
-                "lora_dropout": 0.05,
-                "lora_target_linear": True,
-                "val_set_size": 0.1,
-                "special_tokens": {
-                    "unk_token": "<unk>",
-                    "bos_token": "<s>",
-                    "eos_token": "</s>",
-                },
-                "datasets": [
-                    {
-                        "path": "mhenrichsen/alpaca_2k_test",
-                        "type": "alpaca",
-                    },
-                ],
-                "num_epochs": 2,
-                "micro_batch_size": 8,
-                "gradient_accumulation_steps": 1,
-                "output_dir": temp_dir,
-                "learning_rate": 0.00001,
-                "optimizer": "adamw_torch",
-                "lr_scheduler": "cosine",
-            }
-        )
-        if is_torch_bf16_gpu_available():
-            cfg.bf16 = True
-        else:
-            cfg.fp16 = True
-
-        normalize_config(cfg)
-        cli_args = TrainerCliArgs()
-        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
-
-        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        assert (Path(temp_dir) / "adapter_model.bin").exists()
-
-    @pytest.mark.skipif(not is_auto_gptq_available(), reason="auto-gptq not available")
-    @with_temp_dir
-    def test_lora_gptq_packed(self, temp_dir):
-        # pylint: disable=duplicate-code
-        cfg = DictDefault(
-            {
-                "base_model": "TheBlokeAI/jackfram_llama-68m-GPTQ",
-                "model_type": "AutoModelForCausalLM",
-                "tokenizer_type": "LlamaTokenizer",
-                "sequence_len": 1024,
-                "sample_packing": True,
-                "flash_attention": True,
-                "load_in_8bit": True,
-                "adapter": "lora",
-                "gptq": True,
-                "gptq_disable_exllama": True,
-                "lora_r": 32,
-                "lora_alpha": 64,
-                "lora_dropout": 0.05,
-                "lora_target_linear": True,
-                "val_set_size": 0.1,
-                "special_tokens": {
-                    "unk_token": "<unk>",
-                    "bos_token": "<s>",
-                    "eos_token": "</s>",
-                },
-                "datasets": [
-                    {
-                        "path": "mhenrichsen/alpaca_2k_test",
-                        "type": "alpaca",
-                    },
-                ],
-                "num_epochs": 2,
-                "save_steps": 0.5,
-                "micro_batch_size": 8,
-                "gradient_accumulation_steps": 1,
-                "output_dir": temp_dir,
-                "learning_rate": 0.00001,
-                "optimizer": "adamw_torch",
-                "lr_scheduler": "cosine",
-            }
-        )
-        normalize_config(cfg)
-        cli_args = TrainerCliArgs()
-        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
-
-        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        assert (Path(temp_dir) / "adapter_model.bin").exists()

tests/e2e/test_fused_llama.py

@@ -15,7 +15,7 @@ from axolotl.train import train
 from axolotl.utils.config import normalize_config
 from axolotl.utils.dict import DictDefault
 
-from ..utils import with_temp_dir
+from .utils import with_temp_dir
 
 LOG = logging.getLogger("axolotl.tests.e2e")
 os.environ["WANDB_DISABLED"] = "true"

tests/e2e/test_lora_llama.py

@@ -65,3 +65,96 @@ class TestLoraLlama(unittest.TestCase):
 
         train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
         assert (Path(temp_dir) / "adapter_model.bin").exists()
+
+    @with_temp_dir
+    def test_lora_packing(self, temp_dir):
+        # pylint: disable=duplicate-code
+        cfg = DictDefault(
+            {
+                "base_model": "JackFram/llama-68m",
+                "tokenizer_type": "LlamaTokenizer",
+                "sequence_len": 1024,
+                "sample_packing": True,
+                "flash_attention": True,
+                "load_in_8bit": True,
+                "adapter": "lora",
+                "lora_r": 32,
+                "lora_alpha": 64,
+                "lora_dropout": 0.05,
+                "lora_target_linear": True,
+                "val_set_size": 0.1,
+                "special_tokens": {
+                    "unk_token": "<unk>",
+                    "bos_token": "<s>",
+                    "eos_token": "</s>",
+                },
+                "datasets": [
+                    {
+                        "path": "mhenrichsen/alpaca_2k_test",
+                        "type": "alpaca",
+                    },
+                ],
+                "num_epochs": 2,
+                "micro_batch_size": 8,
+                "gradient_accumulation_steps": 1,
+                "output_dir": temp_dir,
+                "learning_rate": 0.00001,
+                "optimizer": "adamw_torch",
+                "lr_scheduler": "cosine",
+                "bf16": True,
+            }
+        )
+        normalize_config(cfg)
+        cli_args = TrainerCliArgs()
+        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
+
+        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
+        assert (Path(temp_dir) / "adapter_model.bin").exists()
+
+    @with_temp_dir
+    def test_lora_gptq(self, temp_dir):
+        # pylint: disable=duplicate-code
+        cfg = DictDefault(
+            {
+                "base_model": "TheBlokeAI/jackfram_llama-68m-GPTQ",
+                "model_type": "AutoModelForCausalLM",
+                "tokenizer_type": "LlamaTokenizer",
+                "sequence_len": 1024,
+                "sample_packing": True,
+                "flash_attention": True,
+                "load_in_8bit": True,
+                "adapter": "lora",
+                "gptq": True,
+                "gptq_disable_exllama": True,
+                "lora_r": 32,
+                "lora_alpha": 64,
+                "lora_dropout": 0.05,
+                "lora_target_linear": True,
+                "val_set_size": 0.1,
+                "special_tokens": {
+                    "unk_token": "<unk>",
+                    "bos_token": "<s>",
+                    "eos_token": "</s>",
+                },
+                "datasets": [
+                    {
+                        "path": "mhenrichsen/alpaca_2k_test",
+                        "type": "alpaca",
+                    },
+                ],
+                "num_epochs": 2,
+                "save_steps": 0.5,
+                "micro_batch_size": 8,
+                "gradient_accumulation_steps": 1,
+                "output_dir": temp_dir,
+                "learning_rate": 0.00001,
+                "optimizer": "adamw_torch",
+                "lr_scheduler": "cosine",
+            }
+        )
+        normalize_config(cfg)
+        cli_args = TrainerCliArgs()
+        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
+
+        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
+        assert (Path(temp_dir) / "adapter_model.bin").exists()

tests/e2e/test_mamba.py

@@ -7,8 +7,6 @@ import os
 import unittest
 from pathlib import Path
 
-import pytest
-
 from axolotl.cli import load_datasets
 from axolotl.common.cli import TrainerCliArgs
 from axolotl.train import train
@@ -21,10 +19,9 @@ LOG = logging.getLogger("axolotl.tests.e2e")
 os.environ["WANDB_DISABLED"] = "true"
 
 
-@pytest.mark.skip(reason="skipping until upstreamed into transformers")
-class TestMamba(unittest.TestCase):
+class TestMistral(unittest.TestCase):
     """
-    Test case for Mamba models
+    Test case for Llama models using LoRA
     """
 
     @with_temp_dir

tests/e2e/test_mistral_samplepack.py

@@ -15,7 +15,7 @@ from axolotl.train import train
 from axolotl.utils.config import normalize_config
 from axolotl.utils.dict import DictDefault
 
-from ..utils import with_temp_dir
+from .utils import with_temp_dir
 
 LOG = logging.getLogger("axolotl.tests.e2e")
 os.environ["WANDB_DISABLED"] = "true"

tests/e2e/test_mixtral.py

@@ -7,7 +7,6 @@ import os
 import unittest
 from pathlib import Path
 
-import torch
 from transformers.utils import is_torch_bf16_gpu_available
 
 from axolotl.cli import load_datasets
@@ -28,7 +27,7 @@ class TestMixtral(unittest.TestCase):
     """
 
     @with_temp_dir
-    def test_qlora_w_fa2(self, temp_dir):
+    def test_qlora(self, temp_dir):
         # pylint: disable=duplicate-code
         cfg = DictDefault(
             {
@@ -38,18 +37,10 @@ class TestMixtral(unittest.TestCase):
                 "sequence_len": 1024,
                 "load_in_4bit": True,
                 "adapter": "qlora",
-                "lora_r": 4,
-                "lora_alpha": 8,
+                "lora_r": 16,
+                "lora_alpha": 32,
                 "lora_dropout": 0.1,
-                "lora_target_modules": [
-                    "o_proj",
-                    "w3",
-                    "k_proj",
-                    "v_proj",
-                    "w1",
-                    "q_proj",
-                    "w2",
-                ],
+                "lora_target_linear": True,
                 "val_set_size": 0.1,
                 "special_tokens": {},
                 "datasets": [
@@ -74,179 +65,7 @@ class TestMixtral(unittest.TestCase):
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
 
-        model, _ = train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        assert (
-            model.base_model.model.model.layers[0].block_sparse_moe.gate.weight.dtype
-            == torch.uint8
-        )
-        assert (Path(temp_dir) / "adapter_model.bin").exists()
-
-    @with_temp_dir
-    def test_qlora_wo_fa2(self, temp_dir):
-        # pylint: disable=duplicate-code
-        cfg = DictDefault(
-            {
-                "base_model": "hf-internal-testing/Mixtral-tiny",
-                "tokenizer_config": "mistralai/Mixtral-8x7B-v0.1",
-                "flash_attention": False,
-                "sequence_len": 1024,
-                "load_in_4bit": True,
-                "adapter": "qlora",
-                "lora_r": 4,
-                "lora_alpha": 8,
-                "lora_dropout": 0.1,
-                "lora_target_modules": [
-                    "o_proj",
-                    "w3",
-                    "k_proj",
-                    "v_proj",
-                    "w1",
-                    "q_proj",
-                    "w2",
-                ],
-                "val_set_size": 0.1,
-                "special_tokens": {},
-                "datasets": [
-                    {
-                        "path": "mhenrichsen/alpaca_2k_test",
-                        "type": "alpaca",
-                    },
-                ],
-                "num_epochs": 2,
-                "micro_batch_size": 2,
-                "gradient_accumulation_steps": 1,
-                "output_dir": temp_dir,
-                "learning_rate": 0.00001,
-                "optimizer": "adamw_bnb_8bit",
-                "lr_scheduler": "cosine",
-                "max_steps": 20,
-                "save_steps": 10,
-                "eval_steps": 10,
-            }
-        )
-        normalize_config(cfg)
-        cli_args = TrainerCliArgs()
-        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
-
-        model, _ = train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        assert (
-            model.base_model.model.model.layers[0].block_sparse_moe.gate.weight.dtype
-            == torch.uint8
-        )
-        assert (Path(temp_dir) / "adapter_model.bin").exists()
-
-    @with_temp_dir
-    def test_16bit_lora_w_fa2(self, temp_dir):
-        # pylint: disable=duplicate-code
-        cfg = DictDefault(
-            {
-                "base_model": "hf-internal-testing/Mixtral-tiny",
-                "tokenizer_config": "mistralai/Mixtral-8x7B-v0.1",
-                "flash_attention": True,
-                "sequence_len": 1024,
-                "adapter": "lora",
-                "lora_r": 4,
-                "lora_alpha": 8,
-                "lora_dropout": 0.1,
-                "lora_target_modules": [
-                    "o_proj",
-                    "w3",
-                    "k_proj",
-                    "v_proj",
-                    "w1",
-                    "q_proj",
-                    "w2",
-                ],
-                "val_set_size": 0.1,
-                "special_tokens": {},
-                "datasets": [
-                    {
-                        "path": "mhenrichsen/alpaca_2k_test",
-                        "type": "alpaca",
-                    },
-                ],
-                "num_epochs": 2,
-                "micro_batch_size": 2,
-                "gradient_accumulation_steps": 1,
-                "output_dir": temp_dir,
-                "learning_rate": 0.00001,
-                "optimizer": "adamw_bnb_8bit",
-                "lr_scheduler": "cosine",
-                "max_steps": 20,
-                "save_steps": 10,
-                "eval_steps": 10,
-            }
-        )
-        if is_torch_bf16_gpu_available():
-            cfg.bf16 = True
-        else:
-            cfg.fp16 = True
-        normalize_config(cfg)
-        cli_args = TrainerCliArgs()
-        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
-
-        model, _ = train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        assert (
-            model.base_model.model.model.layers[0].block_sparse_moe.gate.weight.dtype
-            == torch.float32
-        )
-        assert (Path(temp_dir) / "adapter_model.bin").exists()
-
-    @with_temp_dir
-    def test_16bit_lora_wo_fa2(self, temp_dir):
-        # pylint: disable=duplicate-code
-        cfg = DictDefault(
-            {
-                "base_model": "hf-internal-testing/Mixtral-tiny",
-                "tokenizer_config": "mistralai/Mixtral-8x7B-v0.1",
-                "flash_attention": False,
-                "sequence_len": 1024,
-                "adapter": "lora",
-                "lora_r": 4,
-                "lora_alpha": 8,
-                "lora_dropout": 0.1,
-                "lora_target_modules": [
-                    "o_proj",
-                    "w3",
-                    "k_proj",
-                    "v_proj",
-                    "w1",
-                    "q_proj",
-                    "w2",
-                ],
-                "val_set_size": 0.1,
-                "special_tokens": {},
-                "datasets": [
-                    {
-                        "path": "mhenrichsen/alpaca_2k_test",
-                        "type": "alpaca",
-                    },
-                ],
-                "num_epochs": 2,
-                "micro_batch_size": 2,
-                "gradient_accumulation_steps": 1,
-                "output_dir": temp_dir,
-                "learning_rate": 0.00001,
-                "optimizer": "adamw_bnb_8bit",
-                "lr_scheduler": "cosine",
-                "max_steps": 20,
-                "save_steps": 10,
-                "eval_steps": 10,
-            }
-        )
-        normalize_config(cfg)
-        if is_torch_bf16_gpu_available():
-            cfg.bf16 = True
-        else:
-            cfg.fp16 = True
-        cli_args = TrainerCliArgs()
-        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
-
-        model, _ = train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
-        assert (
-            model.base_model.model.model.layers[0].block_sparse_moe.gate.weight.dtype
-            == torch.float32
-        )
+        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
         assert (Path(temp_dir) / "adapter_model.bin").exists()
 
     @with_temp_dir

tests/e2e/test_mixtral_samplepack.py

@@ -15,7 +15,7 @@ from axolotl.train import train
 from axolotl.utils.config import normalize_config
 from axolotl.utils.dict import DictDefault
 
-from ..utils import with_temp_dir
+from .utils import with_temp_dir
 
 LOG = logging.getLogger("axolotl.tests.e2e")
 os.environ["WANDB_DISABLED"] = "true"

tests/e2e/test_model_patches.py

@@ -9,7 +9,7 @@ from axolotl.utils.config import normalize_config
 from axolotl.utils.dict import DictDefault
 from axolotl.utils.models import load_model, load_tokenizer
 
-from ..utils import with_temp_dir
+from .utils import with_temp_dir
 
 
 class TestModelPatches(unittest.TestCase):

tests/e2e/test_phi.py

@@ -7,9 +7,6 @@ import os
 import unittest
 from pathlib import Path
 
-import pytest
-from transformers.utils import is_torch_bf16_gpu_available
-
 from axolotl.cli import load_datasets
 from axolotl.common.cli import TrainerCliArgs
 from axolotl.train import train
@@ -24,18 +21,17 @@ os.environ["WANDB_DISABLED"] = "true"
 
 class TestPhi(unittest.TestCase):
     """
-    Test case for Phi2 models
+    Test case for Llama models using LoRA
     """
 
-    @pytest.mark.skip(reason="fixme later")
     @with_temp_dir
-    def test_phi2_ft(self, temp_dir):
+    def test_ft(self, temp_dir):
         # pylint: disable=duplicate-code
         cfg = DictDefault(
             {
-                "base_model": "microsoft/phi-2",
+                "base_model": "microsoft/phi-1_5",
                 "trust_remote_code": True,
-                "model_type": "AutoModelForCausalLM",
+                "model_type": "PhiForCausalLM",
                 "tokenizer_type": "AutoTokenizer",
                 "sequence_len": 512,
                 "sample_packing": False,
@@ -43,6 +39,9 @@ class TestPhi(unittest.TestCase):
                 "adapter": None,
                 "val_set_size": 0.1,
                 "special_tokens": {
+                    "unk_token": "<|endoftext|>",
+                    "bos_token": "<|endoftext|>",
+                    "eos_token": "<|endoftext|>",
                     "pad_token": "<|endoftext|>",
                 },
                 "datasets": [
@@ -58,19 +57,11 @@ class TestPhi(unittest.TestCase):
                 "gradient_accumulation_steps": 1,
                 "output_dir": temp_dir,
                 "learning_rate": 0.00001,
-                "optimizer": "paged_adamw_8bit",
+                "optimizer": "adamw_bnb_8bit",
                 "lr_scheduler": "cosine",
-                "flash_attention": True,
-                "max_steps": 10,
-                "save_steps": 10,
-                "eval_steps": 10,
-                "save_safetensors": True,
+                "bf16": True,
             }
         )
-        if is_torch_bf16_gpu_available():
-            cfg.bf16 = True
-        else:
-            cfg.fp16 = True
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
@@ -78,13 +69,12 @@ class TestPhi(unittest.TestCase):
         train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
         assert (Path(temp_dir) / "pytorch_model.bin").exists()
 
-    @pytest.mark.skip(reason="multipack no longer supported atm")
     @with_temp_dir
     def test_ft_packed(self, temp_dir):
         # pylint: disable=duplicate-code
         cfg = DictDefault(
             {
-                "base_model": "microsoft/phi-2",
+                "base_model": "microsoft/phi-1_5",
                 "trust_remote_code": True,
                 "model_type": "PhiForCausalLM",
                 "tokenizer_type": "AutoTokenizer",
@@ -114,13 +104,9 @@ class TestPhi(unittest.TestCase):
                 "learning_rate": 0.00001,
                 "optimizer": "adamw_bnb_8bit",
                 "lr_scheduler": "cosine",
+                "bf16": True,
             }
         )
-        if is_torch_bf16_gpu_available():
-            cfg.bf16 = True
-        else:
-            cfg.fp16 = True
-
         normalize_config(cfg)
         cli_args = TrainerCliArgs()
         dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)

tests/e2e/test_resume.py

@@ -17,7 +17,7 @@ from axolotl.train import train
 from axolotl.utils.config import normalize_config
 from axolotl.utils.dict import DictDefault
 
-from ..utils import most_recent_subdir, with_temp_dir
+from .utils import most_recent_subdir, with_temp_dir
 
 LOG = logging.getLogger("axolotl.tests.e2e")
 os.environ["WANDB_DISABLED"] = "true"
@@ -29,7 +29,7 @@ class TestResumeLlama(unittest.TestCase):
     """
 
     @with_temp_dir
-    def test_resume_qlora_packed(self, temp_dir):
+    def test_resume_qlora(self, temp_dir):
         # pylint: disable=duplicate-code
         cfg = DictDefault(
             {

tests/prompt_strategies/test_sharegpt.py

@@ -1,153 +0,0 @@
-"""
-Test module for sharegpt integration w chatml
-"""
-import pytest
-from datasets import Dataset
-from tokenizers import AddedToken
-from transformers import AutoTokenizer
-
-from axolotl.datasets import TokenizedPromptDataset
-from axolotl.prompt_strategies.sharegpt import SimpleShareGPTPromptTokenizingStrategy
-from axolotl.prompters import ShareGPTPrompterV2
-
-
-@pytest.fixture(name="sharegpt_dataset")
-def fixture_sharegpt_dataset():
-    return Dataset.from_list(
-        [
-            {
-                "conversations": [
-                    {
-                        "from": "system",
-                        "value": "repeat",
-                    },
-                    {
-                        "from": "human",
-                        "value": "hello",
-                    },
-                    {
-                        "from": "gpt",
-                        "value": "hello",
-                    },
-                    {
-                        "from": "human",
-                        "value": "goodbye",
-                    },
-                    {
-                        "from": "gpt",
-                        "value": "goodbye",
-                    },
-                ]
-            }
-        ]
-    )
-
-
-@pytest.fixture(name="tokenizer")
-def fixture_tokenizer():
-    tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
-    tokenizer.add_special_tokens(
-        {
-            "eos_token": AddedToken(
-                "<|im_end|>", rstrip=False, lstrip=False, normalized=False
-            )
-        }
-    )
-    tokenizer.add_tokens(
-        [
-            AddedToken("<|im_start|>", rstrip=False, lstrip=False, normalized=False),
-        ]
-    )
-
-    return tokenizer
-
-
-class TestSharegpt:
-    """
-    Test class for sharegpt prompter
-    """
-
-    def test_no_double_im_end(self, sharegpt_dataset, tokenizer):
-        strategy = SimpleShareGPTPromptTokenizingStrategy(
-            ShareGPTPrompterV2(
-                conversation="chatml",
-                role_key_model=None,
-                role_key_human=None,
-            ),
-            tokenizer,
-            False,  # train_on_inputs
-            2048,  # sequence_len
-        )
-
-        dataset_wrapper = TokenizedPromptDataset(
-            strategy, sharegpt_dataset, process_count=1
-        )
-
-        input_ids = dataset_wrapper[0]["input_ids"]
-        # fmt: off
-        assert input_ids == [
-            #  28705, 13, is " \n"
-            1,   # bos
-            32001, 1587, 13, 25997, 32000, 28705, 13,  # system
-            32001, 2188, 13, 21558, 32000, 28705, 13,  # human
-            32001, 13892, 13, 21558, 32000, 28705, 13,  # gpt
-            32001, 2188, 13, 12684, 17664, 32000, 28705, 13,   # human
-            32001, 13892, 13, 12684, 17664, 32000, 28705, 13,  # gpt
-        ]
-        # fmt: on
-
-    def test_w_train_on_input(self, sharegpt_dataset, tokenizer):
-        strategy = SimpleShareGPTPromptTokenizingStrategy(
-            ShareGPTPrompterV2(
-                conversation="chatml",
-                role_key_model=None,
-                role_key_human=None,
-            ),
-            tokenizer,
-            False,  # train_on_inputs
-            2048,  # sequence_len
-        )
-
-        dataset_wrapper = TokenizedPromptDataset(
-            strategy, sharegpt_dataset, process_count=1
-        )
-
-        labels = dataset_wrapper[0]["labels"]
-        # fmt: off
-        assert labels == [
-            -100,   # bos
-            -100, -100, -100, -100, -100, -100, -100,  # system
-            -100, -100, -100, -100, -100, -100, -100,  # human
-            -100, -100, 13, 21558, 32000, 28705, 13,  # gpt
-            -100, -100, -100, -100, -100, -100, -100, -100,   # human
-            -100, -100, 13, 12684, 17664, 32000, 28705, 13,  # gpt
-        ]
-        # fmt: on
-
-    def test_no_train_on_input(self, sharegpt_dataset, tokenizer):
-        strategy = SimpleShareGPTPromptTokenizingStrategy(
-            ShareGPTPrompterV2(
-                conversation="chatml",
-                role_key_model=None,
-                role_key_human=None,
-            ),
-            tokenizer,
-            True,  # train_on_inputs
-            2048,  # sequence_len
-        )
-
-        dataset_wrapper = TokenizedPromptDataset(
-            strategy, sharegpt_dataset, process_count=1
-        )
-
-        labels = dataset_wrapper[0]["labels"]
-        # fmt: off
-        assert labels == [
-            1,   # bos
-            32001, 1587, 13, 25997, 32000, 28705, 13,  # system
-            32001, 2188, 13, 21558, 32000, 28705, 13,  # human
-            32001, 13892, 13, 21558, 32000, 28705, 13,  # gpt
-            32001, 2188, 13, 12684, 17664, 32000, 28705, 13,   # human
-            32001, 13892, 13, 12684, 17664, 32000, 28705, 13,  # gpt
-        ]
-        # fmt: on

tests/test_normalize_config.py

@@ -2,9 +2,8 @@
 Test classes for checking functionality of the cfg normalization
 """
 import unittest
-from unittest.mock import patch
 
-from axolotl.utils.config import normalize_cfg_datasets, normalize_config
+from axolotl.utils.config import normalize_config
 from axolotl.utils.dict import DictDefault
 
 
@@ -45,46 +44,3 @@ class NormalizeConfigTestCase(unittest.TestCase):
         normalize_config(cfg)
 
         assert cfg.base_model_config == cfg.base_model
-
-    def test_chat_template_chatml(self):
-        cfg = DictDefault(
-            {
-                "chat_template": "chatml",
-                "datasets": [
-                    {
-                        "path": "lorem/ipsum",
-                        "type": "sharegpt",
-                        "conversation": "vicuna_v1.1",
-                    },
-                    {
-                        "path": "sit/amet",
-                        "type": "sharegpt",
-                    },
-                ],
-            }
-        )
-
-        normalize_cfg_datasets(cfg)
-
-        assert cfg.datasets[0].conversation == "vicuna_v1.1"
-        assert cfg.datasets[1].conversation == "chatml"
-
-    @patch("axolotl.utils.config.is_torch_bf16_gpu_available")
-    def test_bf16_auto_setter_available(self, mock_bf16_avail):
-        cfg = self._get_base_cfg()
-        cfg.bf16 = "auto"
-        mock_bf16_avail.return_value = True
-
-        normalize_config(cfg)
-
-        self.assertTrue(cfg.bf16)
-
-    @patch("axolotl.utils.config.is_torch_bf16_gpu_available")
-    def test_bf16_auto_setter_not_available(self, mock_bf16_avail):
-        cfg = self._get_base_cfg()
-        cfg.bf16 = "auto"
-        mock_bf16_avail.return_value = False
-
-        normalize_config(cfg)
-
-        self.assertFalse(cfg.bf16)

tests/test_validation.py

@@ -6,17 +6,15 @@ import unittest
 from typing import Optional
 
 import pytest
-from transformers.utils import is_torch_bf16_gpu_available
 
 from axolotl.utils.config import validate_config
 from axolotl.utils.dict import DictDefault
-from axolotl.utils.models import check_model_config
 from axolotl.utils.wandb_ import setup_wandb_env_vars
 
 
-class BaseValidation(unittest.TestCase):
+class ValidationTest(unittest.TestCase):
     """
-    Base validation module to setup the log capture
+    Test the validation module
     """
 
     _caplog: Optional[pytest.LogCaptureFixture] = None
@@ -25,12 +23,6 @@ class BaseValidation(unittest.TestCase):
     def inject_fixtures(self, caplog):
         self._caplog = caplog
 
-
-class ValidationTest(BaseValidation):
-    """
-    Test the validation module
-    """
-
     def test_load_4bit_deprecate(self):
         cfg = DictDefault(
             {
@@ -362,10 +354,6 @@ class ValidationTest(BaseValidation):
         with pytest.raises(ValueError, match=regex_exp):
             validate_config(cfg)
 
-    @pytest.mark.skipif(
-        is_torch_bf16_gpu_available(),
-        reason="test should only run on gpus w/o bf16 support",
-    )
     def test_merge_lora_no_bf16_fail(self):
         """
         This is assumed to be run on a CPU machine, so bf16 is not supported.
@@ -694,23 +682,16 @@ class ValidationTest(BaseValidation):
 
         validate_config(cfg)
 
-
-class ValidationCheckModelConfig(BaseValidation):
-    """
-    Test the validation for the config when the model config is available
-    """
-
-    def test_llama_add_tokens_adapter(self):
+    def test_add_tokens_adapter(self):
         cfg = DictDefault(
             {"adapter": "qlora", "load_in_4bit": True, "tokens": ["<|imstart|>"]}
         )
-        model_config = DictDefault({"model_type": "llama"})
 
         with pytest.raises(
             ValueError,
-            match=r".*`lora_modules_to_save` not properly set when adding new tokens*",
+            match=r".*lora_modules_to_save not properly set yet adding new tokens*",
         ):
-            check_model_config(cfg, model_config)
+            validate_config(cfg)
 
         cfg = DictDefault(
             {
@@ -723,9 +704,9 @@ class ValidationCheckModelConfig(BaseValidation):
 
         with pytest.raises(
             ValueError,
-            match=r".*`lora_modules_to_save` not properly set when adding new tokens*",
+            match=r".*lora_modules_to_save not properly set yet adding new tokens*",
         ):
-            check_model_config(cfg, model_config)
+            validate_config(cfg)
 
         cfg = DictDefault(
             {
@@ -736,48 +717,10 @@ class ValidationCheckModelConfig(BaseValidation):
             }
         )
 
-        check_model_config(cfg, model_config)
-
-    def test_phi2_add_tokens_adapter(self):
-        cfg = DictDefault(
-            {"adapter": "qlora", "load_in_4bit": True, "tokens": ["<|imstart|>"]}
-        )
-        model_config = DictDefault({"model_type": "phi-msft"})
-
-        with pytest.raises(
-            ValueError,
-            match=r".*`lora_modules_to_save` not properly set when adding new tokens*",
-        ):
-            check_model_config(cfg, model_config)
-
-        cfg = DictDefault(
-            {
-                "adapter": "qlora",
-                "load_in_4bit": True,
-                "tokens": ["<|imstart|>"],
-                "lora_modules_to_save": ["embed_tokens", "lm_head"],
-            }
-        )
-
-        with pytest.raises(
-            ValueError,
-            match=r".*`lora_modules_to_save` not properly set when adding new tokens*",
-        ):
-            check_model_config(cfg, model_config)
-
-        cfg = DictDefault(
-            {
-                "adapter": "qlora",
-                "load_in_4bit": True,
-                "tokens": ["<|imstart|>"],
-                "lora_modules_to_save": ["embd.wte", "lm_head.linear"],
-            }
-        )
-
-        check_model_config(cfg, model_config)
+        validate_config(cfg)
 
 
-class ValidationWandbTest(BaseValidation):
+class ValidationWandbTest(ValidationTest):
     """
     Validation test for wandb
     """
@@ -835,15 +778,6 @@ class ValidationWandbTest(BaseValidation):
         assert os.environ.get("WANDB_LOG_MODEL", "") == "checkpoint"
         assert os.environ.get("WANDB_DISABLED", "") != "true"
 
-        os.environ.pop("WANDB_PROJECT", None)
-        os.environ.pop("WANDB_NAME", None)
-        os.environ.pop("WANDB_RUN_ID", None)
-        os.environ.pop("WANDB_ENTITY", None)
-        os.environ.pop("WANDB_MODE", None)
-        os.environ.pop("WANDB_WATCH", None)
-        os.environ.pop("WANDB_LOG_MODEL", None)
-        os.environ.pop("WANDB_DISABLED", None)
-
     def test_wandb_set_disabled(self):
         cfg = DictDefault({})
 
@@ -864,6 +798,3 @@ class ValidationWandbTest(BaseValidation):
         setup_wandb_env_vars(cfg)
 
         assert os.environ.get("WANDB_DISABLED", "") != "true"
-
-        os.environ.pop("WANDB_PROJECT", None)
-        os.environ.pop("WANDB_DISABLED", None)