set torchao quant config on config.json of saved model

.axolotl-complete.bash

@@ -1,41 +0,0 @@
-#!/bin/bash
-
-_axolotl_completions() {
-    local cur prev
-    COMPREPLY=()
-    cur="${COMP_WORDS[COMP_CWORD]}"
-    prev="${COMP_WORDS[COMP_CWORD-1]}"
-
-    # If we're completing the first argument (the command)
-    if [[ $COMP_CWORD -eq 1 ]]; then
-        mapfile -t COMPREPLY < <(compgen -W "delinearize-llama4 fetch lm-eval merge-sharded-fsdp-weights quantize vllm-serve evaluate inference merge-lora preprocess train" -- "$cur")
-        return 0
-    fi
-
-    # Commands that should complete with directories and YAML files
-    local -a yaml_commands=("merge-sharded-fsdp-weights" "quantize" "vllm-serve" "evaluate" "inference" "merge-lora" "preprocess" "train")
-
-    # Check if previous word is in our list
-    if [[ " ${yaml_commands[*]} " =~ (^|[[:space:]])$prev($|[[:space:]]) ]]; then
-        # Use filename completion which handles directories properly
-        compopt -o filenames
-        mapfile -t COMPREPLY < <(compgen -f -- "$cur")
-
-        # Filter to only include directories and YAML files
-        local -a filtered=()
-        for item in "${COMPREPLY[@]}"; do
-            if [[ -d "$item" ]] || [[ "$item" == *.yaml ]] || [[ "$item" == *.yml ]]; then
-                filtered+=("$item")
-            fi
-        done
-        COMPREPLY=("${filtered[@]}")
-
-        return 0
-    fi
-
-    # Default: no completion
-    return 0
-}
-
-# Remove the -o nospace option - let filenames handle it
-complete -F _axolotl_completions axolotl

.bandit

@@ -1,3 +1,3 @@
 [bandit]
 exclude = tests
-skips = B101,B615,B102,B110
+skips = B101,B615

.coderabbit.yaml

@@ -12,6 +12,5 @@ reviews:
   auto_review:
     enabled: true
     drafts: false
-    auto_incremental_review: false
 chat:
   auto_reply: true

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+max-line-length = 88
+
+select = C,E,F,W,B,B950
+extend-ignore = E203, E501, W503

.github/CONTRIBUTING.md

@@ -57,13 +57,6 @@ We welcome ideas for improvements and new features. To suggest an enhancement, o
 5. Push your branch to your fork on GitHub.
 6. Open a new pull request against the `main` branch of the axolotl repository. Include a clear and concise description of your changes, referencing any related issues.
 
-#### Skipping CI Checks
-
-You can skip certain CI checks by including specific keywords in your commit messages:
-
-- `[skip ci]` or `skip ci` - Skips all CI checks for that commit
-- `[skip-e2e]` or `skip-e2e` - Skips only end-to-end tests while running other CI checks. You may also include this in the title of your PR to disable end-to-end tests for the entire PR.
-
 ## Style Guidelines
 
 ### Code Style

.github/FUNDING.yml

@@ -1,13 +1,13 @@
 # These are supported funding model platforms
 
-github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+github: [winglian, OpenAccess-AI-Collective] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
 patreon: # Replace with a single Patreon username
 open_collective: # Replace with a single Open Collective username
-ko_fi: # Replace with a single Ko-fi username
+ko_fi: axolotl_ai # Replace with a single Ko-fi username
 tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
 community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
 liberapay: # Replace with a single Liberapay username
 issuehunt: # Replace with a single IssueHunt username
 otechie: # Replace with a single Otechie username
 lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
-custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
+custom: ['https://quickchart.io/qr?text=bitcoin%3Abc1qxlgwlqwfea5s2cxm42xqsfmwjct0rj8w8ea5np&size=480&centerImageUrl=https%3A%2F%2Fupload.wikimedia.org%2Fwikipedia%2Fcommons%2Fthumb%2F4%2F46%2FBitcoin.svg%2F64px-Bitcoin.svg.png'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,11 +15,6 @@
 <!--- Include details of your testing environment, tests ran to see how -->
 <!--- your change affects other areas of the code, etc. -->
 
-## AI Usage Disclaimer
-
-<!--- Was AI (e.g., ChatGPT, Claude, Copilot) used to generate or assist with this PR? -->
-<!--- Please indicate: No / Yes (specify which tool and to what extent) -->
-
 ## Screenshots (if appropriate)
 
 ## Types of changes

.github/workflows/base.yml

@@ -17,55 +17,56 @@ on:
 
 jobs:
   build-base:
-    if: ${{ github.repository_owner == 'axolotl-ai-cloud' && (github.event_name != 'pull_request' || !github.event.pull_request.draft) }}
+    if: github.repository_owner == 'axolotl-ai-cloud'
     timeout-minutes: 480
     # this job needs to be run on self-hosted GPU runners...
     runs-on: ubuntu-latest-m
-    env:
-      HAS_DOCKERHUB_CREDS: ${{ secrets.DOCKERHUB_USERNAME != '' && secrets.DOCKERHUB_TOKEN != '' }}
     strategy:
       fail-fast: false
       matrix:
         include:
+          - cuda: "124"
+            cuda_version: 12.4.1
+            cudnn_version: ""
+            python_version: "3.11"
+            pytorch: 2.6.0
+            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
+            dockerfile: "Dockerfile-base"
+          - cuda: "126"
+            cuda_version: 12.6.3
+            cudnn_version: ""
+            python_version: "3.11"
+            pytorch: 2.6.0
+            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
+            dockerfile: "Dockerfile-base"
+          - cuda: "126"
+            cuda_version: 12.6.3
+            cudnn_version: ""
+            python_version: "3.11"
+            pytorch: 2.7.0
+            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
+            dockerfile: "Dockerfile-base"
+          - cuda: "126"
+            cuda_version: 12.6.3
+            cudnn_version: ""
+            python_version: "3.11"
+            pytorch: 2.7.1
+            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
+            dockerfile: "Dockerfile-base"
+          - cuda: "128"
+            cuda_version: 12.6.3
+            cudnn_version: ""
+            python_version: "3.11"
+            pytorch: 2.7.1
+            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
+            dockerfile: "Dockerfile-base"
           - cuda: "128"
             cuda_version: 12.8.1
             cudnn_version: ""
             python_version: "3.11"
-            pytorch: 2.8.0
+            pytorch: nightly
             torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
-            dockerfile: "Dockerfile-base"
+            dockerfile: "Dockerfile-base-nightly"
-            platforms: "linux/amd64"
-          - cuda: "128"
-            cuda_version: 12.8.1
-            cudnn_version: ""
-            python_version: "3.11"
-            pytorch: 2.9.0
-            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
-            dockerfile: "Dockerfile-base"
-            platforms: "linux/amd64,linux/arm64"
-          - cuda: "128"
-            cuda_version: 12.8.1
-            cudnn_version: ""
-            python_version: "3.11"
-            pytorch: 2.9.1
-            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
-            dockerfile: "Dockerfile-base"
-            platforms: "linux/amd64,linux/arm64"
-          - cuda: "130"
-            cuda_version: 13.0.0
-            cudnn_version: ""
-            python_version: "3.11"
-            pytorch: 2.9.1
-            torch_cuda_arch_list: "9.0+PTX"
-            dockerfile: "Dockerfile-base"
-            platforms: "linux/amd64,linux/arm64"
-#          - cuda: "128"
-#            cuda_version: 12.8.1
-#            cudnn_version: ""
-#            python_version: "3.11"
-#            pytorch: nightly
-#            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
-#            dockerfile: "Dockerfile-base-nightly"
 #          # "next" is for release candidates of pytorch
 #          - cuda: "128"
 #            cuda_version: 12.8.1
@@ -82,10 +83,10 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: |
+            winglian/axolotl-base
             axolotlai/axolotl-base
       - name: Login to Docker Hub
         uses: docker/login-action@v2
-        if: ${{ github.event_name != 'pull_request' && env.HAS_DOCKERHUB_CREDS == 'true' }}
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -96,7 +97,6 @@ jobs:
         with:
           context: .
           file: ./docker/${{ matrix.dockerfile }}
-          platforms: ${{ matrix.platforms }}
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.metadata.outputs.tags }}-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}${{ matrix.axolotl_extras != '' && '-' || '' }}${{ matrix.axolotl_extras }}
           labels: ${{ steps.metadata.outputs.labels }}
@@ -108,47 +108,27 @@ jobs:
             PYTORCH_VERSION=${{ matrix.pytorch }}
             TORCH_CUDA_ARCH_LIST=${{ matrix.torch_cuda_arch_list }}
   build-base-uv:
-    if: ${{ github.repository_owner == 'axolotl-ai-cloud' && (github.event_name != 'pull_request' || !github.event.pull_request.draft) }}
+    if: github.repository_owner == 'axolotl-ai-cloud'
     timeout-minutes: 480
     runs-on: ubuntu-latest-m
-    env:
-      HAS_DOCKERHUB_CREDS: ${{ secrets.DOCKERHUB_USERNAME != '' && secrets.DOCKERHUB_TOKEN != '' }}
     strategy:
       fail-fast: false
       matrix:
         include:
+          - cuda: "126"
+            cuda_version: 12.6.3
+            cudnn_version: ""
+            python_version: "3.11"
+            pytorch: 2.6.0
+            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
+            dockerfile: "Dockerfile-uv-base"
           - cuda: "128"
             cuda_version: 12.8.1
             cudnn_version: ""
             python_version: "3.11"
-            pytorch: 2.8.0
+            pytorch: 2.7.1
             torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
             dockerfile: "Dockerfile-uv-base"
-            platforms: "linux/amd64"
-          - cuda: "128"
-            cuda_version: 12.8.1
-            cudnn_version: ""
-            python_version: "3.11"
-            pytorch: 2.9.1
-            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
-            dockerfile: "Dockerfile-uv-base"
-            platforms: "linux/amd64,linux/arm64"
-          - cuda: "128"
-            cuda_version: 12.8.1
-            cudnn_version: ""
-            python_version: "3.11"
-            pytorch: 2.9.0
-            torch_cuda_arch_list: "7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
-            dockerfile: "Dockerfile-uv-base"
-            platforms: "linux/amd64,linux/arm64"
-          - cuda: "130"
-            cuda_version: 13.0.0
-            cudnn_version: ""
-            python_version: "3.11"
-            pytorch: 2.9.1
-            torch_cuda_arch_list: "9.0+PTX"
-            dockerfile: "Dockerfile-uv-base"
-            platforms: "linux/amd64,linux/arm64"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -160,7 +140,6 @@ jobs:
             axolotlai/axolotl-base-uv
       - name: Login to Docker Hub
         uses: docker/login-action@v2
-        if: ${{ github.event_name != 'pull_request' && env.HAS_DOCKERHUB_CREDS == 'true' }}
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -171,7 +150,6 @@ jobs:
         with:
           context: .
           file: ./docker/${{ matrix.dockerfile }}
-          platforms: ${{ matrix.platforms }}
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.metadata.outputs.tags }}-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}${{ matrix.axolotl_extras != '' && '-' || '' }}${{ matrix.axolotl_extras }}
           labels: ${{ steps.metadata.outputs.labels }}

.github/workflows/docs.yml

@@ -12,9 +12,6 @@ jobs:
     build-deploy:
         runs-on: ubuntu-latest
         steps:
-        - name: cleanup node
-          run: |
-            sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
         - name: Check out repository
           uses: actions/checkout@v4
         - name: Set up Quarto

.github/workflows/lint.yml

@@ -3,7 +3,6 @@ on:
   # check on PRs, and manual triggers
   merge_group:
   pull_request:
-      types: [opened, synchronize, reopened, ready_for_review]
       paths:
        - '**.py'
        - 'requirements.txt'
@@ -17,7 +16,6 @@ jobs:
   pre-commit:
     name: pre-commit
     runs-on: ubuntu-latest
-    if: ${{ !github.event.pull_request.draft }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5

.github/workflows/main.yml

@@ -15,31 +15,26 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - cuda: 126
+            cuda_version: 12.6.3
+            python_version: "3.11"
+            pytorch: 2.6.0
+            axolotl_extras:
+          - cuda: 126
+            cuda_version: 12.6.3
+            python_version: "3.11"
+            pytorch: 2.7.0
+            axolotl_extras: vllm
+          - cuda: 126
+            cuda_version: 12.6.3
+            python_version: "3.11"
+            pytorch: 2.7.1
+            axolotl_extras:
           - cuda: 128
             cuda_version: 12.8.1
             python_version: "3.11"
-            pytorch: 2.8.0
+            pytorch: 2.7.1
             axolotl_extras:
-            platforms: "linux/amd64"
-          - cuda: 128
-            cuda_version: 12.8.1
-            python_version: "3.11"
-            pytorch: 2.9.0
-            axolotl_extras:
-            platforms: "linux/amd64,linux/arm64"
-          - cuda: 128
-            cuda_version: 12.8.1
-            python_version: "3.11"
-            pytorch: 2.9.1
-            axolotl_extras:
-            platforms: "linux/amd64,linux/arm64"
-            is_latest: true
-          - cuda: 130
-            cuda_version: 13.0.0
-            python_version: "3.11"
-            pytorch: 2.9.1
-            axolotl_extras:
-            platforms: "linux/amd64,linux/arm64"
     runs-on: axolotl-gpu-runner
     steps:
       - name: Checkout
@@ -49,6 +44,7 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: |
+            winglian/axolotl
             axolotlai/axolotl
           tags: |
             type=ref,event=branch
@@ -65,7 +61,6 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          platforms: ${{ matrix.platforms }}
           build-args: |
             BASE_TAG=${{ github.ref_type == 'tag' && 'main' || github.ref_name }}-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}
             CUDA=${{ matrix.cuda }}
@@ -87,31 +82,27 @@ jobs:
     strategy:
       matrix:
         include:
-          - cuda: 128
+          - cuda: 126
-            cuda_version: 12.8.1
+            cuda_version: 12.6.3
             python_version: "3.11"
-            pytorch: 2.8.0
+            pytorch: 2.6.0
             axolotl_extras:
-            platforms: "linux/amd64"
+          - cuda: 126
-          - cuda: 128
+            cuda_version: 12.6.3
-            cuda_version: 12.8.1
             python_version: "3.11"
-            pytorch: 2.9.0
+            pytorch: 2.7.0
             axolotl_extras:
-            platforms: "linux/amd64,linux/arm64"
+          - cuda: 126
-          - cuda: 128
+            cuda_version: 12.6.3
-            cuda_version: 12.8.1
             python_version: "3.11"
-            pytorch: 2.9.1
+            pytorch: 2.7.1
             axolotl_extras:
             is_latest: true
-            platforms: "linux/amd64,linux/arm64"
+          - cuda: 128
-          - cuda: 130
+            cuda_version: 12.8.1
-            cuda_version: 13.0.0
             python_version: "3.11"
-            pytorch: 2.9.1
+            pytorch: 2.7.1
             axolotl_extras:
-            platforms: "linux/amd64,linux/arm64"
     runs-on: axolotl-gpu-runner
     steps:
       - name: Checkout
@@ -121,6 +112,7 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: |
+            winglian/axolotl-cloud
             axolotlai/axolotl-cloud
           tags: |
             type=ref,event=branch
@@ -136,7 +128,6 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          platforms: ${{ matrix.platforms }}
           build-args: |
             BASE_TAG=${{ github.ref_type == 'tag' && 'main' || github.ref_name }}-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}${{ matrix.axolotl_extras != '' && '-' || '' }}${{ matrix.axolotl_extras }}
             CUDA=${{ matrix.cuda }}
@@ -154,18 +145,11 @@ jobs:
     strategy:
       matrix:
         include:
-          - cuda: 128
+          - cuda: 126
-            cuda_version: 12.8.1
+            cuda_version: 12.6.3
             python_version: "3.11"
-            pytorch: 2.9.1
+            pytorch: 2.6.0
             axolotl_extras:
-            is_latest: true
-          - cuda: 130
-            cuda_version: 13.0.0
-            python_version: "3.11"
-            pytorch: 2.9.1
-            axolotl_extras:
-            is_latest:
     runs-on: axolotl-gpu-runner
     steps:
       - name: Checkout
@@ -175,6 +159,7 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: |
+            winglian/axolotl-cloud-term
             axolotlai/axolotl-cloud-term
           tags: |
             type=ref,event=branch
@@ -190,7 +175,6 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          platforms: linux/amd64,linux/arm64
           build-args: |
             BASE_TAG=${{ github.ref_type == 'tag' && 'main' || github.ref_name }}-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}${{ matrix.axolotl_extras != '' && '-' || '' }}${{ matrix.axolotl_extras }}
             CUDA=${{ matrix.cuda }}

.github/workflows/multi-gpu-e2e.yml

@@ -19,35 +19,32 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
-env:
-  MODAL_IMAGE_BUILDER_VERSION: "2025.06"
-
 jobs:
   test-axolotl-multigpu:
-    if: ${{ ! contains(github.event.commits[0].message, '[skip e2e]') && github.repository_owner == 'axolotl-ai-cloud' && (github.event_name != 'pull_request' || !github.event.pull_request.draft) }}
+    if: ${{ ! contains(github.event.commits[0].message, '[skip e2e]') && github.repository_owner == 'axolotl-ai-cloud' }}
     strategy:
       fail-fast: false
       matrix:
         include:
-          - cuda: 128
+          - cuda: 126
-            cuda_version: 12.8.1
+            cuda_version: 12.6.3
             python_version: "3.11"
-            pytorch: 2.8.0
+            pytorch: 2.6.0
-            axolotl_extras: fbgemm-gpu
+            axolotl_extras:
             num_gpus: 2
             nightly_build: "true"
-          - cuda: 128
+          - cuda: 126
-            cuda_version: 12.8.1
+            cuda_version: 12.6.3
             python_version: "3.11"
-            pytorch: 2.9.1
+            pytorch: 2.7.0
-            axolotl_extras: fbgemm-gpu
+            axolotl_extras: vllm
             num_gpus: 2
             nightly_build: "true"
-          - cuda: 130
+          - cuda: 126
-            cuda_version: 13.0.0
+            cuda_version: 12.6.3
             python_version: "3.11"
-            pytorch: 2.9.1
+            pytorch: 2.7.1
-            axolotl_extras: fbgemm-gpu
+            axolotl_extras:
             num_gpus: 2
             nightly_build: "true"
     runs-on: [self-hosted, modal]
@@ -62,7 +59,7 @@ jobs:
       - name: Install Modal
         run: |
           python -m pip install --upgrade pip
-          pip install modal==1.3.0.post1 jinja2
+          pip install modal==1.0.2 jinja2
       - name: Update env vars
         run: |
           echo "BASE_TAG=main-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}" >> $GITHUB_ENV
@@ -75,4 +72,4 @@ jobs:
           echo "CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }}" >> $GITHUB_ENV
       - name: Run tests job on Modal
         run: |
-          modal run -m cicd.multigpu
+          modal run cicd.multigpu

.github/workflows/nightlies.yml

@@ -12,15 +12,15 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - cuda: 128
+          - cuda: 126
-            cuda_version: 12.8.1
+            cuda_version: 12.6.3
             python_version: "3.11"
-            pytorch: 2.8.0
+            pytorch: 2.6.0
             axolotl_extras:
-          - cuda: 128
+          - cuda: 126
-            cuda_version: 12.8.1
+            cuda_version: 12.6.3
             python_version: "3.11"
-            pytorch: 2.9.1
+            pytorch: 2.7.1
             axolotl_extras:
     runs-on: axolotl-gpu-runner
     steps:
@@ -31,6 +31,7 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: |
+            winglian/axolotl
             axolotlai/axolotl
           tags: |
             type=raw,value={{ branch }}-{{ date 'YYYYMMDD' }}
@@ -64,15 +65,15 @@ jobs:
     strategy:
       matrix:
         include:
-          - cuda: 128
+          - cuda: 126
-            cuda_version: 12.8.1
+            cuda_version: 12.6.3
             python_version: "3.11"
-            pytorch: 2.8.0
+            pytorch: 2.6.0
             axolotl_extras:
-          - cuda: 128
+          - cuda: 126
-            cuda_version: 12.8.1
+            cuda_version: 12.6.3
             python_version: "3.11"
-            pytorch: 2.9.1
+            pytorch: 2.7.1
             axolotl_extras:
     runs-on: axolotl-gpu-runner
     steps:
@@ -83,6 +84,7 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: |
+            winglian/axolotl-cloud
             axolotlai/axolotl-cloud
           tags: |
             type=raw,value={{ branch }}-{{ date 'YYYYMMDD' }}

.github/workflows/precommit-autoupdate.yml

@@ -2,7 +2,7 @@ name: Pre-commit auto-update
 
 on:
   schedule:
-    - cron: '0 0 1 * *'  # Run monthly
+    - cron: '0 0 * * 0'  # Run weekly
   workflow_dispatch:  # Manual kickoff
 
 jobs:

.github/workflows/preview-docs.yml

@@ -2,7 +2,7 @@ name: Preview
 on:
   workflow_dispatch:
   pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
+    types: [opened, synchronize, reopened]
 
     # Run the workflow only when one of these files changes
     paths:
@@ -11,7 +11,6 @@ on:
       - '_quarto.yml'
       - docs/scripts/generate_config_docs.py
       - src/axolotl/utils/schemas/**.py
-      - .github/workflows/preview-docs.yml
 
 permissions:
   checks: write
@@ -26,12 +25,7 @@ permissions:
 jobs:
   preview:
     runs-on: ubuntu-latest
-    if: ${{ !github.event.pull_request.draft }}
     steps:
-      - name: cleanup node
-        run: |
-          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
-
       - name: Check out repository
         uses: actions/checkout@v4
         with:
@@ -58,7 +52,6 @@ jobs:
 
       - name: Netlify Publish
         uses: nwtgck/actions-netlify@v3.0
-        if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
         id: netlify
         with:
           publish-dir: './_site'

.github/workflows/tests-nightly.yml

@@ -26,7 +26,7 @@ jobs:
       max-parallel: 2
       matrix:
         python_version: ["3.11"]
-        pytorch_version: ["2.8.0", "2.9.0", "2.9.1"]
+        pytorch_version: ["2.6.0", "2.7.0"]
     timeout-minutes: 20
 
     steps:
@@ -52,7 +52,7 @@ jobs:
 
       - name: Install PyTorch
         run: |
-          pip3 install torch==${{ matrix.pytorch_version }} torchvision
+          pip3 install torch==${{ matrix.pytorch_version }}
 
       - name: Update requirements.txt
         run: |
@@ -99,17 +99,17 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - cuda: 128
+          - cuda: 126
-            cuda_version: 12.8.1
+            cuda_version: 12.6.3
             python_version: "3.11"
-            pytorch: 2.8.0
+            pytorch: 2.6.0
             num_gpus: 1
             axolotl_extras:
             nightly_build: "true"
-          - cuda: 128
+          - cuda: 126
-            cuda_version: 12.8.1
+            cuda_version: 12.6.3
             python_version: "3.11"
-            pytorch: 2.9.1
+            pytorch: 2.7.1
             num_gpus: 1
             axolotl_extras:
             nightly_build: "true"
@@ -123,7 +123,7 @@ jobs:
       - name: Install Modal
         run: |
           python -m pip install --upgrade pip
-          pip install modal==1.3.0.post1 jinja2
+          pip install modal==1.0.2 jinja2
       - name: Update env vars
         run: |
           echo "BASE_TAG=main-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}" >> $GITHUB_ENV
@@ -148,10 +148,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - cuda: 128
+          - cuda: 126
-            cuda_version: 12.8.1
+            cuda_version: 12.6.3
             python_version: "3.11"
-            pytorch: 2.9.1
+            pytorch: 2.7.1
             num_gpus: 2
             axolotl_extras:
             nightly_build: "true"
@@ -165,7 +165,7 @@ jobs:
       - name: Install Modal
         run: |
           python -m pip install --upgrade pip
-          pip install modal==1.3.0.post1 jinja2
+          pip install modal==1.0.2 jinja2
       - name: Update env vars
         run: |
           echo "BASE_TAG=main-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}" >> $GITHUB_ENV

.github/workflows/tests.yml

@@ -13,7 +13,6 @@ on:
       - 'cicd/cicd.sh'
       - 'cicd/Dockerfile.jinja'
   pull_request:
-      types: [opened, synchronize, reopened, ready_for_review]
       paths:
        - '**.py'
        - 'requirements.txt'
@@ -35,7 +34,6 @@ jobs:
   pre-commit:
     name: pre-commit
     runs-on: ubuntu-latest
-    if: ${{ !github.event.pull_request.draft }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -49,29 +47,23 @@ jobs:
   pytest:
     name: PyTest
     runs-on: ubuntu-latest
-    if: ${{ !github.event.pull_request.draft }}
 #    needs: [preload-cache]
     strategy:
       fail-fast: false
       matrix:
         python_version: ["3.11"]
-        pytorch_version: ["2.8.0", "2.9.0", "2.9.1"]
+        pytorch_version: ["2.6.0", "2.7.0", "2.7.1"]
     timeout-minutes: 20
 
     steps:
-      - name: cleanup node
-        run: |
-          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
-
       - name: Check out repository code
         uses: actions/checkout@v4
 
       - name: Restore Cache from S3
         id: hf-cache-restore-s3
         run: |
-          mkdir -p ~/.cache/huggingface/hub
+          mkdir -p /home/runner/.cache/huggingface/hub
-          curl -L https://d1dttdx32dkk5p.cloudfront.net/hf-cache.tar.zst | tar -xpf - -C ~/.cache/huggingface/hub/  --use-compress-program unzstd --strip-components=1
+          curl -L https://d1dttdx32dkk5p.cloudfront.net/hf-cache.tar.zst | tar -xf - -C /home/runner/.cache/huggingface/hub/  --use-compress-program unzstd
-          ls -ltr ~/.cache/huggingface/hub/
 
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -86,20 +78,16 @@ jobs:
 
       - name: Install PyTorch
         run: |
-          pip3 install --no-cache-dir torch==${{ matrix.pytorch_version }} torchvision
+          pip3 install torch==${{ matrix.pytorch_version }}
 
       - name: Install dependencies
         run: |
           pip3 show torch
-          pip3 install --no-cache-dir --no-build-isolation -U -e .
+          pip3 install --no-build-isolation -U -e .
           python scripts/unsloth_install.py | sh
           python scripts/cutcrossentropy_install.py | sh
           pip3 install -r requirements-dev.txt -r requirements-tests.txt
 
-      - name: cleanup pip cache
-        run: |
-          find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \;
-
       - name: Make sure PyTorch version wasn't clobbered
         run: |
           python -c "import torch; assert '${{ matrix.pytorch_version }}' in torch.__version__"
@@ -112,23 +100,12 @@ jobs:
         run: |
           huggingface-cli download --repo-type=dataset axolotl-ai-internal/axolotl-oss-dataset-fixtures
 
-      - name: Show HF cache
-        run: hf cache scan
-
       - name: Run tests
         run: |
-          df -h
+          pytest -v --durations=10 -n8 --dist loadfile --ignore=tests/e2e/ --ignore=tests/patched/ --ignore=tests/cli/ tests/ --cov=axolotl --cov-report=xml
-          pytest -v --durations=10 -n4 --dist loadfile --ignore=tests/e2e/ --ignore=tests/patched/ --ignore=tests/cli/ --ignore=tests/monkeypatch/ tests/ --cov=axolotl --cov-report=xml
-          df -h
-          pytest -v --durations=10 tests/monkeypatch/ --cov=axolotl --cov-append --cov-report=xml
-          df -h
           pytest -v --durations=10 tests/patched/ --cov=axolotl --cov-append --cov-report=xml
-          df -h
           pytest -v --durations=10 tests/cli/ --cov=axolotl --cov-append --cov-report=xml
 
-      - name: Show HF cache
-        run: hf cache scan
-
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5
         with:
@@ -137,31 +114,29 @@ jobs:
           flags: unittests,pytorch-${{ matrix.pytorch_version }}
           fail_ci_if_error: false
 
+      - name: cleanup pip cache
+        run: |
+          find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \;
+
   pytest-sdist:
     name: PyTest from Source Dist
     runs-on: ubuntu-latest
-    if: ${{ !github.event.pull_request.draft }}
     strategy:
       fail-fast: false
       matrix:
         python_version: ["3.11"]
-        pytorch_version: ["2.8.0", "2.9.0", "2.9.1"]
+        pytorch_version: ["2.6.0", "2.7.0", "2.7.1"]
     timeout-minutes: 20
 
     steps:
-      - name: cleanup node
-        run: |
-          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
-
       - name: Check out repository code
         uses: actions/checkout@v4
 
       - name: Restore Cache from S3
         id: hf-cache-restore-s3
         run: |
-          mkdir -p ~/.cache/huggingface/hub
+          mkdir -p /home/runner/.cache/huggingface/hub
-          curl -L https://d1dttdx32dkk5p.cloudfront.net/hf-cache.tar.zst | tar -xpf - -C ~/.cache/huggingface/hub/  --use-compress-program unzstd --strip-components=1
+          curl -L https://d1dttdx32dkk5p.cloudfront.net/hf-cache.tar.zst | tar -xf - -C /home/runner/.cache/huggingface/hub/  --use-compress-program unzstd
-          ls -ltr ~/.cache/huggingface/hub/
 
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -172,25 +147,21 @@ jobs:
       - name: upgrade pip
         run: |
           pip3 install --upgrade pip
-          pip3 install --upgrade packaging==23.2 setuptools==75.8.0 setuptools_scm build wheel psutil
+          pip3 install --upgrade packaging==23.2 setuptools==75.8.0 setuptools_scm build wheel
 
       - name: Install PyTorch
         run: |
-          pip3 install --no-cache-dir torch==${{ matrix.pytorch_version }} torchvision
+          pip3 install torch==${{ matrix.pytorch_version }}
 
       - name: Install dependencies
         run: |
           pip3 show torch
           python -m build --no-isolation --sdist
-          pip3 install --no-cache-dir --no-build-isolation dist/axolotl*.tar.gz
+          pip3 install --no-build-isolation dist/axolotl*.tar.gz
           python scripts/unsloth_install.py | sh
           python scripts/cutcrossentropy_install.py | sh
           pip3 install -r requirements-dev.txt -r requirements-tests.txt
 
-      - name: cleanup pip cache
-        run: |
-          find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \;
-
       - name: Make sure PyTorch version wasn't clobbered
         run: |
           python -c "import torch; assert '${{ matrix.pytorch_version }}' in torch.__version__"
@@ -200,64 +171,40 @@ jobs:
           axolotl --help
 
       - name: Show HF cache
-        run: hf cache scan
+        run: huggingface-cli scan-cache
 
       - name: Run tests
         run: |
-          pytest -v --durations=10 -n4 --dist loadfile --ignore=tests/e2e/ --ignore=tests/patched/ --ignore=tests/cli/ --ignore=tests/monkeypatch/ tests/ --cov=axolotl --cov-report=xml
+          pytest -v --durations=10 -n8 --dist loadfile --ignore=tests/e2e/ --ignore=tests/patched/ --ignore=tests/cli/ tests/
-          pytest -v --durations=10 tests/monkeypatch/ --cov=axolotl --cov-append --cov-report=xml
+          pytest -v --durations=10 tests/patched/
           pytest -v --durations=10 tests/cli/
 
-      - name: Show HF cache
+      - name: cleanup pip cache
-        run: hf cache scan
+        run: |
-
+          find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \;
-  gate-skip-e2e:
-    needs: [pre-commit, pytest, pytest-sdist]
-    runs-on: ubuntu-latest
-    outputs:
-      skip: ${{ steps.compute.outputs.skip }}
-    steps:
-      - uses: actions/github-script@v7
-        id: compute
-        with:
-          script: |
-            const token = /\[skip-e2e\]/i;
-            let msg = '';
-            if (context.eventName === 'push') {
-              msg = context.payload.head_commit?.message || '';
-            } else if (context.eventName === 'pull_request') {
-              const { owner, repo } = context.repo;
-              const prNumber = context.payload.pull_request.number;
-              const commits = await github.paginate(
-                github.rest.pulls.listCommits,
-                { owner, repo, pull_number: prNumber, per_page: 100 }
-              );
-              msg = commits.at(-1)?.commit?.message || '';
-            }
-            const title = context.payload.pull_request?.title || '';
-            const body  = context.payload.pull_request?.body  || '';
-            const skip = token.test(msg) || token.test(title) || token.test(body);
-            core.setOutput('skip', String(skip));
 
   docker-e2e-tests-1st:
     # Run this job first as a gate for running the remainder of the test matrix
-    if: >
+    if: ${{ ! contains(github.event.commits[0].message, '[skip e2e]') && github.repository_owner == 'axolotl-ai-cloud' }}
-      github.repository_owner == 'axolotl-ai-cloud' &&
-      (github.event_name != 'pull_request' || !github.event.pull_request.draft) &&
-      needs.gate-skip-e2e.outputs.skip != 'true'
     # this job needs to be run on self-hosted GPU runners...
     runs-on: [self-hosted, modal]
     timeout-minutes: 120
-    needs: [pre-commit, pytest, pytest-sdist, gate-skip-e2e]
+    needs: [pre-commit, pytest, pytest-sdist]
 
     strategy:
       fail-fast: false
       matrix:
         include:
-          - cuda: 128
+          - cuda: 126
-            cuda_version: 12.8.1
+            cuda_version: 12.6.3
             python_version: "3.11"
-            pytorch: 2.8.0
+            pytorch: 2.7.1
+            num_gpus: 1
+            axolotl_extras:
+          - cuda: 126
+            cuda_version: 12.6.3
+            python_version: "3.11"
+            pytorch: 2.6.0
             num_gpus: 1
             axolotl_extras:
             dockerfile: "Dockerfile-uv.jinja"
@@ -271,7 +218,7 @@ jobs:
       - name: Install Modal
         run: |
           python -m pip install --upgrade pip
-          pip install modal==1.3.0.post1 jinja2
+          pip install modal==1.0.2 jinja2
       - name: Update env vars
         run: |
           echo "BASE_TAG=main-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}" >> $GITHUB_ENV
@@ -288,38 +235,28 @@ jobs:
           modal run cicd.e2e_tests
 
   docker-e2e-tests:
-    if: >
+    if: github.repository_owner == 'axolotl-ai-cloud'
-      github.repository_owner == 'axolotl-ai-cloud' &&
-      (github.event_name != 'pull_request' || !github.event.pull_request.draft) &&
-      needs.gate-skip-e2e.outputs.skip != 'true'
     # this job needs to be run on self-hosted GPU runners...
     runs-on: [self-hosted, modal]
     timeout-minutes: 120
     # Only run the remainder of the matrix if the first e2e check passed;
     # this is to save on wasted compute costs for known failures that get caught in the first run
-    needs: [pre-commit, pytest, gate-skip-e2e, docker-e2e-tests-1st]
+    needs: [pre-commit, pytest, docker-e2e-tests-1st]
 
     strategy:
       fail-fast: false
       matrix:
         include:
-          - cuda: 128
+          - cuda: 126
-            cuda_version: 12.8.1
+            cuda_version: 12.6.3
             python_version: "3.11"
-            pytorch: 2.8.0
+            pytorch: 2.6.0
-            num_gpus: 1
-            gpu_type: "B200"
-            axolotl_extras: fbgemm-gpu
-          - cuda: 128
-            cuda_version: 12.8.1
-            python_version: "3.11"
-            pytorch: 2.9.1
             num_gpus: 1
             axolotl_extras:
-          - cuda: 130
+          - cuda: 128
-            cuda_version: 13.0.0
+            cuda_version: 12.8.1
             python_version: "3.11"
-            pytorch: 2.9.1
+            pytorch: 2.7.1
             num_gpus: 1
             axolotl_extras:
     steps:
@@ -332,7 +269,7 @@ jobs:
       - name: Install Modal
         run: |
           python -m pip install --upgrade pip
-          pip install modal==1.3.0.post1 jinja2
+          pip install modal==1.0.2 jinja2
       - name: Update env vars
         run: |
           echo "BASE_TAG=main-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}" >> $GITHUB_ENV
@@ -342,7 +279,6 @@ jobs:
           echo "CUDA=${{ matrix.cuda }}" >> $GITHUB_ENV
           echo "MODAL_IMAGE_BUILDER_VERSION=2024.10" >> $GITHUB_ENV
           echo "N_GPUS=${{ matrix.num_gpus }}" >> $GITHUB_ENV
-          echo "GPU_TYPE=${{ matrix.gpu_type || 'L40S'}}" >> $GITHUB_ENV
           echo "CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }}" >> $GITHUB_ENV
           echo "E2E_DOCKERFILE=${{ matrix.dockerfile || 'Dockerfile.jinja'}}" >> $GITHUB_ENV
       - name: Run tests job on Modal
@@ -353,16 +289,15 @@ jobs:
     runs-on: [self-hosted, modal]
     timeout-minutes: 90
     needs: [docker-e2e-tests]
-    if: ${{ !github.event.pull_request.draft }}
 
     strategy:
       fail-fast: false
       matrix:
         include:
-          - cuda: 128
+          - cuda: 124
-            cuda_version: 12.8.1
+            cuda_version: 12.4.1
             python_version: "3.11"
-            pytorch: 2.9.1
+            pytorch: 2.6.0
             num_gpus: 1
             axolotl_extras:
     steps:
@@ -375,7 +310,7 @@ jobs:
       - name: Install Modal
         run: |
           python -m pip install --upgrade pip
-          pip install modal==1.3.0.post1 jinja2
+          pip install modal==1.0.2 jinja2
       - name: Update env vars
         run: |
           echo "BASE_TAG=main-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}" >> $GITHUB_ENV

.gitignore

@@ -190,6 +190,3 @@ out/
 
 # vim
 *.swp
-
-# scm auto-versioning
-src/axolotl/_version.py

.isort.cfg

@@ -0,0 +1,4 @@
+[settings]
+profile=black
+known_third_party=wandb,comet_ml
+known_local_folder=src,tests

.pre-commit-config.yaml

@@ -3,21 +3,31 @@ default_language_version:
 
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v6.0.0
+    rev: v5.0.0
     hooks:
     -   id: check-yaml
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
     -   id: no-commit-to-branch
         args: ['--branch', 'main']
--   repo: https://github.com/astral-sh/ruff-pre-commit
+-   repo: https://github.com/psf/black
-    rev: v0.14.10
+    rev: 25.1.0
     hooks:
-    -   id: ruff
+    -   id: black
-        args: [--fix]
+-   repo: https://github.com/pycqa/isort
-    -   id: ruff-format
+    rev: 6.0.1
+    hooks:
+      - id: isort
+-   repo: https://github.com/PyCQA/flake8
+    rev: 7.3.0
+    hooks:
+    - id: flake8
+-   repo: https://github.com/pylint-dev/pylint
+    rev: v3.3.7
+    hooks:
+    - id: pylint
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.19.1
+    rev: v1.16.1
     hooks:
     - id: mypy
       additional_dependencies:
@@ -26,7 +36,7 @@ repos:
             'pydantic>=2.5.3',
         ]
 -   repo: https://github.com/PyCQA/bandit
-    rev: 1.9.2
+    rev: 1.8.6
     hooks:
     -   id: bandit
         args: [

.pylintrc

@@ -0,0 +1,15 @@
+[MASTER]
+init-hook="from pylint.config import find_default_config_files; import sys; sys.path.append(next(find_default_config_files()).parent.as_posix())"
+
+[TYPECHECK]
+
+# List of members which are set dynamically and missed by Pylint inference
+# system, and so shouldn't trigger E1101 when accessed.
+generated-members=numpy.*, torch.*
+
+
+[pylint.messages_control]
+disable=missing-function-docstring, line-too-long, import-error,
+    too-many-arguments, too-many-locals, too-many-statements, too-many-branches, too-few-public-methods,
+    too-many-instance-attributes, fixme, import-outside-toplevel, logging-fstring-interpolation,
+    too-many-positional-arguments, possibly-used-before-assignment

.runpod/Dockerfile

@@ -10,7 +10,6 @@ ARG BASE_VOLUME="/runpod-volume"
 ENV BASE_VOLUME=$BASE_VOLUME
 ENV HF_DATASETS_CACHE="${BASE_VOLUME}/huggingface-cache/datasets"
 ENV HUGGINGFACE_HUB_CACHE="${BASE_VOLUME}/huggingface-cache/hub"
-ENV HF_HUB_CACHE="${BASE_VOLUME}/huggingface-cache/hub"
 ENV TRANSFORMERS_CACHE="${BASE_VOLUME}/huggingface-cache/hub"
 
 COPY .runpod/src /src

.runpod/README.md

@@ -119,15 +119,14 @@ datasets:
 
 ## Dataset Processing
 
-| Option                            | Default                    | Description                         |
+| Option                        | Default                    | Description                       |
-| --------------------------------- | -------------------------- | ----------------------------------- |
+| ----------------------------- | -------------------------- | --------------------------------- |
-| `dataset_prepared_path`           | `"data/last_run_prepared"` | Path for prepared dataset           |
+| `dataset_prepared_path`       | `"data/last_run_prepared"` | Path for prepared dataset         |
-| `push_dataset_to_hub`             | `""`                       | Push dataset to HF hub              |
+| `push_dataset_to_hub`         | `""`                       | Push dataset to HF hub            |
-| `dataset_processes`               | `4`                        | Number of preprocessing processes   |
+| `dataset_processes`           | `4`                        | Number of preprocessing processes |
-| `dataset_keep_in_memory`          | `false`                    | Keep dataset in memory              |
+| `dataset_keep_in_memory`      | `false`                    | Keep dataset in memory            |
-| `shuffle_merged_datasets`         | `true`                     | Shuffle merged datasets             |
+| `shuffle_merged_datasets`     | `true`                     | Shuffle merged datasets           |
-| `shuffle_before_merging_datasets` | `false`                    | Shuffle each dataset before merging |
+| `dataset_exact_deduplication` | `true`                     | Deduplicate datasets              |
-| `dataset_exact_deduplication`     | `true`                     | Deduplicate datasets                |
 
 ## LoRA Configuration
 
@@ -185,6 +184,7 @@ datasets:
 | `flash_attention`          | `false` | Use flash attention           |
 | `flash_attn_cross_entropy` | `false` | Flash attention cross entropy |
 | `flash_attn_rms_norm`      | `false` | Flash attention RMS norm      |
+| `flash_attn_fuse_qkv`      | `false` | Fuse QKV operations           |
 | `flash_attn_fuse_mlp`      | `false` | Fuse MLP operations           |
 | `sdp_attention`            | `false` | Use scaled dot product        |
 | `s2_attention`             | `false` | Use shifted sparse attention  |

.runpod/src/config/config.yaml

@@ -296,6 +296,7 @@
 # flash_attention:
 # flash_attn_cross_entropy:  # Whether to use flash-attention cross entropy implementation - advanced use only
 # flash_attn_rms_norm:  # Whether to use flash-attention rms norm implementation - advanced use only
+# flash_attn_fuse_qkv: # Whether to fuse QKV into a single operation
 # flash_attn_fuse_mlp: # Whether to fuse part of the MLP into a single operation
 # # Whether to use scaled-dot-product attention
 # # https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html
@@ -540,6 +541,7 @@ xformers_attention: ${XFORMERS_ATTENTION}
 flash_attention: ${FLASH_ATTENTION}
 flash_attn_cross_entropy: ${FLASH_ATTN_CROSS_ENTROPY}
 flash_attn_rms_norm: ${FLASH_ATTN_RMS_NORM}
+flash_attn_fuse_qkv: ${FLASH_ATTN_FUSE_QKV}
 flash_attn_fuse_mlp: ${FLASH_ATTN_FUSE_MLP}
 sdp_attention: ${SDP_ATTENTION}
 s2_attention: ${S2_ATTENTION}

CITATION.cff

@@ -1,10 +0,0 @@
-cff-version: 1.2.0
-type: software
-title: "Axolotl: Open Source LLM Post-Training"
-message: "If you use this software, please cite it as below."
-authors:
-  - name: "Axolotl maintainers and contributors"
-repository-code: "https://github.com/axolotl-ai-cloud/axolotl"
-url: "https://axolotl.ai/"
-license: Apache-2.0
-date-released: "2023-05-30"

README.md

@@ -5,9 +5,6 @@
         <img alt="Axolotl" src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/887513285d98132142bf5db2a74eb5e0928787f1/image/axolotl_logo_digital_black.svg" width="400" height="104" style="max-width: 100%;">
     </picture>
 </p>
-  <p align="center">
-      <strong>A Free and Open Source LLM Fine-tuning Framework</strong><br>
-  </p>
 
 <p align="center">
     <img src="https://img.shields.io/github/license/axolotl-ai-cloud/axolotl.svg?color=blue" alt="GitHub License">
@@ -20,7 +17,6 @@
     <br/>
     <a href="https://discord.com/invite/HhrNrHJPRb"><img src="https://img.shields.io/badge/discord-7289da.svg?style=flat-square&logo=discord" alt="discord" style="height: 20px;"></a>
     <a href="https://twitter.com/axolotl_ai"><img src="https://img.shields.io/twitter/follow/axolotl_ai?style=social" alt="twitter" style="height: 20px;"></a>
-    <a href="https://colab.research.google.com/github/axolotl-ai-cloud/axolotl/blob/main/examples/colab-notebooks/colab-axolotl-example.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="google-colab" style="height: 20px;"></a>
     <br/>
     <img src="https://github.com/axolotl-ai-cloud/axolotl/actions/workflows/tests-nightly.yml/badge.svg" alt="tests-nightly">
     <img src="https://github.com/axolotl-ai-cloud/axolotl/actions/workflows/multi-gpu-e2e.yml/badge.svg" alt="multigpu-semi-weekly tests">
@@ -29,59 +25,37 @@
 
 ## 🎉 Latest Updates
 
-- 2025/12: Axolotl now includes support for [Kimi-Linear](https://docs.axolotl.ai/docs/models/kimi-linear.html), [Plano-Orchestrator](https://docs.axolotl.ai/docs/models/plano.html), [MiMo](https://docs.axolotl.ai/docs/models/mimo.html), [InternVL 3.5](https://docs.axolotl.ai/docs/models/internvl3_5.html), [Olmo3](https://docs.axolotl.ai/docs/models/olmo3.html), [Trinity](https://docs.axolotl.ai/docs/models/trinity.html), and [Ministral3](https://docs.axolotl.ai/docs/models/ministral3.html).
+- 2025/06: Magistral with mistral-common tokenizer support has been added to Axolotl. See [examples](https://github.com/axolotl-ai-cloud/axolotl/tree/main/examples/magistral) to start training your own Magistral models with Axolotl!
-- 2025/10: New model support has been added in Axolotl for: [Qwen3 Next](https://docs.axolotl.ai/docs/models/qwen3-next.html), [Qwen2.5-vl, Qwen3-vl](https://github.com/axolotl-ai-cloud/axolotl/tree/main/examples/qwen2_5-vl), [Qwen3, Qwen3MoE](https://docs.axolotl.ai/docs/models/qwen3.html), [Granite 4](https://docs.axolotl.ai/docs/models/granite4.html), [HunYuan](https://docs.axolotl.ai/docs/models/hunyuan.html), [Magistral 2509](https://docs.axolotl.ai/docs/models/magistral/vision.html), [Apertus](https://docs.axolotl.ai/docs/models/apertus.html), and [Seed-OSS](https://docs.axolotl.ai/docs/models/seed-oss.html).
-- 2025/09: Axolotl now has text diffusion training. Read more [here](https://github.com/axolotl-ai-cloud/axolotl/tree/main/src/axolotl/integrations/diffusion).
-- 2025/08: QAT has been updated to include NVFP4 support. See [PR](https://github.com/axolotl-ai-cloud/axolotl/pull/3107).
-- 2025/07:
-  - ND Parallelism support has been added into Axolotl. Compose Context Parallelism (CP), Tensor Parallelism (TP), and Fully Sharded Data Parallelism (FSDP) within a single node and across multiple nodes. Check out the [blog post](https://huggingface.co/blog/accelerate-nd-parallel) for more info.
-  - Axolotl adds more models: [GPT-OSS](https://docs.axolotl.ai/docs/models/gpt-oss.html), [Gemma 3n](https://docs.axolotl.ai/docs/models/gemma3n.html), [Liquid Foundation Model 2 (LFM2)](https://docs.axolotl.ai/docs/models/LiquidAI.html), and [Arcee Foundation Models (AFM)](https://docs.axolotl.ai/docs/models/arcee.html).
-  - FP8 finetuning with fp8 gather op is now possible in Axolotl via `torchao`. Get started [here](https://docs.axolotl.ai/docs/mixed_precision.html#sec-fp8)!
-  - [Voxtral](https://docs.axolotl.ai/docs/models/voxtral.html), [Magistral 1.1](https://docs.axolotl.ai/docs/models/magistral.html), and [Devstral](https://docs.axolotl.ai/docs/models/devstral.html) with mistral-common tokenizer support has been integrated in Axolotl!
-  - TiledMLP support for single-GPU to multi-GPU training with DDP, DeepSpeed and FSDP support has been added to support Arctic Long Sequence Training. (ALST). See [examples](https://github.com/axolotl-ai-cloud/axolotl/tree/main/examples/alst) for using ALST with Axolotl!
 - 2025/05: Quantization Aware Training (QAT) support has been added to Axolotl. Explore the [docs](https://docs.axolotl.ai/docs/qat.html) to learn more!
-
+- 2025/04: Llama 4 support has been added in Axolotl. See [examples](https://github.com/axolotl-ai-cloud/axolotl/tree/main/examples/llama-4) to start training your own Llama 4 models with Axolotl's linearized version!
-<details>
-
-<summary>Expand older updates</summary>
-
 - 2025/03: Axolotl has implemented Sequence Parallelism (SP) support. Read the [blog](https://huggingface.co/blog/axolotl-ai-co/long-context-with-sequence-parallelism-in-axolotl) and [docs](https://docs.axolotl.ai/docs/sequence_parallelism.html) to learn how to scale your context length when fine-tuning.
-- 2025/06: Magistral with mistral-common tokenizer support has been added to Axolotl. See [docs](https://docs.axolotl.ai/docs/models/magistral.html) to start training your own Magistral models with Axolotl!
-- 2025/04: Llama 4 support has been added in Axolotl. See [docs](https://docs.axolotl.ai/docs/models/llama-4.html) to start training your own Llama 4 models with Axolotl's linearized version!
 - 2025/03: (Beta) Fine-tuning Multimodal models is now supported in Axolotl. Check out the [docs](https://docs.axolotl.ai/docs/multimodal.html) to fine-tune your own!
 - 2025/02: Axolotl has added LoRA optimizations to reduce memory usage and improve training speed for LoRA and QLoRA in single GPU and multi-GPU training (DDP and DeepSpeed). Jump into the [docs](https://docs.axolotl.ai/docs/lora_optims.html) to give it a try.
 - 2025/02: Axolotl has added GRPO support. Dive into our [blog](https://huggingface.co/blog/axolotl-ai-co/training-llms-w-interpreter-feedback-wasm) and [GRPO example](https://github.com/axolotl-ai-cloud/grpo_code) and have some fun!
 - 2025/01: Axolotl has added Reward Modelling / Process Reward Modelling fine-tuning support. See [docs](https://docs.axolotl.ai/docs/reward_modelling.html).
 
-</details>
-
 ## ✨ Overview
 
-Axolotl is a free and open-source tool designed to streamline post-training and fine-tuning for the latest large language models (LLMs).
+Axolotl is a tool designed to streamline post-training for various AI models.
 
 Features:
 
-- **Multiple Model Support**: Train various models like GPT-OSS, LLaMA, Mistral, Mixtral, Pythia, and many more models available on the Hugging Face Hub.
+- **Multiple Model Support**: Train various models like LLaMA, Mistral, Mixtral, Pythia, and more. We are compatible with HuggingFace transformers causal language models.
-- **Multimodal Training**: Fine-tune vision-language models (VLMs) including LLaMA-Vision, Qwen2-VL, Pixtral, LLaVA, SmolVLM2, and audio models like Voxtral with image, video, and audio support.
+- **Training Methods**: Full fine-tuning, LoRA, QLoRA, GPTQ, QAT, Preference Tuning (DPO, IPO, KTO, ORPO), RL (GRPO), Multimodal, and Reward Modelling (RM) / Process Reward Modelling (PRM).
-- **Training Methods**: Full fine-tuning, LoRA, QLoRA, GPTQ, QAT, Preference Tuning (DPO, IPO, KTO, ORPO), RL (GRPO), and Reward Modelling (RM) / Process Reward Modelling (PRM).
+- **Easy Configuration**: Re-use a single YAML file between dataset preprocess, training, evaluation, quantization, and inference.
-- **Easy Configuration**: Re-use a single YAML configuration file across the full fine-tuning pipeline: dataset preprocessing, training, evaluation, quantization, and inference.
 - **Performance Optimizations**: [Multipacking](https://docs.axolotl.ai/docs/multipack.html), [Flash Attention](https://github.com/Dao-AILab/flash-attention), [Xformers](https://github.com/facebookresearch/xformers), [Flex Attention](https://pytorch.org/blog/flexattention/), [Liger Kernel](https://github.com/linkedin/Liger-Kernel), [Cut Cross Entropy](https://github.com/apple/ml-cross-entropy/tree/main), [Sequence Parallelism (SP)](https://docs.axolotl.ai/docs/sequence_parallelism.html), [LoRA optimizations](https://docs.axolotl.ai/docs/lora_optims.html), [Multi-GPU training (FSDP1, FSDP2, DeepSpeed)](https://docs.axolotl.ai/docs/multi-gpu.html), [Multi-node training (Torchrun, Ray)](https://docs.axolotl.ai/docs/multi-node.html), and many more!
 - **Flexible Dataset Handling**: Load from local, HuggingFace, and cloud (S3, Azure, GCP, OCI) datasets.
 - **Cloud Ready**: We ship [Docker images](https://hub.docker.com/u/axolotlai) and also [PyPI packages](https://pypi.org/project/axolotl/) for use on cloud platforms and local hardware.
 
 
 
-## 🚀 Quick Start - LLM Fine-tuning in Minutes
+## 🚀 Quick Start
 
 **Requirements**:
 
 - NVIDIA GPU (Ampere or newer for `bf16` and Flash Attention) or AMD GPU
 - Python 3.11
-- PyTorch ≥2.8.0
+- PyTorch ≥2.6.0
-
-### Google Colab
-
-[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/axolotl-ai-cloud/axolotl/blob/main/examples/colab-notebooks/colab-axolotl-example.ipynb#scrollTo=msOCO4NRmRLa)
 
 ### Installation
 
@@ -105,20 +79,6 @@ docker run --gpus '"all"' --rm -it axolotlai/axolotl:main-latest
 
 Other installation approaches are described [here](https://docs.axolotl.ai/docs/installation.html).
 
-#### Cloud Providers
-
-<details>
-
-- [RunPod](https://runpod.io/gsc?template=v2ickqhz9s&ref=6i7fkpdz)
-- [Vast.ai](https://cloud.vast.ai?ref_id=62897&template_id=bdd4a49fa8bce926defc99471864cace&utm_source=github&utm_medium=developer_community&utm_campaign=template_launch_axolotl&utm_content=readme)
-- [PRIME Intellect](https://app.primeintellect.ai/dashboard/create-cluster?image=axolotl&location=Cheapest&security=Cheapest&show_spot=true)
-- [Modal](https://www.modal.com?utm_source=github&utm_medium=github&utm_campaign=axolotl)
-- [Novita](https://novita.ai/gpus-console?templateId=311)
-- [JarvisLabs.ai](https://jarvislabs.ai/templates/axolotl)
-- [Latitude.sh](https://latitude.sh/blueprint/989e0e79-3bf6-41ea-a46b-1f246e309d5c)
-
-</details>
-
 ### Your First Fine-tune
 
 ```bash
@@ -158,31 +118,16 @@ That's it! Check out our [Getting Started Guide](https://docs.axolotl.ai/docs/ge
 
 Contributions are welcome! Please see our [Contributing Guide](https://github.com/axolotl-ai-cloud/axolotl/blob/main/.github/CONTRIBUTING.md) for details.
 
-## 📈 Telemetry
-
-Axolotl has opt-out telemetry that helps us understand how the project is being used
-and prioritize improvements. We collect basic system information, model types, and
-error rates—never personal data or file paths. Telemetry is enabled by default. To
-disable it, set AXOLOTL_DO_NOT_TRACK=1. For more details, see our [telemetry documentation](https://docs.axolotl.ai/docs/telemetry.html).
-
 ## ❤️ Sponsors
 
+Thank you to our sponsors who help make Axolotl possible:
+
+- [Modal](https://www.modal.com?utm_source=github&utm_medium=github&utm_campaign=axolotl) - Modal lets you run
+jobs in the cloud, by just writing a few lines of Python. Customers use Modal to deploy Gen AI models at large scale,
+fine-tune large language models, run protein folding simulations, and much more.
+
 Interested in sponsoring? Contact us at [wing@axolotl.ai](mailto:wing@axolotl.ai)
 
-## 📝 Citing Axolotl
-
-If you use Axolotl in your research or projects, please cite it as follows:
-
-```bibtex
-@software{axolotl,
-  title = {Axolotl: Open Source LLM Post-Training},
-  author = {{Axolotl maintainers and contributors}},
-  url = {https://github.com/axolotl-ai-cloud/axolotl},
-  license = {Apache-2.0},
-  year = {2023}
-}
-```
-
 ## 📜 License
 
 This project is licensed under the Apache 2.0 License - see the [LICENSE](LICENSE) file for details.

TODO.md

@@ -0,0 +1,10 @@
+# todo list
+
+- [] Validation of parameters for combinations that won't work
+
+
+
+## things that are known not to work
+
+- FSDP offload and gradient_checkpointing - https://github.com/pytorch/pytorch/issues/82203
+- adamw_bnb_8bit doesn't play well with FSDP offload

_quarto.yml

@@ -1,8 +1,6 @@
 project:
   type: website
-  pre-render:
+  pre-render: docs/scripts/generate_config_docs.py
-   - docs/scripts/generate_config_docs.py
-   - docs/scripts/generate_examples_docs.py
 
 quartodoc:
   dir: docs/api
@@ -37,30 +35,25 @@ quartodoc:
         - cli.train
         - cli.evaluate
         - cli.args
-        - cli.art
         - cli.checks
         - cli.config
-        - cli.delinearize_llama4
         - cli.inference
         - cli.merge_lora
         - cli.merge_sharded_fsdp_weights
         - cli.preprocess
-        - cli.quantize
+        - cli.sweeps
+        - cli.utils
         - cli.vllm_serve
         - cli.cloud.base
         - cli.cloud.modal_
-        - cli.utils
+        - cli.quantize
-        - cli.utils.args
-        - cli.utils.fetch
-        - cli.utils.load
-        - cli.utils.sweeps
-        - cli.utils.train
     - title: Trainers
       desc: Training implementations
       contents:
         - core.trainers.base
         - core.trainers.trl
         - core.trainers.mamba
+        - core.trainers.relora
         - core.trainers.dpo.trainer
         - core.trainers.grpo.trainer
         - core.trainers.grpo.sampler
@@ -155,7 +148,7 @@ quartodoc:
         - utils.distributed
         - utils.dict
         - utils.optimizers.adopt
-        - utils.data.streaming
+        - utils.data.pretraining
         - utils.data.sft
         - utils.quantization
     - title: Schemas
@@ -242,48 +235,7 @@ website:
             - docs/getting-started.qmd
             - docs/installation.qmd
             - docs/inference.qmd
-            - section: "Model Guides"
-              contents:
-                - docs/models/kimi-linear.qmd
-                - docs/models/plano.qmd
-                - docs/models/mimo.qmd
-                - docs/models/internvl3_5.qmd
-                - docs/models/olmo3.qmd
-                - docs/models/trinity.qmd
-                - docs/models/arcee.qmd
-                - docs/models/mistral.qmd
-                - section: "Ministral3"
-                  contents:
-                    - docs/models/ministral3.qmd
-                    - docs/models/ministral3/think.qmd
-                    - docs/models/ministral3/vision.qmd
-                - section: "Magistral"
-                  contents:
-                    - docs/models/magistral.qmd
-                    - docs/models/magistral/think.qmd
-                    - docs/models/magistral/vision.qmd
-                - docs/models/ministral.qmd
-                - docs/models/mistral-small.qmd
-                - docs/models/voxtral.qmd
-                - docs/models/devstral.qmd
-                - docs/models/llama-4.qmd
-                - docs/models/llama-2.qmd
-                - docs/models/qwen3-next.qmd
-                - docs/models/qwen3.qmd
-                - docs/models/gemma3n.qmd
-                - docs/models/apertus.qmd
-                - docs/models/gpt-oss.qmd
-                - docs/models/seed-oss.qmd
-                - docs/models/phi.qmd
-                - docs/models/smolvlm2.qmd
-                - docs/models/granite4.qmd
-                - docs/models/LiquidAI.qmd
-                - docs/models/hunyuan.qmd
-                - docs/models/jamba.qmd
-                - docs/models/orpheus.qmd
-
             - docs/cli.qmd
-            - docs/telemetry.qmd
             - docs/config-reference.qmd
             - text: "API Reference"
               href: docs/api
@@ -310,16 +262,12 @@ website:
             - docs/dataset_loading.qmd
             - docs/qat.qmd
             - docs/quantize.qmd
-            - docs/optimizations.qmd
 
         - section: "Core Concepts"
           contents:
             - docs/batch_vs_grad.qmd
             - docs/dataset_preprocessing.qmd
-            - docs/streaming.qmd
             - docs/multipack.qmd
-            - docs/mixed_precision.qmd
-            - docs/optimizers.qmd
 
         - section: "Advanced Features"
           contents:
@@ -329,7 +277,6 @@ website:
             - docs/custom_integrations.qmd
             - docs/sequence_parallelism.qmd
             - docs/gradient_checkpointing.qmd
-            - docs/nd_parallelism.qmd
 
         - section: "Troubleshooting"
           contents:

cicd/Dockerfile-uv.jinja

@@ -11,7 +11,7 @@ ENV NIGHTLY_BUILD="{{ NIGHTLY_BUILD }}"
 ENV HF_HOME="{{ HF_HOME }}"
 
 RUN apt-get update && \
-    apt-get install -y --allow-change-held-packages vim curl nano libnccl2 libnccl-dev ibverbs-providers ibverbs-utils infiniband-diags librdmacm-dev librdmacm1 rdmacm-utils slurm-wlm
+    apt-get install -y --allow-change-held-packages vim curl nano libnccl2 libnccl-dev
 
 WORKDIR /workspace
 
@@ -32,7 +32,6 @@ RUN if [ "$NIGHTLY_BUILD" = "true" ] ; then \
     fi
 
 RUN uv pip install packaging==23.2 setuptools==75.8.0
-RUN uv pip install torchvision
 RUN if [ "$AXOLOTL_EXTRAS" != "" ] ; then \
         uv pip install --no-build-isolation -e .[deepspeed,flash-attn,ring-flash-attn,optimizers,ray,$AXOLOTL_EXTRAS] $AXOLOTL_ARGS; \
     else \

cicd/Dockerfile.jinja

@@ -1,6 +1,6 @@
 FROM axolotlai/axolotl-base:{{ BASE_TAG }}
 
-ENV TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 8.7 8.9 9.0+PTX"
+ENV TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6+PTX"
 ENV AXOLOTL_EXTRAS="{{ AXOLOTL_EXTRAS }}"
 ENV AXOLOTL_ARGS="{{ AXOLOTL_ARGS }}"
 ENV CUDA="{{ CUDA }}"
@@ -9,10 +9,10 @@ ENV GITHUB_REF="{{ GITHUB_REF }}"
 ENV GITHUB_SHA="{{ GITHUB_SHA }}"
 ENV NIGHTLY_BUILD="{{ NIGHTLY_BUILD }}"
 ENV HF_HOME="{{ HF_HOME }}"
-ENV AXOLOTL_DATASET_NUM_PROC="8"
+ENV AXOLOTL_DATASET_PROCESSES="8"
 
 RUN apt-get update && \
-    apt-get install -y --allow-change-held-packages vim curl nano libnccl2 libnccl-dev ibverbs-providers ibverbs-utils infiniband-diags librdmacm-dev librdmacm1 rdmacm-utils slurm-wlm
+    apt-get install -y --allow-change-held-packages vim curl nano libnccl2 libnccl-dev
 
 WORKDIR /workspace
 
@@ -32,7 +32,7 @@ RUN if [ "$NIGHTLY_BUILD" = "true" ] ; then \
         sed -i 's#^datasets.*#datasets @ git+https://github.com/huggingface/datasets.git@main#' requirements.txt; \
     fi
 
-RUN pip install packaging==23.2 setuptools==75.8.0 psutil
+RUN pip install packaging==23.2 setuptools==75.8.0
 RUN if [ "$AXOLOTL_EXTRAS" != "" ] ; then \
         pip install --no-build-isolation -e .[deepspeed,flash-attn,ring-flash-attn,optimizers,ray,$AXOLOTL_EXTRAS] $AXOLOTL_ARGS; \
     else \

cicd/multigpu.py

@@ -2,6 +2,8 @@
 modal application to run axolotl gpu tests in Modal
 """
 
+# pylint: disable=duplicate-code
+
 import os
 import pathlib
 import tempfile
@@ -61,7 +63,7 @@ def run_cmd(cmd: str, run_folder: str):
 
     # Propagate errors from subprocess.
     if exit_code := subprocess.call(cmd.split(), cwd=run_folder):  # nosec
-        exit(exit_code)
+        exit(exit_code)  # pylint: disable=consider-using-sys-exit
 
 
 @app.function(

cicd/multigpu.sh

@@ -2,7 +2,7 @@
 set -e
 
 # Only run two tests at a time to avoid OOM on GPU (with coverage collection)
-pytest -v --durations=10 -n2 --maxfail=4 \
+pytest -v -n2 \
   --ignore=/workspace/axolotl/tests/e2e/multigpu/solo/ \
   --ignore=/workspace/axolotl/tests/e2e/multigpu/patched/ \
   /workspace/axolotl/tests/e2e/multigpu/ \
@@ -19,7 +19,5 @@ pytest -v  --durations=10 -n1 /workspace/axolotl/tests/e2e/multigpu/patched/ \
   --cov-append \
   --cov-report=xml:multigpu-coverage.xml
 
-# Upload coverage to Codecov if CODECOV_TOKEN is available
+# Upload coverage to Codecov
-if [ -n "$CODECOV_TOKEN" ]; then
+codecov upload-process -t "${CODECOV_TOKEN}" -f multigpu-coverage.xml -F multigpu,docker-tests,pytorch-${PYTORCH_VERSION} || true
-  codecov upload-process -t "${CODECOV_TOKEN}" -f multigpu-coverage.xml -F multigpu,docker-tests,pytorch-${PYTORCH_VERSION} || true
-fi

cicd/single_gpu.py

@@ -1,5 +1,7 @@
 """Modal app to run axolotl GPU tests"""
 
+# pylint: disable=duplicate-code
+
 import os
 import pathlib
 import tempfile
@@ -57,21 +59,12 @@ VOLUME_CONFIG = {
 }
 
 N_GPUS = int(os.environ.get("N_GPUS", 1))
-GPU_TYPE = os.environ.get("GPU_TYPE", "L40S")
+GPU_CONFIG = f"L40S:{N_GPUS}"
-GPU_CONFIG = f"{GPU_TYPE}:{N_GPUS}"
 
 
 def run_cmd(cmd: str, run_folder: str):
     import subprocess  # nosec
 
-    sp_env = os.environ.copy()
-    sp_env["AXOLOTL_DATASET_NUM_PROC"] = "8"
-
     # Propagate errors from subprocess.
-    try:
+    if exit_code := subprocess.call(cmd.split(), cwd=run_folder):  # nosec
-        exit_code = subprocess.call(cmd.split(), cwd=run_folder, env=sp_env)  # nosec
+        exit(exit_code)  # pylint: disable=consider-using-sys-exit
-        if exit_code:
-            print(f"Command '{cmd}' failed with exit code {exit_code}")
-            return exit_code
-    except Exception as e:  # pylint: disable=broad-except
-        print(f"Command '{cmd}' failed with exception {e}")

codecov.yml

@@ -12,7 +12,7 @@ coverage:
       default:
         # basic
         target: auto
-        threshold: 1%
+        threshold: 0%
         base: auto
         # advanced
         branches: null
@@ -27,7 +27,7 @@ coverage:
       default:
         # basic
         target: auto
-        threshold: 1%
+        threshold: 0%
         base: auto
         # advanced
         branches: null

deepspeed_configs/zero3.json

@@ -7,9 +7,9 @@
     "reduce_bucket_size": "auto",
     "stage3_prefetch_bucket_size": "auto",
     "stage3_param_persistence_threshold": "auto",
-    "max_live_parameters": 0,
+    "stage3_max_live_parameters": 0,
-    "max_reuse_distance": 0,
+    "stage3_max_reuse_distance": 0,
-    "gather_16bit_weights_on_model_save": true
+    "stage3_gather_16bit_weights_on_model_save": true
   },
   "bf16": {
     "enabled": "auto"

deepspeed_configs/zero3_bf16.json

@@ -7,9 +7,9 @@
     "reduce_bucket_size": "auto",
     "stage3_prefetch_bucket_size": "auto",
     "stage3_param_persistence_threshold": "auto",
-    "max_live_parameters": 0,
+    "stage3_max_live_parameters": 0,
-    "max_reuse_distance": 0,
+    "stage3_max_reuse_distance": 0,
-    "gather_16bit_weights_on_model_save": true
+    "stage3_gather_16bit_weights_on_model_save": true
   },
   "bf16": {
     "enabled": true

deepspeed_configs/zero3_bf16_cpuoffload_all.json

@@ -17,9 +17,9 @@
     "reduce_bucket_size": "auto",
     "stage3_prefetch_bucket_size": "auto",
     "stage3_param_persistence_threshold": "auto",
-    "max_live_parameters": 0,
+    "stage3_max_live_parameters": 0,
-    "max_reuse_distance": 0,
+    "stage3_max_reuse_distance": 0,
-    "gather_16bit_weights_on_model_save": true
+    "stage3_gather_16bit_weights_on_model_save": true
   },
   "bf16": {
     "enabled": true

deepspeed_configs/zero3_bf16_cpuoffload_params.json

@@ -13,9 +13,9 @@
     "reduce_bucket_size": "auto",
     "stage3_prefetch_bucket_size": "auto",
     "stage3_param_persistence_threshold": "auto",
-    "max_live_parameters": 0,
+    "stage3_max_live_parameters": 0,
-    "max_reuse_distance": 0,
+    "stage3_max_reuse_distance": 0,
-    "gather_16bit_weights_on_model_save": true
+    "stage3_gather_16bit_weights_on_model_save": true
   },
   "bf16": {
     "enabled": true

devtools/dev_chat_template.yml

@@ -13,7 +13,7 @@ datasets:
 val_set_size: 0
 output_dir: temp_debug/axolotl_outputs/model
 dataset_prepared_path: temp_debug/axolotl_outputs/data
-dataset_num_proc: 1
+dataset_processes: 1
 
 sequence_len: 4096
 sample_packing: false

docker/Dockerfile

@@ -6,14 +6,11 @@ ARG AXOLOTL_EXTRAS=""
 ARG AXOLOTL_ARGS=""
 ARG CUDA="118"
 ARG PYTORCH_VERSION="2.1.2"
-ARG TARGETARCH
 
 ENV PYTORCH_VERSION=$PYTORCH_VERSION
 
 RUN apt-get update && \
-    apt-get install -y --allow-change-held-packages vim curl nano libnccl2 libnccl-dev rsync s3fs && \
+    apt-get install -y --allow-change-held-packages vim curl nano libnccl2 libnccl-dev rsync s3fs
-    rm -rf /var/cache/apt/archives && \
-    rm -rf /var/lib/apt/lists/*
 
 WORKDIR /workspace
 
@@ -21,26 +18,22 @@ RUN git clone --depth=1 https://github.com/axolotl-ai-cloud/axolotl.git
 
 WORKDIR /workspace/axolotl
 
-# If AXOLOTL_EXTRAS is set, append it in brackets; don't install deepspeed with arm64
+# If AXOLOTL_EXTRAS is set, append it in brackets
-RUN if [ "$TARGETARCH" = "arm64" ]; then \
+RUN if [ "$AXOLOTL_EXTRAS" != "" ] ; then \
-        BASE_EXTRAS="flash-attn,ring-flash-attn,optimizers,ray"; \
+        pip install --no-build-isolation -e .[deepspeed,flash-attn,ring-flash-attn,optimizers,ray,$AXOLOTL_EXTRAS] $AXOLOTL_ARGS; \
     else \
-        BASE_EXTRAS="deepspeed,flash-attn,ring-flash-attn,optimizers,ray"; \
+        pip install --no-build-isolation -e .[deepspeed,flash-attn,ring-flash-attn,optimizers,ray] $AXOLOTL_ARGS; \
-    fi && \
+    fi
-    if [ "$AXOLOTL_EXTRAS" != "" ]; then \
-        pip install --no-build-isolation -e .[$BASE_EXTRAS,$AXOLOTL_EXTRAS] $AXOLOTL_ARGS; \
-    else \
-        pip install --no-build-isolation -e .[$BASE_EXTRAS] $AXOLOTL_ARGS; \
-    fi && \    python scripts/unsloth_install.py | sh && \
-    python scripts/cutcrossentropy_install.py | sh && \
-    pip install pytest && \
-    pip cache purge
 
-# fix so that git fetch/pull from remote works with shallow clone
+RUN python scripts/unsloth_install.py | sh
+RUN python scripts/cutcrossentropy_install.py | sh
+
+# So we can test the Docker image
+RUN pip install pytest
+
+# fix so that git fetch/pull from remote works
 RUN git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && \
-    git config --get remote.origin.fetch && \
+    git config --get remote.origin.fetch
-    git config --global credential.helper store
 
-COPY .axolotl-complete.bash /root/.axolotl-complete.bash
+# helper for huggingface-login cli
-RUN chmod +x /root/.axolotl-complete.bash && \
+RUN git config --global credential.helper store
-    echo 'source /root/.axolotl-complete.bash' >> ~/.bashrc

docker/Dockerfile-base

@@ -2,91 +2,42 @@ ARG CUDA_VERSION="11.8.0"
 ARG CUDNN_VERSION="8"
 ARG UBUNTU_VERSION="22.04"
 ARG MAX_JOBS=4
-ARG TARGETARCH
 
 FROM nvidia/cuda:$CUDA_VERSION-cudnn$CUDNN_VERSION-devel-ubuntu$UBUNTU_VERSION AS base-builder
 
 ENV PATH="/root/miniconda3/bin:${PATH}"
 
-ARG TARGETARCH
+ARG PYTHON_VERSION="3.10"
-ARG PYTHON_VERSION="3.11"
 ARG PYTORCH_VERSION="2.1.2"
-ARG CUDA="128"
+ARG CUDA="118"
 ARG TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 9.0+PTX"
 
 ENV PYTHON_VERSION=$PYTHON_VERSION
 ENV TORCH_CUDA_ARCH_LIST=$TORCH_CUDA_ARCH_LIST
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends \
+    && apt-get install -y wget git build-essential ninja-build git-lfs libaio-dev pkg-config && rm -rf /var/lib/apt/lists/* \
-        wget git build-essential ninja-build git-lfs libaio-dev pkg-config \
+    && wget \
-        ibverbs-providers ibverbs-utils infiniband-diags  \
+    https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
-        librdmacm-dev librdmacm1 rdmacm-utils slurm-wlm \
-    && rm -rf /var/cache/apt/archives \
-    && rm -rf /var/lib/apt/lists/* \
-    && if [ "$TARGETARCH" = "amd64" ]; then \
-        MINICONDA_ARCH="x86_64"; \
-    elif [ "$TARGETARCH" = "arm64" ]; then \
-        MINICONDA_ARCH="aarch64"; \
-    else \
-        echo "Unsupported architecture: $TARGETARCH"; exit 1; \
-    fi \
-    && wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${MINICONDA_ARCH}.sh \
     && mkdir /root/.conda \
-    && bash Miniconda3-latest-Linux-${MINICONDA_ARCH}.sh -b \
+    && bash Miniconda3-latest-Linux-x86_64.sh -b \
-    && rm -f Miniconda3-latest-Linux-${MINICONDA_ARCH}.sh \
+    && rm -f Miniconda3-latest-Linux-x86_64.sh \
-    && conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main \
-    && conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r \
     && conda create -n "py${PYTHON_VERSION}" python="${PYTHON_VERSION}"
 
 ENV PATH="/root/miniconda3/envs/py${PYTHON_VERSION}/bin:${PATH}"
 
 WORKDIR /workspace
 
-RUN python3 -m pip install --upgrade pip && pip3 install -U packaging==23.2 setuptools==75.8.0 wheel psutil && \
+RUN python3 -m pip install --upgrade pip && pip3 install -U packaging==23.2 setuptools==75.8.0 wheel && \
     python3 -m pip install --no-cache-dir -U torch==${PYTORCH_VERSION}+cu${CUDA} torchvision --extra-index-url https://download.pytorch.org/whl/cu$CUDA && \
-    python3 -m pip cache purge
+    python3 -m pip install --no-cache-dir "causal_conv1d @ git+https://github.com/Dao-AILab/causal-conv1d.git@main" && \
-
+    python3 -m pip install --no-cache-dir "mamba_ssm @ git+https://github.com/state-spaces/mamba.git@main"
-RUN if [ "$CUDA" != "130" ] ; then \
-        CAUSAL_CONV1D_FORCE_CXX11_ABI=TRUE CAUSAL_CONV1D_FORCE_BUILD=TRUE python3 -m pip install --no-cache-dir "causal_conv1d @ git+https://github.com/Dao-AILab/causal-conv1d.git@v1.5.4"; \
-        python3 -m pip install --no-cache-dir "mamba_ssm @ git+https://github.com/state-spaces/mamba.git@main"; \
-        python3 -m pip cache purge; \
-    fi
 
 RUN git lfs install --skip-repo && \
     pip3 install awscli && \
     # The base image ships with `pydantic==1.8.2` which is not working
-    pip3 install -U --no-cache-dir pydantic==1.10.10 && \
+    pip3 install -U --no-cache-dir pydantic==1.10.10
-    pip3 cache purge
 
-RUN case "$PYTORCH_VERSION" in \
+RUN if [ "$PYTORCH_VERSION" = "2.6.0" ] && [ "$CUDA" = "124" ] ; then \
-        2.9.[0-9]*) \
+        FLASH_ATTENTION_FORCE_BUILD="TRUE" pip3 install --no-build-isolation flash-attn==2.8.0.post2; \
-            if [ "$CUDA" = "128" ]; then \
+    fi
-                if [ "$TARGETARCH" = "amd64" ]; then \
-                    WHL_FILE="flash_attn-2.8.3+cu128torch2.9-cp311-cp311-linux_x86_64.whl"; \
-                    WHL_VERSION="v0.5.4"; \
-                elif [ "$TARGETARCH" = "arm64" ]; then \
-                    WHL_FILE="flash_attn-2.8.3+cu128torch2.9-cp311-cp311-linux_aarch64.whl"; \
-                    WHL_VERSION="v0.6.4"; \
-                else \
-                    echo "Unsupported architecture: $TARGETARCH"; exit 1; \
-                fi; \
-                wget -nv https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/${WHL_VERSION}/${WHL_FILE}; \
-                pip3 install --no-cache-dir ${WHL_FILE}; \
-                rm ${WHL_FILE}; \
-            elif [ "$CUDA" = "130" ]; then \
-                if [ "$TARGETARCH" = "amd64" ]; then \
-                    WHL_FILE="flash_attn-2.8.3+cu130torch2.9-cp311-cp311-linux_x86_64.whl"; \
-                    WHL_VERSION="v0.5.4"; \
-                elif [ "$TARGETARCH" = "arm64" ]; then \
-                    WHL_FILE="flash_attn-2.8.3+cu130torch2.9-cp311-cp311-linux_aarch64.whl"; \
-                    WHL_VERSION="v0.6.4"; \
-                else \
-                    echo "Unsupported architecture: $TARGETARCH"; exit 1; \
-                fi; \
-                wget -nv https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/${WHL_VERSION}/${WHL_FILE}; \
-                pip3 install --no-cache-dir ${WHL_FILE}; \
-                rm ${WHL_FILE}; \
-            fi \
-            ;; \
-    esac

docker/Dockerfile-base-nightly

@@ -22,22 +22,18 @@ RUN apt-get update \
     && mkdir /root/.conda \
     && bash Miniconda3-latest-Linux-x86_64.sh -b \
     && rm -f Miniconda3-latest-Linux-x86_64.sh \
-    && conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main \
-    && conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r \
     && conda create -n "py${PYTHON_VERSION}" python="${PYTHON_VERSION}"
 
 ENV PATH="/root/miniconda3/envs/py${PYTHON_VERSION}/bin:${PATH}"
 
 WORKDIR /workspace
 
-RUN python3 -m pip install --upgrade pip && pip3 install -U packaging==23.2 setuptools==75.8.0 wheel && \
+RUN python3 -m pip install --upgrade pip && pip3 install packaging && \
     python3 -m pip install --no-cache-dir -U torch --extra-index-url https://download.pytorch.org/whl/nightly/cu$CUDA && \
     python3 -m pip install --no-cache-dir "causal_conv1d @ git+https://github.com/Dao-AILab/causal-conv1d.git@main" && \
-    python3 -m pip install --no-cache-dir "mamba_ssm @ git+https://github.com/state-spaces/mamba.git@main" && \
+    python3 -m pip install --no-cache-dir "mamba_ssm @ git+https://github.com/state-spaces/mamba.git@main"
-    python3 -m pip cache purge
 
 RUN git lfs install --skip-repo && \
     pip3 install awscli && \
     # The base image ships with `pydantic==1.8.2` which is not working
-    pip3 install -U --no-cache-dir pydantic==1.10.10 && \
+    pip3 install -U --no-cache-dir pydantic==1.10.10
-    pip3 cache purge

docker/Dockerfile-cloud

@@ -14,10 +14,7 @@ COPY scripts/motd /etc/motd
 
 RUN pip install jupyterlab notebook ipywidgets && \
     jupyter lab clean
-RUN apt update && \
+RUN apt install --yes --no-install-recommends openssh-server tmux iproute2 nvtop && \
-    apt install --yes --no-install-recommends openssh-server tmux iproute2 nvtop && \
-    rm -rf /var/cache/apt/archives && \
-    rm -rf /var/lib/apt/lists/* && \
     mkdir -p ~/.ssh && \
     chmod 700 ~/.ssh && \
     printf "\n[[ -z \"\$TMUX\"  ]] && { tmux attach-session -t ssh_tmux || tmux new-session -s ssh_tmux; exit; }\n" >> ~/.bashrc && \

docker/Dockerfile-cloud-no-tmux

@@ -9,15 +9,13 @@ ENV HF_HUB_ENABLE_HF_TRANSFER="1"
 EXPOSE 8888
 EXPOSE 22
 
-COPY scripts/cloud-entrypoint.sh /root/cloud-entrypoint.sh
+COPY scripts/cloud-entrypoint-term.sh /root/cloud-entrypoint.sh
 COPY scripts/motd /etc/motd
 
 RUN pip install jupyterlab notebook ipywidgets && \
     jupyter lab clean
-RUN apt update && \
+RUN apt install --yes --no-install-recommends openssh-server tmux sudo && \
-    apt install --yes --no-install-recommends openssh-server tmux iproute2 nvtop ibverbs-providers ibverbs-utils infiniband-diags librdmacm-dev librdmacm1 rdmacm-utils slurm-wlm && \
+    pip3 install -U --no-cache-dir grpcio ray[default]==2.9.3 && \
-    rm -rf /var/cache/apt/archives && \
-    rm -rf /var/lib/apt/lists/* && \
     mkdir -p ~/.ssh && \
     chmod 700 ~/.ssh && \
     printf "[ ! -z \"\$TERM\" -a -r /etc/motd ] && cat /etc/motd\n" >> ~/.bashrc && \

docker/Dockerfile-uv-base

@@ -2,7 +2,6 @@ ARG CUDA_VERSION="12.6.3"
 ARG CUDNN_VERSION=""
 ARG UBUNTU_VERSION="22.04"
 ARG MAX_JOBS=4
-ARG TARGETARCH
 
 FROM nvidia/cuda:$CUDA_VERSION-cudnn$CUDNN_VERSION-devel-ubuntu$UBUNTU_VERSION AS base-builder
 
@@ -31,36 +30,7 @@ RUN uv venv --no-project --relocatable axolotl-venv
 ENV PATH="/workspace/axolotl-venv/bin:${PATH}"
 
 RUN uv pip install packaging setuptools wheel psutil \
-    && uv pip install torch==${PYTORCH_VERSION} torchvision \
+    && uv pip install torch==${PYTORCH_VERSION} \
+    && uv pip install --no-build-isolation "causal_conv1d @ git+https://github.com/Dao-AILab/causal-conv1d.git@main" \
+    && uv pip install "mamba_ssm @ git+https://github.com/state-spaces/mamba.git@main" \
     && uv pip install awscli pydantic
-
-RUN if [ "$TARGETARCH" = "amd64" ]; then \
-        uv pip install --no-build-isolation "causal_conv1d @ git+https://github.com/Dao-AILab/causal-conv1d.git@main"; \
-        uv pip install "mamba_ssm @ git+https://github.com/state-spaces/mamba.git@main"; \
-    fi
-
-RUN case "$PYTORCH_VERSION" in \
-        2.9.[0-9]*) \
-            if [ "$TARGETARCH" = "amd64" ]; then \
-                if [ "$CUDA" = "128" ]; then \
-                    wget -nv https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.5.4/flash_attn-2.8.3+cu128torch2.9-cp311-cp311-linux_x86_64.whl; \
-                    uv pip install --no-cache-dir flash_attn-2.8.3+cu128torch2.9-cp311-cp311-linux_x86_64.whl; \
-                    rm flash_attn-2.8.3+cu128torch2.9-cp311-cp311-linux_x86_64.whl; \
-                elif [ "$CUDA" = "130" ]; then \
-                    wget -nv https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.5.4/flash_attn-2.8.3+cu130torch2.9-cp311-cp311-linux_x86_64.whl; \
-                    uv pip install --no-cache-dir flash_attn-2.8.3+cu130torch2.9-cp311-cp311-linux_x86_64.whl; \
-                    rm flash_attn-2.8.3+cu130torch2.9-cp311-cp311-linux_x86_64.whl; \
-                fi \
-            elif [ "$TARGETARCH" = "arm64" ]; then \
-                if [ "$CUDA" = "128" ]; then \
-                    wget -nv https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.6.4/flash_attn-2.8.3+cu128torch2.9-cp311-cp311-linux_aarch64.whl; \
-                    uv pip install --no-cache-dir flash_attn-2.8.3+cu128torch2.9-cp311-cp311-linux_aarch64.whl; \
-                    rm flash_attn-2.8.3+cu128torch2.9-cp311-cp311-linux_aarch64.whl; \
-                elif [ "$CUDA" = "130" ]; then \
-                    wget -nv https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.6.4/flash_attn-2.8.3+cu130torch2.9-cp311-cp311-linux_aarch64.whl; \
-                    uv pip install --no-cache-dir flash_attn-2.8.3+cu130torch2.9-cp311-cp311-linux_aarch64.whl; \
-                    rm flash_attn-2.8.3+cu130torch2.9-cp311-cp311-linux_aarch64.whl; \
-                fi \
-            fi \
-            ;; \
-    esac

docs/.gitignore

@@ -3,5 +3,3 @@ _site/
 /api/*.qmd
 /api/*.html
 config-reference.qmd
-models/**/*.qmd
-models/**/*.html

docs/checkpoint_saving.qmd

@@ -1,86 +0,0 @@
----
-title: "Checkpoint Saving"
-format:
-  html:
-    toc: true
-    toc-depth: 2
-    number-sections: true
-execute:
-  enabled: false
----
-
-## Overview
-
-Axolotl supports on-demand checkpoint saving during training. You can trigger checkpoints via file-based triggers (for programmatic control) or Control+C (for interactive use).
-
-## File-Based Checkpoint Trigger
-
-### Configuration
-
-Enable in your config:
-
-```yaml
-dynamic_checkpoint:
-  enabled: true
-  check_interval: 100  # Optional: check every N steps (default: 100)
-  trigger_file_path: "axolotl_checkpoint.save"  # Optional: custom filename
-```
-
-**Options:**
-- `enabled`: `true` to enable (required)
-- `check_interval`: Steps between file checks. Default: 100. Lower = faster response, higher I/O overhead.
-- `trigger_file_path`: Custom trigger filename. Default: `axolotl_checkpoint.save`
-
-### How It Works
-
-1. Rank 0 checks for trigger file every `check_interval` steps in `output_dir`
-2. When detected, file is deleted and checkpoint is saved
-3. In distributed training, rank 0 broadcasts to synchronize all ranks
-
-### Usage
-
-**Command line:**
-```bash
-touch /path/to/output_dir/axolotl_checkpoint.save
-```
-
-**Programmatic:**
-```python
-from pathlib import Path
-Path("/path/to/output_dir/axolotl_checkpoint.save").touch()
-```
-
-Checkpoint saves within the next `check_interval` steps. The trigger file is auto-deleted after detection, so you can create it multiple times.
-
-**Custom filename:**
-```yaml
-dynamic_checkpoint:
-  enabled: true
-  trigger_file_path: "my_trigger.save"
-```
-```bash
-touch /path/to/output_dir/my_trigger.save
-```
-
-## Control+C (SIGINT) Checkpoint
-
-Pressing `Ctrl+C` during training saves the model state and exits gracefully. **Note:** This saves only the model weights, not optimizer state. For resumable checkpoints, use the file-based trigger.
-
-## Best Practices
-
-- **Check interval**: Lower values (10-50) for fast training, default 100 for slower training
-- **Distributed training**: Create trigger file once; rank 0 handles synchronization
-- **Resume**: Dynamic checkpoints can be resumed like regular checkpoints via `resume_from_checkpoint`
-
-## Example
-
-```yaml
-output_dir: ./outputs/lora-out
-save_steps: 500  # Scheduled checkpoints
-
-dynamic_checkpoint:
-  enabled: true
-  check_interval: 50
-```
-
-This enables scheduled checkpoints every 500 steps plus on-demand saves via file trigger (checked every 50 steps).

docs/cli.qmd

@@ -23,20 +23,6 @@ axolotl <command> [config.yml] [options]
 
 The config file can be local or a URL to a raw YAML file.
 
-### Launcher Arguments
-
-For commands that support multi-GPU (`train`, `evaluate`, ...), you can pass launcher-specific arguments using the `--` separator:
-
-```bash
-# Pass torchrun arguments
-axolotl train config.yml --launcher torchrun -- --nproc_per_node=2 --nnodes=1
-
-# Pass accelerate arguments
-axolotl train config.yml --launcher accelerate -- --config_file=accelerate_config.yml --num_processes=4
-```
-
-Arguments after `--` are passed directly to the launcher (torchrun, accelerate launch, etc.).
-
 ## Command Reference
 
 ### fetch
@@ -94,11 +80,7 @@ axolotl train config.yml \
     --num-epochs 3
 
 # Training without accelerate
-axolotl train config.yml --launcher python
+axolotl train config.yml --no-accelerate
-
-# Pass launcher-specific arguments using -- separator
-axolotl train config.yml --launcher torchrun -- --nproc_per_node=2 --nnodes=1
-axolotl train config.yml --launcher accelerate -- --config_file=accelerate_config.yml
 
 # Resume training from checkpoint
 axolotl train config.yml --resume-from-checkpoint path/to/checkpoint
@@ -193,9 +175,6 @@ Evaluates a model's performance (loss etc) on the train and eval datasets.
 ```bash
 # Basic evaluation
 axolotl evaluate config.yml
-
-# Evaluation with launcher arguments
-axolotl evaluate config.yml --launcher torchrun -- --nproc_per_node=2
 ```
 
 ### lm-eval
@@ -308,6 +287,9 @@ axolotl preprocess config.yml --cloud cloud_config.yml
 # Train on cloud
 axolotl train config.yml --cloud cloud_config.yml
 
+# Train without accelerate on cloud
+axolotl train config.yml --cloud cloud_config.yml --no-accelerate
+
 # Run lm-eval on cloud
 axolotl lm-eval config.yml --cloud cloud_config.yml
 ```

docs/dataset-formats/conversation.qmd

@@ -212,26 +212,10 @@ Instead of passing `tools` via the system prompt, an alternative method would be
 Tools need to follow [JSON schema](https://json-schema.org/learn/getting-started-step-by-step).
 :::
 
-::: {.callout-warning}
-If you have tool arguments with same name but different dtypes (like `"time": string` and `"time": number`), please save `arguments: ` as JSON string to prevent `datasets` from having casting issues.
-
-```
-"arguments": "{\"...\": \"...\"}"
-```
-
-The same is applicable for tool parameters.
-
-```
-"parameters": "{\"...\": \"...\"}"
-```
-
-:::
-
-Example config for Llama4:
 ```yaml
 chat_template: llama4
 datasets:
-  - path: Nanobit/text-tools-2k-test
+  - path: ...
     type: chat_template
     # field_tools: tools # default is `tools`
 ```

docs/dataset-formats/index.qmd

@@ -61,7 +61,7 @@ While we recommend `.jsonl`, you can also use the other formats (`csv`, `parquet
 
 ### Pre-training without streaming
 
-In the case that the dataset is small and can be loaded entirely into memory, another approach to running pre-training is to use the `completion` format. This would mean that the entire dataset is pre-tokenized instead of on-demand in streaming.
+On the rare case that the dataset is small and can be loaded entirely into memory, another approach to running pre-training is to use the `completion` format. This would mean that the entire dataset is pre-tokenized instead of on-demand in streaming.
 
 One benefit of this is that the tokenization can be performed separately on a CPU-only machine, and then transferred to a GPU machine for training to save costs.
 

docs/debugging.qmd

@@ -29,7 +29,7 @@ While debugging it's helpful to simplify your test scenario as much as possible.
 1. **Make sure you are using the latest version of axolotl**:  This project changes often and bugs get fixed fast.  Check your git branch and make sure you have pulled the latest changes from `main`.
 1. **Eliminate concurrency**: Restrict the number of processes to 1 for both training and data preprocessing:
     - Set `CUDA_VISIBLE_DEVICES` to a single GPU, ex: `export CUDA_VISIBLE_DEVICES=0`.
-    - Set `dataset_num_proc: 1` in your axolotl config or run the training command with `--dataset_num_proc=1`.
+    - Set `dataset_processes: 1` in your axolotl config or run the training command with `--dataset_processes=1`.
 2. **Use a small dataset**: Construct or use a small dataset from HF Hub. When using a small dataset, you will often have to make sure `sample_packing: False` and `eval_sample_packing: False` to avoid errors.  If you are in a pinch and don't have time to construct a small dataset but want to use from the HF Hub, you can shard the data (this will still tokenize the entire dataset, but will only use a fraction of the data for training.  For example, to shard the dataset into 20 pieces, add the following to your axolotl config):
 
     ```yaml
@@ -101,7 +101,7 @@ For example, to mimic the command `cd devtools && CUDA_VISIBLE_DEVICES=0 acceler
                 "-m", "axolotl.cli.train", "dev_chat_template.yml",
                 // The flags below simplify debugging by overriding the axolotl config
                 // with the debugging tips above.  Modify as needed.
-                "--dataset_num_proc=1",      // limits data preprocessing to one process
+                "--dataset_processes=1",      // limits data preprocessing to one process
                 "--max_steps=1",              // limits training to just one step
                 "--batch_size=1",             // minimizes batch size
                 "--micro_batch_size=1",       // minimizes batch size

docs/docker.qmd

@@ -32,8 +32,11 @@ main-base-py{python_version}-cu{cuda_version}-{pytorch_version}
 
 Tags examples:
 
-- `main-base-py3.11-cu128-2.8.0`
+- `main-base-py3.11-cu128-2.7.1`
-- `main-base-py3.11-cu128-2.9.1`
+- `main-base-py3.11-cu126-2.7.1`
+- `main-base-py3.11-cu126-2.7.0`
+- `main-base-py3.11-cu126-2.6.0`
+- `main-base-py3.11-cu124-2.6.0`
 
 ## Main
 
@@ -71,12 +74,15 @@ There may be some extra tags appended to the image, like `-vllm` which installs
 
 Tags examples:
 
-- `main-py3.11-cu128-2.8.0`
+- `main-py3.11-cu128-2.7.1`
-- `main-py3.11-cu128-2.9.1`
+- `main-py3.11-cu126-2.7.1`
+- `main-py3.11-cu126-2.7.0`
+- `main-py3.11-cu126-2.6.0`
+- `main-py3.11-cu124-2.6.0`
 - `main-latest`
 - `main-20250303-py3.11-cu124-2.6.0`
 - `main-20250303-py3.11-cu126-2.6.0`
-- `0.12.0`
+- `0.10.1`
 
 ## Cloud
 

docs/faq.qmd

@@ -63,14 +63,6 @@ description: Frequently asked questions
 
 > A: There seems to be a wheel issue with FA2 2.8.0 on CUDA 12.4. Try CUDA 12.6 instead or downgrade to FA2 2.7.4. Please refer to the upstream issue: https://github.com/Dao-AILab/flash-attention/issues/1717.
 
-**Q: Can we mix text and text+image datasets for VLM training?**
-
-> A: Yes, you can for newer VLM arch. The ones that would not work are LLaVA / Pixtral arch. If you notice one not working, please let us know!
-
-**Q: Why is `memory/max_*` different from `nvidia-smi`?**
-
-> A: We use `torch` APIs to retrieve this information. You can see https://docs.pytorch.org/docs/stable/notes/cuda.html#cuda-memory-management for more information.
-
 ### Chat templates
 
 **Q: `jinja2.exceptions.UndefinedError: 'dict object' has no attribute 'content' / 'role' / ____`**
@@ -144,11 +136,3 @@ description: Frequently asked questions
 >   dynamic: false
 >   mode: max-autotune-no-cudagraphs
 > ```
-
-**Q: `ValueError("Backward pass should have cleared tracker of all tensors")`
-
-> A: This may happen due to edge cases in using the modern OffloadActivations context manager for CUDA streams. If you encounter this error, you may have success using the naive implementation with `offload_activations: legacy` in your YAML.
-
-**Q: `Error parsing tool_calls arguments as JSON.`
-
-> A: There is an error parsing string arguments to a dict. Please check your dataset and the error message for more details.

docs/fsdp_qlora.qmd

@@ -1,5 +1,5 @@
 ---
-title: "FSDP + QLoRA"
+title: "FDSP + QLoRA"
 description: Use FSDP with QLoRA to fine-tune large LLMs on consumer GPUs.
 format:
   html:
@@ -23,12 +23,6 @@ To enable `QLoRA` with `FSDP`, you need to perform the following steps:
 2. Enable FSDP in your axolotl config, as [described here](multi-gpu.qmd#sec-fsdp).
 3. Use one of the supported model types: `llama`, `mistral` or `mixtral`.
 
-## Enabling Swap for FSDP2
-
-If available memory is insufficient even after FSDP's CPU offloading, you can enable swap memory usage by setting `cpu_offload_pin_memory: false` alongside `offload_params: true` in FSDP config.
-
-This disables memory pinning, allowing FSDP to use disk swap space as fallback. Disabling memory pinning itself incurs performance overhead, and actually having to use swap adds more, but it may enable training larger models that would otherwise cause OOM errors on resource constrained systems.
-
 ## Example Config
 
 [examples/llama-2/qlora-fsdp.yml](../examples/llama-2/qlora-fsdp.yml) contains an example of how to enable QLoRA + FSDP in axolotl.

docs/installation.qmd

@@ -26,7 +26,7 @@ Follow the instructions at: [https://pytorch.org/get-started/locally/](https://p
 :::
 
 ::: {.callout-important}
-For Blackwell GPUs, please use Pytorch 2.9.1 and CUDA 12.8.
+For Blackwell GPUs, please use Pytorch 2.7.0 and CUDA 12.8.
 :::
 
 ### PyPI Installation (Recommended) {#sec-pypi}
@@ -111,7 +111,7 @@ docker run --privileged --gpus '"all"' --shm-size 10g --rm -it \
 :::
 
 ::: {.callout-important}
-For Blackwell GPUs, please use `axolotlai/axolotl:main-py3.11-cu128-2.9.1` or the cloud variant `axolotlai/axolotl-cloud:main-py3.11-cu128-2.9.1`.
+For Blackwell GPUs, please use `axolotlai/axolotl:main-py3.11-cu128-2.7.0` or the cloud variant `axolotlai/axolotl-cloud:main-py3.11-cu128-2.7.0`.
 :::
 
 Please refer to the [Docker documentation](docker.qmd) for more information on the different Docker images that are available.
@@ -124,17 +124,14 @@ For providers supporting Docker:
 
 - Use `axolotlai/axolotl-cloud:main-latest`
 - Available on:
-    - [RunPod](https://runpod.io/gsc?template=v2ickqhz9s&ref=6i7fkpdz)
+  - [Latitude.sh](https://latitude.sh/blueprint/989e0e79-3bf6-41ea-a46b-1f246e309d5c)
-    - [Vast.ai](https://cloud.vast.ai?ref_id=62897&template_id=bdd4a49fa8bce926defc99471864cace&utm_source=axolotl&utm_medium=partner&utm_campaign=template_launch_july2025&utm_content=docs_link)
+  - [JarvisLabs.ai](https://jarvislabs.ai/templates/axolotl)
-    - [PRIME Intellect](https://app.primeintellect.ai/dashboard/create-cluster?image=axolotl&location=Cheapest&security=Cheapest&show_spot=true)
+  - [RunPod](https://runpod.io/gsc?template=v2ickqhz9s&ref=6i7fkpdz)
-    - [Modal](https://www.modal.com?utm_source=github&utm_medium=github&utm_campaign=axolotl)
+  - [Novita](https://novita.ai/gpus-console?templateId=311)
-    - [Novita](https://novita.ai/gpus-console?templateId=311)
-    - [JarvisLabs.ai](https://jarvislabs.ai/templates/axolotl)
-    - [Latitude.sh](https://latitude.sh/blueprint/989e0e79-3bf6-41ea-a46b-1f246e309d5c)
 
 ### Google Colab {#sec-colab}
 
-[![](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/axolotl-ai-cloud/axolotl/blob/main/examples/colab-notebooks/colab-axolotl-example.ipynb#scrollTo=msOCO4NRmRLa)
+Use our [example notebook](../examples/colab-notebooks/colab-axolotl-example.ipynb).
 
 ## Platform-Specific Instructions {#sec-platform-specific}
 

docs/lora_optims.qmd

@@ -5,11 +5,10 @@ description: "Custom autograd functions and Triton kernels in Axolotl for optimi
 
 Inspired by [Unsloth](https://github.com/unslothai/unsloth), we've implemented two
 optimizations for LoRA and QLoRA fine-tuning, supporting both single GPU and multi-GPU
-(including the DDP, DeepSpeed, and FSDP2 settings) training. These include (1) SwiGLU
+(in the DDP and DeepSpeed settings) training. These include (1) SwiGLU and GEGLU activation function
-and GEGLU activation function Triton kernels, and (2) LoRA MLP and attention custom
+Triton kernels, and (2) LoRA MLP and attention custom autograd functions. Our goal was
-autograd functions. Our goal was to leverage operator fusion and tensor re-use in order
+to leverage operator fusion and tensor re-use in order to improve speed and reduce
-to improve speed and reduce memory usage during the forward and backward passes of
+memory usage during the forward and backward passes of these calculations.
-these calculations.
 
 We currently support several common model architectures, including (but not limited to):
 
@@ -132,5 +131,6 @@ computation path.
 ## Future Work
 
 - Support for additional model architectures
+- Support for the FSDP setting
 - Support for dropout and bias
 - Additional operator fusions

docs/lr_groups.qmd

@@ -27,9 +27,3 @@ learning_rate: 2e-5
 In this example, we have a default learning rate of 2e-5 across the entire model, but we have a separate learning rate
 of 1e-6 for all the self attention `o_proj` modules across all layers, and a learning are of 1e-5 to the 3rd layer's
 self attention `q_proj` module.
-
-::: {.callout-note}
-
-We currently only support varying `lr` for now. If you're interested in adding support for others (`weight_decay`), we welcome PRs. See https://github.com/axolotl-ai-cloud/axolotl/blob/613bcf90e58f3ab81d3827e7fc572319908db9fb/src/axolotl/core/trainers/mixins/optimizer.py#L17
-
-:::

docs/mixed_precision.qmd

@@ -1,149 +0,0 @@
----
-title: "Mixed Precision Training"
-format:
-  html:
-    toc: true
-    toc-depth: 3
-    number-sections: true
-    code-tools: true
-execute:
-  enabled: false
----
-
-Mixed precision training uses lower precision data types to reduce memory usage and increase training speed while maintaining model quality. Axolotl supports several mixed precision formats:
-
-- **FP16** - Half precision 16-bit (Pascal generation+)
-- **BF16** - Brain Float 16-bit (Ampere generation+)
-- **FP8** - 8-bit floating point (Hopper generation+)
-
-## FP16 Mixed Precision {#sec-fp16}
-
-### Overview {#sec-fp16-overview}
-
-FP16 is the traditional half-precision format, supported on older GPUs but can be less numerically stable than BF16.
-
-### Configuration {#sec-fp16-config}
-
-```{.yaml}
-fp16: true
-```
-
-### FP16 Considerations {#sec-fp16-considerations}
-
-- May require gradient scaling to prevent underflow
-- Less numerically stable than BF16
-- Can cause training instability with some model architectures
-- Consider using BF16 if your hardware supports it
-
-## BF16 Mixed Precision {#sec-bf16}
-
-### Overview {#sec-bf16-overview}
-
-BF16 (Brain Float 16) offers better numerical stability than FP16 and is the recommended mixed precision format for modern GPUs. It provides the same dynamic range as FP32 while using half the memory.
-
-### Configuration {#sec-bf16-config}
-
-```{.yaml}
-# Automatic BF16 detection (recommended)
-bf16: auto
-
-# Or explicitly enable
-bf16: true
-
-# For evaluation with BF16
-bf16: full  # Equivalent to bf16_full_eval in the HF trainer
-```
-
-## FP8 Mixed Precision {#sec-fp8}
-
-::: {.callout-note}
-FP8 support is experimental and requires compatible hardware (H100, H200) and recent PyTorch versions with TorchAO.
-:::
-
-### What is FP8? {#sec-fp8-overview}
-
-FP8 (8-bit floating point) can provide significant time savings compared to FP16/BF16 while maintaining training stability. Axolotl's implementation uses PyTorch's TorchAO library with "tensorwise" scaling strategy.
-
-### Requirements {#sec-fp8-software}
-
-- Hopper+ GPUs (H100/H200)
-- PyTorch 2.7+ (+ compatible TorchAO version)
-- CUDA 12.4+
-
-### Configuration {#sec-fp8-config}
-
-Add to your YAML config:
-
-```{.yaml}
-# Enable FP8 mixed precision
-fp8: true
-
-# Optional: Enable FP8 for FSDP all-gather operations
-fp8_enable_fsdp_float8_all_gather: true
-
-# Enable torch.compile (almost always necessary for FP8 speedups)
-torch_compile: true
-```
-
-::: {.callout-important}
-**torch.compile is critical for FP8 performance**
-
-FP8 training requires `torch_compile: true` to see meaningful speedups. Without compilation, FP8 may actually be slower and use more memory than FP16/BF16.
-:::
-
-### Advanced FP8 Configs {#sec-fp8-advanced}
-
-For [FSDP](multi-gpu.qmd#sec-fsdp) (Fully Sharded Data Parallel) training:
-
-```{.yaml}
-fp8: true
-fp8_enable_fsdp_float8_all_gather: true
-
-torch_compile: true
-
-# FSDP configuration
-fsdp_version: 2
-fsdp_config:
-  offload_params: false
-  cpu_ram_efficient_loading: true
-  auto_wrap_policy: TRANSFORMER_BASED_WRAP
-  transformer_layer_cls_to_wrap: LlamaDecoderLayer
-  state_dict_type: FULL_STATE_DICT
-  reshard_after_forward: true
-```
-
-## Best Practices {#sec-best-practices}
-
-### Choosing Precision Format {#sec-choosing-format}
-
-- **Start with automatic detection**: `bf16: auto`
-- **For Hopper+ (H100/H200)**: Try FP8 + torch.compile for maximum speed
-- **For Ampere (A100/RTX 30/40)**: Use BF16
-- **For older Pascal/Turing GPUs**: Use FP16 with caution
-- **For very old or unsupported GPUs**: Use FP32
-
-### Validation and Testing {#sec-validation}
-
-Always validate your mixed precision setup:
-
-- **Start with a small dataset** to verify stability
-- **Monitor loss curves** for irregularities
-- **Compare with FP32 baseline** when possible
-- **Test evaluation metrics** match expectations
-
-### FP8 Particulars {#sec-fp8-details}
-
-- Use cases
-  - Single GPU training
-  - Multi GPU training with FSDP2 or Deepspeed
-- Speedups
-  - Please refer to the [TorchAO FP8 training benchmarks](https://github.com/pytorch/ao/tree/main/torchao/float8#rowwise-scaling) for expected matmul speedups for different (M, K, N) settings
-  - Concrete number for LLaMA 3 8B training can be found [here](https://github.com/pytorch/ao/tree/main/torchao/float8#training-benchmarks)
-- Known issues:
-  - FP8 + DDP + `torch.compile` (causes [error](https://gist.github.com/djsaunde/0c1664c32e44a64d31b5e01b4aafe5c4))
-  - FP8 + FSDP2 + `torch.compile` + FSDP2 activation checkpointing tends to be _slower_ than the BF16 equivalent training
-  - Flash Attention 2 does not play nicely with `torch.compile`
-
-See `examples/llama-3/3b-fp8-fsdp2.yaml` for an optimized example config. Enabling FP8 mixed precision + FP8 all-gather training results in ~10% faster iterations per second vs. BF16 for a relatively small (3B param) model
-
-For more information on multi-GPU training, see our [Multi-GPU guide](multi-gpu.qmd).

docs/multi-gpu.qmd

@@ -4,7 +4,7 @@ format:
   html:
     toc: true
     toc-depth: 3
-    # number-sections: true
+    number-sections: true
     code-tools: true
 execute:
   enabled: false
@@ -14,18 +14,12 @@ This guide covers advanced training configurations for multi-GPU setups using Ax
 
 ## Overview {#sec-overview}
 
-When training on multiple GPUs, Axolotl supports 3 sharding/parallelism strategies. Additionally, you can layer specific optimization features on top of that strategy.
+Axolotl supports several methods for multi-GPU training:
 
-You generally cannot combine these strategies; they are mutually exclusive.
+- DeepSpeed (recommended)
-
+- FSDP (Fully Sharded Data Parallel)
-1.  **DeepSpeed**: Powerful optimization library, supports ZeRO stages 1-3.
+- Sequence parallelism
-2.  **FSDP (Fully Sharded Data Parallel)**: PyTorch's native sharding implementation (Recommended).
+- FSDP + QLoRA
-3.  **DDP (Distributed Data Parallel)**: PyTorch's native parallelism implementation (Default if neither of the above are selected).
-
-These features can often be combined with the strategies above:
-
-*   **Sequence Parallelism**: Splits long sequences across GPUs (Compatible with DDP, DeepSpeed, and FSDP).
-*   **FSDP + QLoRA**: Combines 4-bit quantization with FSDP (Specific to FSDP).
 
 ## DeepSpeed {#sec-deepspeed}
 
@@ -69,9 +63,16 @@ Start from Stage 1 -> Stage 2 -> Stage 3.
 
 :::
 
-## Fully Sharded Data Parallel (FSDP) {#sec-fsdp}
+::: {.callout-tip}
 
-FSDP allows you to shard model parameters, gradients, and optimizer states across data parallel workers.
+Using ZeRO Stage 3 with Single-GPU training
+
+ZeRO Stage 3 can be used for training on a single GPU by manually setting the environment variables:
+`WORLD_SIZE=1 LOCAL_RANK=0 MASTER_ADDR=0.0.0.0 MASTER_PORT=29500`
+
+:::
+
+## Fully Sharded Data Parallel (FSDP) {#sec-fsdp}
 
 ::: {.callout-note}
 
@@ -79,10 +80,6 @@ FSDP2 is recommended for new users. FSDP1 is deprecated and will be removed in a
 
 :::
 
-### FSDP + QLoRA {#sec-fsdp-qlora}
-
-For combining FSDP with QLoRA, see our [dedicated guide](fsdp_qlora.qmd).
-
 ### Migrating from FSDP1 to FSDP2 {#sec-migrate-fsdp1-fsdp2}
 
 To migrate your config from FSDP1 to FSDP2, you must use the `fsdp_version` top-level config field to specify the FSDP version, and
@@ -100,10 +97,9 @@ fsdp_sync_module_states | **REMOVED**
 fsdp_cpu_ram_efficient_loading | cpu_ram_efficient_loading
 fsdp_state_dict_type | state_dict_type
 fsdp_use_orig_params | **REMOVED**
-fsdp_activation_checkpointing | activation_checkpointing
 
-For more details, please see the migration guide in the [torchtitan repo](https://github.com/pytorch/torchtitan/blob/main/docs/fsdp.md). In Axolotl,
+
-if you were using the following FSDP1 config:
+For example, if you were using the following FSDP1 config:
 
 ```{.yaml}
 fsdp_version: 1
@@ -157,6 +153,10 @@ single sequence causes OOM errors during model training.
 
 See our [dedicated guide](sequence_parallelism.qmd) for more information.
 
+### FSDP + QLoRA {#sec-fsdp-qlora}
+
+For combining FSDP with QLoRA, see our [dedicated guide](fsdp_qlora.qmd).
+
 ## Performance Optimization {#sec-performance}
 
 ### Liger Kernel Integration {#sec-liger}

docs/multi-node.qmd

@@ -69,19 +69,11 @@ export NCCL_BUFFSIZE=2097152
 
 Run the following on each node:
 
-### Option 1: New Axolotl CLI with launcher args (Recommended)
-
-```bash
-axolotl train config.yaml --launcher torchrun -- --nnodes $num_nodes --nproc_per_node $gpu_per_node --rdzv_id $rdzv_id --rdzv_backend c10d --rdzv_endpoint "$head_node_ip:$head_node_port"
-```
-
-### Option 2: Direct torchrun (Legacy)
-
 ```bash
 torchrun --nnodes $num_nodes --nproc_per_node $gpu_per_node --rdzv_id $rdzv_id --rdzv_backend c10d --rdzv_endpoint "$head_node_ip:$head_node_port" -m axolotl.cli.train config.yaml
 ```
 
-Please make sure to substitute the placeholder variables:
+Please make sure to substitute the placeholder variables.
 
 - `num_nodes`: Number of nodes (containing GPUs)
 - `gpu_per_node`: Number of gpus per node
@@ -89,6 +81,8 @@ Please make sure to substitute the placeholder variables:
 - `head_node_port`: Port of the head node (make sure other machines can connect to this. Default 29400)
 - `rdzv_id`: A unique job ID that is used by the job across nodes.
 
-The new CLI approach (Option 1) is recommended as it provides consistent argument handling and works seamlessly with other Axolotl CLI features.
+::: {.callout-note}
+You need to call `axolotl.cli.train` instead of `axolotl train` as the latter calls accelerate under the hood
+:::
 
 More info on the available configs can be found on the Pytorch docs [here](https://pytorch.org/docs/stable/elastic/run.html)

docs/multimodal.qmd

@@ -13,15 +13,9 @@ format:
 - [Pixtral](#sec-pixtral)
 - [Llava-1.5](#sec-llava-15)
 - [Mistral-Small-3.1](#sec-mistral-small-31)
-- [Magistral-Small-2509](#sec-magistral-small-2509)
-- [Voxtral](#sec-voxtral)
 - [Gemma-3](#sec-gemma-3)
-- [Gemma-3n](#sec-gemma-3n)
 - [Qwen2-VL](#sec-qwen2-vl)
 - [Qwen2.5-VL](#sec-qwen25-vl)
-- [SmolVLM2](#sec-smolvlm2)
-- [LFM2-VL](#sec-lfm2-vl)
-- [Intern-VL](#sec-intern-vl)
 
 ## Usage
 
@@ -36,13 +30,14 @@ skip_prepare_dataset: true
 remove_unused_columns: false  # leave columns in place as they are needed to handle image embeddings during training
 sample_packing: false  # not yet supported with multimodal
 
-chat_template:  # see in next section if specified
+chat_template:  # see in next section
 
 # example dataset
 datasets:
   - path: HuggingFaceH4/llava-instruct-mix-vsft
     type: chat_template
     split: train[:1%]
+    field_messages: messages
 
 # (optional) if doing lora, only finetune the Language model,
 # leave the vision model and vision tower frozen
@@ -57,14 +52,10 @@ image_resize_algorithm: bilinear
 
 Please see [examples](https://github.com/axolotl-ai/axolotl/tree/main/examples) folder for full configs.
 
-::: {.callout-tip}
+::: {.callout-warning}
 Some of our chat_templates have been extended to support broader dataset types. This should not break any existing configs.
 :::
 
-::: {.callout-note}
-As of now, we do not truncate nor drop samples based on `sequence_len` as each arch has different ways to process non-text tokens. We are looking for help on this.
-:::
-
 ### Mllama {#sec-mllama}
 
 ```yaml
@@ -99,34 +90,10 @@ chat_template: llava
 
 ### Mistral-Small-3.1 {#sec-mistral-small-31}
 
-::: {.callout-tip}
-Please make sure to install vision lib via `pip install 'mistral-common[opencv]==1.8.5'`
-:::
-
 ```yaml
 base_model: mistralai/Mistral-Small-3.1-24B-Instruct-2503
-```
 
-### Magistral-Small-2509 {#sec-magistral-small-2509}
+chat_template: mistral_v7_tekken
-
-::: {.callout-tip}
-Please make sure to install vision lib via `pip install 'mistral-common[opencv]==1.8.5'`
-:::
-
-```yaml
-base_model: mistralai/Magistral-Small-2509
-```
-
-### Voxtral {#sec-voxtral}
-
-::: {.callout-tip}
-Please make sure to install audio lib via `pip3 install librosa==0.11.0 'mistral_common[audio]==1.8.3'`
-:::
-
-```yaml
-base_model: mistralai/Voxtral-Mini-3B-2507
-
-processor_type: VoxtralProcessor
 ```
 
 ### Gemma-3 {#sec-gemma-3}
@@ -143,22 +110,6 @@ base_model: google/gemma-3-4b-it
 chat_template: gemma3
 ```
 
-### Gemma-3n {#sec-gemma-3n}
-
-::: {.callout-warning}
-The model's initial loss and grad norm will be very high. We suspect this to be due to the Conv in the vision layers.
-:::
-
-::: {.callout-tip}
-Please make sure to install `timm` via `pip3 install timm==1.0.17`
-:::
-
-```yaml
-base_model: google/gemma-3n-E2B-it
-
-chat_template: gemma3n
-```
-
 ### Qwen2-VL {#sec-qwen2-vl}
 
 ```yaml
@@ -175,53 +126,13 @@ base_model: Qwen/Qwen2.5-VL-7B-Instruct
 chat_template: qwen2_vl  # same as qwen2-vl
 ```
 
-### Qwen3-VL {#sec-qwen3-vl}
-
-```yaml
-base_model: Qwen/Qwen3-VL-4B-Instruct
-
-chat_template: qwen2_vl  # same as qwen2-vl
-```
-
-### SmolVLM2 {#sec-smolvlm2}
-
-::: {.callout-tip}
-Please make sure to install `num2words` via `pip3 install num2words==0.5.14`
-:::
-
-```yaml
-base_model: HuggingFaceTB/SmolVLM2-500M-Video-Instruct
-```
-
-### LFM2-VL {#sec-lfm2-vl}
-
-::: {.callout-warning}
-Please uninstall `causal-conv1d` via `pip3 uninstall -y causal-conv1d`
-:::
-
-```yaml
-base_model: LiquidAI/LFM2-VL-450M
-```
-
-### Intern-VL {#sec-intern-vl}
-
-::: {.callout-tip}
-Please make sure to install `timm` via `pip3 install timm==1.0.19`
-:::
-
-```yaml
-base_model: OpenGVLab/InternVL3_5-8B
-```
-
 ## Dataset Format
 
 For multi-modal datasets, we adopt an extended `chat_template` format similar to OpenAI's Message format.
 
 - A message is a list of `role` and `content`.
 - `role` can be `system`, `user`, `assistant`, etc.
-- `content` is a list of `type` and (`text`, `image`, `path`, `url`, `base64`, or `audio`).
+- `content` is a list of `type` and (`text` or `image` or `path` or `url` or `base64`).
-
-### Image
 
 ::: {.callout-note}
 For backwards compatibility:
@@ -230,43 +141,15 @@ For backwards compatibility:
 - If `content` is a string, it will be converted to a list with `type` as `text`.
 :::
 
+::: {.callout-tip}
 For image loading, you can use the following keys within `content` alongside `"type": "image"`:
 
 - `"path": "/path/to/image.jpg"`
 - `"url": "https://example.com/image.jpg"`
 - `"base64": "..."`
 - `"image": PIL.Image`
-
-### Audio
-
-For audio loading, you can use the following keys within `content` alongside `"type": "audio"`:
-
-- `"path": "/path/to/audio.mp3"`
-- `"url": "https://example.com/audio.mp3"`
-- `"audio": np.ndarray`
-
-::: {.callout-tip}
-
-You may need to install `librosa` via `pip3 install librosa==0.11.0`.
-
 :::
 
-### Video
-
-::: {.callout-warning}
-
-This is not well tested at the moment. We welcome contributors!
-
-:::
-
-For video loading, you can use the following keys within `content` alongside `"type": "video"`:
-
-- `"path": "/path/to/video.mp4"`
-- `"url": "https://example.com/video.mp4"`
-- `"video": np.ndarray | list[PIL.Image.Image] | torch.Tensor` (or list of the aforementioned)
-
-### Example
-
 Here is an example of a multi-modal dataset:
 ```json
 [
@@ -295,9 +178,3 @@ Here is an example of a multi-modal dataset:
   }
 ]
 ```
-
-## FAQ
-
-1. `PIL.UnidentifiedImageError: cannot identify image file ...`
-
-`PIL` could not retrieve the file at `url` using `requests`. Please check for typo. One alternative reason is that the request is blocked by the server.

docs/nd_parallelism.qmd

@@ -1,108 +0,0 @@
----
-title: "N-D Parallelism (Beta)"
----
-
-Axolotl enables training models at scale by composing different parallelism techniques. This is essential when:
-
-- A model's weights are too large to fit on a single GPU's memory.
-- A model's activations, especially with very long contexts, are too large for a single GPU.
-- You want to accelerate training by using multiple GPUs or nodes.
-
-or combinations of the above!
-
-## Core Concepts
-
-Parallelism strategies can be combined. The key is understanding how each one divides the workload. PyTorch's `DeviceMesh` is the modern way to manage these combinations, creating a logical grid of your GPUs and assigning different parallel strategies to different dimensions of the grid.
-
-### Data Parallelism {#sec-dp}
-
-Data Parallelism focuses on splitting the global data batch across GPUs.
-
-- Distributed Data Parallel (DDP): The classic approach. The full model is replicated on every GPU. Each GPU processes a different slice of the data batch. Gradients are then averaged across all GPUs after the backward pass to keep the models synchronized. This can substantially improve data throughput compared to single-device training, but requires that each GPU is able to hold the entire model, its gradients, and optimizer states.
-
-- [Fully Sharded Data Parallel (FSDP)](multi-gpu.qmd#fully-sharded-data-parallel-(fsdp)): A highly memory-efficient form of data parallelism (inspired by DeepSpeed's ZeRO). Instead of replicating the model, FSDP shards the model's *parameters, gradients, and optimizer states* across the GPUs in the data-parallel group. During computation, each GPU receives the specific parameters it needs via an `all_gather` operation just before they are used, and they can be discarded immediately after (`reshard-after-forward`).
-    - FSDP maps to ZeRO stages:
-        - ZeRO-2 (`reshard_after_forward=False`): Shards gradients and optimizer states. Model weights are replicated on each GPU.
-        - ZeRO-3 (`reshard_after_forward=True`): Shards gradients, optimizer states, AND model parameters. This provides the most memory savings at the cost of more communication (re-gathering parameters for both forward and backward passes).
-
-### [Experimental] Tensor Parallelism (TP) {#sec-tp}
-
-Also known as "horizontal model parallelism," as described in the [Megatron-LM paper](https://arxiv.org/pdf/1909.08053.pdf). Instead of splitting the batch, TP splits the model's layers themselves across GPUs.
-
-- How it works: For a linear layer `Y = XA`, the weight matrix `A` is split column-wise (`A = [A_1, A_2]`). The computation becomes `Y_1 = XA_1` and `Y_2 = XA_2`, which can happen in parallel on different GPUs. The final output `Y` is simply the concatenation of `Y_1` and `Y_2`. Check [this comment](https://github.com/huggingface/transformers/issues/10321#issuecomment-783543530) for more detailed info.
-- Requirement: TP involves frequent, small communications within a forward/backward pass. It requires a very fast interconnect between GPUs (e.g., NVLink) and is typically not recommended across different nodes.
-
-### Context Parallelism (CP) {#sec-cp}
-
-Context Parallelism, also called [Sequence Parallelism](sequence_parallelism.qmd), addresses the memory bottleneck from long sequences. The input sequence itself is split along the sequence length dimension and distributed across GPUs.
-
-- How it works: If you have a sequence of 8192 tokens and a `context_parallel_size` of 4, each GPU will only handle a chunk of 2048 tokens.
-- The Challenge: Attention is not local; every token needs to "attend to" every other token. Splitting the sequence breaks this.
-- The Solution (`ring-flash-attention`): An efficient communication protocol is used. To compute attention for its local sequence chunk, each GPU passes its Key-Value (KV) cache to its neighbor in a "ring." After `N-1` steps, every GPU has seen the KV-cache from all other GPUs, allowing it to compute the correct attention values for its chunk. This is implemented using the highly optimized `flash-attention` kernel at each step.
-
-### Hybrid Sharding Data Parallel (HSDP) {#sec-hsdp}
-
-HSDP is a 2D strategy that intelligently combines FSDP and DDP, typically for multi-node training.
-
-- Intra-Node (within a machine): Use FSDP. This is efficient because GPUs on the same node have fast interconnects (NVLink), making the `all_gather` operations for sharded parameters fast.
-- Inter-Node (across machines): Use DDP. The gradient synchronization between nodes is less frequent than FSDP's parameter gathering, making it a better fit for the slower node-to-node network (e.g., Ethernet/Infiniband).
-- Example: With 2 nodes of 8 GPUs each (16 total), you could have `dp_shard_size=8` (FSDP within each node) and `dp_replicate_size=2` (DDP across the two nodes).
-
-## Usage
-
-```yaml
-# FSDP config. See https://docs.axolotl.ai/docs/multi-gpu.html#sec-fsdp
-fsdp_version: 2
-fsdp_config:
-  # ...
-
-# The number of GPUs to shard the model parameters across (FSDP dimension).
-dp_shard_size: 4
-
-# The number of times to replicate the sharded model (DDP dimension).
-dp_replicate_size: 2
-
-# Number of GPUs for Tensor Parallelism.
-tensor_parallel_size: 1  # (default is 1, no TP)
-
-# Number of GPUs for Context/Sequence Parallelism.
-context_parallel_size: 1 # (default is 1, no CP)
-```
-
-Note: We recommend FSDP. DeepSpeed is only compatible with `tensor_parallel_size`.
-
-## Examples
-
-::: {.callout-tip}
-See our example configs [here](https://github.com/axolotl-ai-cloud/axolotl/tree/main/examples/distributed-parallel).
-:::
-
-1.  HSDP on 2 nodes with 4 GPUs each (8 GPUs total):
-    - You want FSDP within each node and DDP across nodes.
-    - Set `dp_shard_size: 4` and `dp_replicate_size: 2`.
-
-2.  FSDP + TP on a single 8-GPU node:
-    - You want to split the model across 4 GPUs using FSDP, and further split each layer across 2 GPUs with TP.
-    - Set `dp_shard_size: 4` and `tensor_parallel_size: 2`.
-
-3.  FSDP + CP on a single 8-GPU node for long context:
-    - You want to shard the model across all 8 GPUs and also split the sequence length across all 8 GPUs.
-    - Set `dp_shard_size: 8` and `context_parallel_size: 8`. Note: this means the data parallel group and context parallel group are the same. A more common setup might be to shard across a smaller group.
-
-## Support Matrix
-
-This matrix describes how different parallelism methods can be combined in Axolotl.
-
-| Combination | `dp_replicate_size` | `dp_shard_size` | `tp_size` | `cp_size` | Status & Notes |
-| --- | :---: | :---: |:---:|:---:|---|
-| **FSDP** (ZeRO-3) | 1 | >1 | 1 | 1 | ✅ Fully supported. Shards model across all GPUs. |
-| **HSDP** | >1 | >1 | 1 | 1 | ✅ Fully supported. FSDP intra-node, DDP inter-node. |
-| **FSDP + TP** | 1 | >1 | >1 | 1 | ✅ **2D Parallelism**. Shards the model across a `dp_shard` group, and TP-splits layers within the `tp` group. |
-| **HSDP + TP** | >1 | >1 | >1 | 1 | ✅ **3D Parallelism**. A powerful but complex combination. |
-| **FSDP + CP** | 1 | >1 | 1 | >1 | ✅ **2D Parallelism**. Combines FSDP with context parallelism. |
-| **FSDP + TP + CP**| 1 | >1 | >1| >1| ✅ **3D Parallelism**. Another advanced combination. |
-| DDP + TP/CP | >1 | 1 | >1 | >1 | ❌ **Not Supported**. The `ParallelismConfig` explicitly prevents this, as composing pure DDP with TP or CP is currently not supported. You should use FSDP + TP/CP instead (`dp_shard_size > 1`). |
-| Just TP / CP | 1 | 1 | >1 | >1 | ✅ Supported. Useful for inference or when the model fits on one GPU but context is too long. |
-
-- `tp_size` refers to `tensor_parallel_size`
-- `cp_size` refers to `context_parallel_size`

docs/optimizations.qmd

@@ -1,133 +0,0 @@
----
-title: Optimizations Guide
-description: A guide to the performance and memory optimizations available in Axolotl.
----
-
-Axolotl includes numerous optimizations to speed up training, reduce memory usage, and handle large models.
-
-This guide provides a high-level overview and directs you to the detailed documentation for each feature.
-
-## Speed Optimizations
-
-These optimizations focus on increasing training throughput and reducing total training time.
-
-### Sample Packing
-
-Improves GPU utilization by combining multiple short sequences into a single packed sequence for training. This requires enabling one of the [attention](#attention-implementations) implementations below.
-
-- **Config:** `sample_packing: true`
-- **Learn more:** [Sample Packing](multipack.qmd)
-
-### Attention Implementations
-
-Using an optimized attention implementation is critical for training speed.
-
-- **[Flash Attention 2](https://github.com/Dao-AILab/flash-attention)**: `flash_attention: true`. **(Recommended)** The industry standard for fast attention on modern GPUs. Requires Ampere or higher. For AMD, check [AMD Support](https://github.com/Dao-AILab/flash-attention?tab=readme-ov-file#amd-rocm-support).
-- **[Flex Attention](https://pytorch.org/blog/flexattention/)**: `flex_attention: true`.
-- **[SDP Attention](https://docs.pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html)**: `sdp_attention: true`. PyTorch's native implementation.
-- **[Xformers](https://github.com/facebookresearch/xformers)**: `xformers_attention: true`. Works with FP16.
-
-*Note: You should only enable one attention backend.*
-
-### LoRA Optimizations
-
-Leverages optimized kernels to accelerate LoRA training and reduce memory usage.
-
-- **Learn more:** [LoRA Optimizations Documentation](lora_optims.qmd)
-
-## Memory Optimizations
-
-These techniques help you fit larger models or use bigger batch sizes on your existing hardware.
-
-### Parameter Efficient Finetuning (LoRA & QLoRA)
-
-Drastically reduces memory by training a small set of "adapter" parameters instead of the full model. This is the most common and effective memory-saving technique.
-
-- Examples: Find configs with `lora` or `qlora` in the [examples directory](https://github.com/axolotl-ai-cloud/axolotl/tree/main/examples/llama-3).
-- Config Reference: See `adapter`, `load_in_4bit`, and `load_in_8bit` in the [Configuration Reference](config-reference.qmd).
-
-### Gradient Checkpointing & Activation Offloading
-
-These techniques save VRAM by changing how activations are handled.
-
-- Gradient Checkpointing: re-computes activations during the backward pass, trading compute time for VRAM.
-- Activation Offloading: moves activations to CPU RAM or disk, trading I/O overhead for VRAM.
-- Learn more: [Gradient Checkpointing and Offloading Docs](gradient_checkpointing.qmd)
-
-### Cut Cross Entropy (CCE)
-
-Reduces VRAM usage by using an optimized cross-entropy loss calculation.
-
-- **Learn more:** [Custom Integrations - CCE](custom_integrations.qmd#cut-cross-entropy)
-
-### Liger Kernels
-
-Provides efficient Triton kernels to improve training speed and reduce memory usage.
-
-- **Learn more:** [Custom Integrations - Liger Kernels](custom_integrations.qmd#liger-kernels)
-
-## Long Context Models
-
-Techniques to train models on sequences longer than their original context window.
-
-### RoPE Scaling
-
-Extends a model's context window by interpolating its Rotary Position Embeddings.
-
-- **Config:** Pass the `rope_scaling` config under the `overrides_of_model_config: `. To learn how to set RoPE, check the respective model config.
-
-### Sequence Parallelism
-
-Splits long sequences across multiple GPUs, enabling training with sequence lengths that would not fit on a single device.
-
-- **Learn more:** [Sequence Parallelism Documentation](sequence_parallelism.qmd)
-
-### Artic Long Sequence Training (ALST)
-
-ALST is a recipe that combines several techniques to train long-context models efficiently. It typically involves:
-
-- TiledMLP to reduce memory usage in MLP layers.
-- Tiled Loss functions (like [CCE](#cut-cross-entropy-(cce) or [Liger](#liger-kernels)).
-- Activation Offloading to CPU.
-
-- Example: [ALST Example Configuration](https://github.com/axolotl-ai-cloud/axolotl/tree/main/examples/alst)
-
-## Large Models (Distributed Training)
-
-To train models that don't fit on a single GPU, you'll need to use a distributed training strategy like FSDP or DeepSpeed. These frameworks shard the model weights, gradients, and optimizer states across multiple GPUs and nodes.
-
-- **Learn more:** [Multi-GPU Guide](multi-gpu.qmd)
-- **Learn more:** [Multi-Node Guide](multi-node.qmd)
-
-### N-D Parallelism (Beta)
-
-For advanced scaling, Axolotl allows you to compose different parallelism techniques (e.g., Data, Tensor, Sequence Parallelism). This is a powerful approach to train an extremely large model by overcoming multiple bottlenecks at once.
-
-- **Learn more:** [N-D Parallelism Guide](nd_parallelism.qmd)
-
-
-## Quantization
-
-Techniques to reduce the precision of model weights for memory savings.
-
-### 4-bit Training (QLoRA)
-
-The recommended approach for quantization-based training. It loads the base model in 4-bit using `bitsandbytes` and then trains QLoRA adapters. See [Adapter Finetuning](#adapter-finetuning-lora-qlora) for details.
-
-### FP8 Training
-
-Enables training with 8-bit floating point precision on supported hardware (e.g., NVIDIA Hopper series GPUs) for significant speed and memory gains.
-
-- **Example:** [Llama 3 FP8 FSDP Example](https://github.com/axolotl-ai-cloud/axolotl/blob/main/examples/llama-3/3b-fp8-fsdp2.yaml)
-
-### Quantization Aware Training (QAT)
-
-Simulates quantization effects during training, helping the model adapt and potentially improving the final accuracy of the quantized model.
-
-- **Learn more:** [QAT Documentation](qat.qmd)
-
-### GPTQ
-
-Allows you to finetune LoRA adapters on top of a model that has already been quantized using the GPTQ method.
-
-- **Example:** [GPTQ LoRA Example](https://github.com/axolotl-ai-cloud/axolotl/blob/main/examples/llama-2/gptq-lora.yml)

docs/optimizers.qmd

@@ -1,129 +0,0 @@
----
-title: Optimizers
-description: Configuring optimizers
----
-
-## Overview
-
-Axolotl supports all optimizers supported by [transformers OptimizerNames](https://github.com/huggingface/transformers/blob/51f94ea06d19a6308c61bbb4dc97c40aabd12bad/src/transformers/training_args.py#L142-L187)
-
-Here is a list of optimizers supported by transformers as of `v4.54.0`:
-
-- `adamw_torch`
-- `adamw_torch_fused`
-- `adamw_torch_xla`
-- `adamw_torch_npu_fused`
-- `adamw_apex_fused`
-- `adafactor`
-- `adamw_anyprecision`
-- `adamw_torch_4bit`
-- `adamw_torch_8bit`
-- `ademamix`
-- `sgd`
-- `adagrad`
-- `adamw_bnb_8bit`
-- `adamw_8bit`  # alias for adamw_bnb_8bit
-- `ademamix_8bit`
-- `lion_8bit`
-- `lion_32bit`
-- `paged_adamw_32bit`
-- `paged_adamw_8bit`
-- `paged_ademamix_32bit`
-- `paged_ademamix_8bit`
-- `paged_lion_32bit`
-- `paged_lion_8bit`
-- `rmsprop`
-- `rmsprop_bnb`
-- `rmsprop_bnb_8bit`
-- `rmsprop_bnb_32bit`
-- `galore_adamw`
-- `galore_adamw_8bit`
-- `galore_adafactor`
-- `galore_adamw_layerwise`
-- `galore_adamw_8bit_layerwise`
-- `galore_adafactor_layerwise`
-- `lomo`
-- `adalomo`
-- `grokadamw`
-- `schedule_free_radam`
-- `schedule_free_adamw`
-- `schedule_free_sgd`
-- `apollo_adamw`
-- `apollo_adamw_layerwise`
-- `stable_adamw`
-
-
-## Custom Optimizers
-
-Enable custom optimizers by passing a string to the `optimizer` argument. Each optimizer will receive beta and epsilon args, however, some may accept additional args which are detailed below.
-
-### optimi_adamw
-
-```yaml
-optimizer: optimi_adamw
-```
-
-### ao_adamw_4bit
-
-Deprecated: Please use `adamw_torch_4bit`.
-
-### ao_adamw_8bit
-
-Deprecated: Please use `adamw_torch_8bit`.
-
-### ao_adamw_fp8
-
-
-```yaml
-optimizer: ao_adamw_fp8
-```
-
-### adopt_adamw
-
-GitHub: [https://github.com/iShohei220/adopt](https://github.com/iShohei220/adopt)
-Paper: [https://arxiv.org/abs/2411.02853](https://arxiv.org/abs/2411.02853)
-
-```yaml
-optimizer: adopt_adamw
-```
-
-### came_pytorch
-
-GitHub: [https://github.com/yangluo7/CAME/tree/master](https://github.com/yangluo7/CAME/tree/master)
-Paper: [https://arxiv.org/abs/2307.02047](https://arxiv.org/abs/2307.02047)
-
-```yaml
-optimizer: came_pytorch
-
-# optional args (defaults below)
-adam_beta1: 0.9
-adam_beta2: 0.999
-adam_beta3: 0.9999
-adam_epsilon: 1e-30
-adam_epsilon2: 1e-16
-```
-
-### muon
-
-Blog: [https://kellerjordan.github.io/posts/muon/](https://kellerjordan.github.io/posts/muon/)
-Paper: [https://arxiv.org/abs/2502.16982v1](https://arxiv.org/abs/2502.16982v1)
-
-```yaml
-optimizer: muon
-```
-
-### dion
-
-Microsoft's Dion (DIstributed OrthoNormalization) optimizer is a scalable and communication-efficient
-orthonormalizing optimizer that uses low-rank approximations to reduce gradient communication.
-
-GitHub: [https://github.com/microsoft/dion](https://github.com/microsoft/dion)
-Paper: [https://arxiv.org/pdf/2504.05295](https://arxiv.org/pdf/2504.05295)
-Note: Implementation written for PyTorch 2.7+ for DTensor
-
-```yaml
-optimizer: dion
-dion_lr: 0.01
-dion_momentum: 0.95
-lr: 0.00001  # learning rate for embeddings and parameters that fallback to AdamW
-```

docs/qat.qmd

@@ -23,18 +23,10 @@ To enable QAT in axolotl, add the following to your configuration file:
 
 ```yaml
 qat:
-  activation_dtype: # Optional[str] = "int8". Fake quantization layout to use for activation quantization. Valid options are "int4", "int8", "float8"
+  activation_dtype: # Optional[str] = "int8". Fake quantization layout to use for activation quantization. Valid options are "int4" and "int8"
-  weight_dtype: # Optional[str] = "int8". Fake quantization layout to use for weight quantization. Valid options are "int4", "fp8", and "nvfp4".
+  weight_dtype: # Optional[str] = "int8". Fake quantization layout to use for weight quantization. Valid options are "int4" and "int8"
   group_size: # Optional[int] = 32. The number of elements in each group for per-group fake quantization
   fake_quant_after_n_steps: # Optional[int] = None. The number of steps to apply fake quantization after
 ```
 
-We support the following quantization schemas:
-
-- `Int4WeightOnly` (requires the `fbgemm-gpu` extra when installing Axolotl)
-- `Int8DynamicActivationInt4Weight`
-- `Float8DynamicActivationFloat8Weight`
-- `Float8DynamicActivationInt4Weight`
-- `NVFP4`
-
 Once you have finished training, you must quantize your model by using the same quantization configuration which you used to train the model with. You can use the [`quantize`](./quantize.qmd) command to do this.

docs/quantize.qmd

@@ -22,8 +22,8 @@ Quantization is configured using the `quantization` key in your configuration fi
 ```yaml
 base_model: # The path to the model to quantize.
 quantization:
-  activation_dtype: # Optional[str] = "int8". Fake quantization layout to use for activation quantization. Valid options are "int4", "int8", "float8"
+  weight_dtype: # Optional[str] = "int8". Fake quantization layout to use for weight quantization. Valid options are uintX for X in [1, 2, 3, 4, 5, 6, 7], or int4, or int8
-  weight_dtype: # Optional[str] = "int8". Fake quantization layout to use for weight quantization. Valid options are "int4", "fp8", and "nvfp4".
+  activation_dtype: # Optional[str] = "int8". Fake quantization layout to use for activation quantization. Valid options are "int4" and "int8"
   group_size: # Optional[int] = 32. The number of elements in each group for per-group fake quantization
   quantize_embedding: # Optional[bool] = False. Whether to quantize the embedding layer.
 
@@ -39,8 +39,9 @@ you used to train the model:
 # qat.yml
 qat:
   activation_dtype: int8
-  weight_dtype: int4
+  weight_dtype: int8
   group_size: 256
+  quantize_embedding: true
 
 output_dir: # The path to the output directory used during training where the final checkpoint has been saved.
 ```
@@ -50,11 +51,3 @@ axolotl quantize qat.yml
 ```
 
 This ensures that an identical quantization configuration is used to quantize the model as was used to train it.
-
-
-::: {.callout-note}
-
-If you have configured pushing to hub with `hub_model_id`, your model hub name will have the quantization schema appended to it,
-e.g. `axolotl-ai-cloud/qat-nvfp4-llama3B` will become `axolotl-ai-cloud/qat-nvfp4-llama3B-nvfp4w`
-
-:::

docs/reward_modelling.qmd

@@ -11,7 +11,6 @@ We support the reward modelling techniques supported by `trl`.
 ### (Outcome) Reward Models
 
 Outcome reward models are trained using data which contains preference annotations for an entire interaction between the user and model (e.g. rather than per-turn or per-step).
-For improved training stability, you can use the `center_rewards_coefficient` parameter to encourage mean-zero reward outputs ([see TRL docs](https://huggingface.co/docs/trl/v0.10.1/en/reward_trainer#centering-rewards)).
 
 ```yaml
 base_model: google/gemma-2-2b

docs/rlhf.qmd

@@ -219,21 +219,6 @@ DPO supports the following types with the following dataset format:
 }
 ```
 
-#### chat_template.argilla_chat
-
-```json
-{
-    "chosen": [
-        {"role": "user", "content": "..."},
-        {"role": "assistant", "content": "..."}
-    ],
-    "rejected": [
-        {"role": "user", "content": "..."},
-        {"role": "assistant", "content": "..."}
-    ]
-}
-```
-
 #### chat_template.default
 
 ```yaml
@@ -597,116 +582,6 @@ To see other examples of custom reward functions, please see [TRL GRPO Docs](htt
 
 To see all configs, please see [TRLConfig](https://github.com/axolotl-ai-cloud/axolotl/blob/v0.9.2/src/axolotl/utils/schemas/trl.py).
 
-#### OpenEnv Rollout Functions
-
-GRPO supports custom rollout functions for OpenEnv-style environments, enabling interactive tasks like web browsing, code execution, or tool use. This allows you to implement custom generation logic that interacts with external environments.
-
-For example, to implement a simple math-solving environment with step-by-step verification:
-
-```python
-# math_env.py
-import re
-
-def math_solver_rollout(model, processing_class, prompts, generation_config=None):
-    """
-    Custom rollout function that generates step-by-step math solutions.
-
-    Args:
-        model: The language model
-        processing_class: The tokenizer/processing_class
-        prompts: List of prompt dicts (with 'messages' key for chat format)
-        generation_config: Optional generation configuration
-
-    Returns:
-        List of completion strings
-    """
-    completions = []
-
-    for prompt in prompts:
-        # Apply chat template to prompt
-        messages = prompt.get("messages", [])
-        formatted_prompt = processing_class.apply_chat_template(
-            messages, processing_class=False, add_generation_prompt=True
-        )
-
-        # Generate step-by-step solution
-        full_response = ""
-        for step in range(5):  # Max 5 reasoning steps
-            current_input = formatted_prompt + full_response + "\nNext step:"
-            inputs = processing_class(current_input, return_tensors="pt").to(model.device)
-
-            outputs = model.generate(
-                **inputs,
-                max_new_tokens=100,
-                generation_config=generation_config,
-            )
-            step_text = processing_class.decode(
-                outputs[0][inputs.input_ids.shape[1]:],
-                skip_special_tokens=True
-            )
-
-            # Check if solution is complete
-            if "FINAL ANSWER:" in step_text:
-                full_response += step_text
-                break
-            full_response += step_text + "\n"
-
-        completions.append(full_response)
-
-    return completions
-
-def math_reward(prompts, completions, answers, **kwargs):
-    """Reward function that checks mathematical correctness"""
-    rewards = []
-    for completion, correct_answer in zip(completions, answers):
-        # Extract predicted answer
-        match = re.search(r"FINAL ANSWER:\s*(.+)", completion)
-        predicted = match.group(1).strip() if match else ""
-
-        # Compare with correct answer
-        reward = 1.0 if predicted == str(correct_answer) else 0.0
-        rewards.append(reward)
-
-    return rewards
-
-def math_transform(cfg, *args, **kwargs):
-    """Transform dataset to GRPO format with answer field"""
-    def transform_fn(example, processing_class=None):
-        return {
-            "prompt": [{"role": "user", "content": example["question"]}],
-            "answer": str(example["answer"]),
-        }
-    return transform_fn, {"remove_columns": ["question"]}
-```
-
-```yaml
-rl: grpo
-
-trl:
-  beta: 0.001
-  max_completion_length: 512
-  num_generations: 4
-  rollout_func: "math_env.math_solver_rollout"  # Custom rollout function
-  reward_funcs: ["math_env.math_reward"]
-  reward_weights: [1.0]
-
-datasets:
-  - path: openai/gsm8k
-    name: main
-    type: math_env.math_transform
-```
-
-The `rollout_func` parameter accepts a fully qualified name (e.g., `module_name.function_name`) that points to a callable function in your local directory. The function receives:
-
-- `model`: The language model
-- `processing_class`: The tokenizer/processing class
-- `prompts`: List of prompt dictionaries
-- `generation_config` (optional): Generation configuration
-
-And should return a list of completion strings.
-
-For more OpenEnv examples, see [TRL OpenEnv Documentation](https://huggingface.co/docs/trl/main/en/openenv).
-
 #### GRPO with DAPO/Dr. GRPO loss
 
 The DAPO paper and subsequently Dr. GRPO paper proposed an alternative loss function for GRPO to remediate the penalty in longer responses.

docs/scripts/examples-allowlist.yml

@@ -1,90 +0,0 @@
-examples:
-  # December 2025
-  - name: kimi-linear
-    title: Kimi Linear
-  - name: plano
-    title: Plano Orchestrator
-  - name: mimo
-    title: MiMo
-  - name: internvl3_5
-    title: InternVL 3.5
-
-  # AllenAI
-  - name: olmo3
-    title: OLMo 3
-
-  # ArceeAI
-  - name: trinity
-    title: Trinity
-  - name: arcee
-    title: Arcee AFM
-
-  # MistralAI
-  - name: ministral3/think
-    title: Ministral 3 Thinking
-  - name: ministral3/vision
-    title: Ministral 3 Vision
-  - name: magistral/think
-    title: Magistral Thinking
-  - name: magistral/vision
-    title: Magistral Vision
-  - name: ministral
-    title: Ministral
-  - name: mistral-small
-    title: Mistral Small 3.1/3.2
-  - name: voxtral
-    title: Voxtral
-  - name: devstral
-    title: Devstral
-  - name: mistral
-    title: Mistral 7B
-
-  # Meta
-  - name: llama-4
-    title: Llama 4
-  - name: llama-2
-    title: Llama 2
-
-  # Alibaba
-  - name: qwen3-next
-    title: Qwen 3 Next
-  - name: qwen3
-    title: Qwen 3
-
-  # Google
-  - name: gemma3n
-    title: Gemma 3n
-
-  # Swiss AI
-  - name: apertus
-    title: Apertus
-
-  # GPT-OSS
-  - name: gpt-oss
-    title: GPT-OSS
-  - name: seed-oss
-    title: Seed-OSS
-
-  # Microsoft
-  - name: phi
-    title: Phi
-
-  # SmolVLM
-  - name: smolvlm2
-    title: SmolVLM 2
-
-  # IBM
-  - name: granite4
-    title: Granite 4
-
-  # LiquidAI
-  - name: LiquidAI
-    title: Liquid Foundation Models 2
-
-  # Other
-  - name: hunyuan
-    title: Hunyuan
-  - name: jamba
-    title: Jamba
-  - name: orpheus
-    title: Orpheus

docs/scripts/generate_config_docs.py

@@ -47,6 +47,7 @@ class QuartoGenerator:
         """Check if a type is a Pydantic BaseModel."""
         return inspect.isclass(type_obj) and issubclass(type_obj, BaseModel)
 
+    # pylint: disable=too-many-return-statements
     def _extract_nested_type(self, field_type) -> Any:
         """Extract the actual type from complex type annotations."""
         # Handle Annotated types (Python 3.9+)
@@ -123,6 +124,7 @@ class QuartoGenerator:
 
         return field_type
 
+    # pylint: disable=too-many-return-statements
     def _extract_all_pydantic_models_from_type(
         self, field_type
     ) -> list[type[BaseModel]]:
@@ -316,6 +318,7 @@ class QuartoGenerator:
 
         return all_groups
 
+    # pylint: disable=too-many-return-statements
     def _extract_field_groups_from_source(
         self, model_class: type[BaseModel]
     ) -> list[dict]:
@@ -500,7 +503,7 @@ class QuartoGenerator:
                     nested_schema = nested_model.model_json_schema()
                     nested_properties = nested_schema.get("properties", {})
                     nested_required = nested_schema.get("required", [])
-                except Exception:
+                except Exception:  # pylint: disable=broad-exception-caught
                     # Fallback: use model fields directly
                     nested_properties = {}
                     nested_required = []
@@ -604,7 +607,7 @@ class QuartoGenerator:
             schema = model_class.model_json_schema()
             properties = schema.get("properties", {})
             required = schema.get("required", [])
-        except Exception as e:
+        except Exception as e:  # pylint: disable=broad-exception-caught
             print(
                 f"Warning: Could not generate JSON schema ({e}). Using model fields instead."
             )

docs/scripts/generate_examples_docs.py

@@ -1,424 +0,0 @@
-"""
-auto generate example docs from allowlist
-"""
-
-import re
-import shutil
-import sys
-from pathlib import Path
-
-import yaml
-
-# Paths
-THIS = Path(__file__).resolve()
-ROOT = THIS.parents[2]  # repo root (docs/scripts -> docs -> ROOT)
-EXAMPLES_DIR = ROOT / "examples"
-OUTPUT_DIR = ROOT / "docs" / "models"
-ALLOWLIST_YML = THIS.parent / "examples-allowlist.yml"
-
-
-def slugify(name: str) -> str:
-    """Convert a name to a slug (lowercase, hyphens for spaces)."""
-    s = re.sub(r"[^a-zA-Z0-9\s\-]+", "", name.strip())
-    s = re.sub(r"\s+", "-", s).strip("-").lower()
-    return s or "example"
-
-
-def read_allowlist():
-    with open(ALLOWLIST_YML, "r", encoding="utf-8") as f:
-        data = yaml.safe_load(f) or {}
-    items = data.get("examples", [])
-    if not isinstance(items, list):
-        raise ValueError("`examples` must be a list in examples-allowlist.yml")
-    return items
-
-
-def find_readme(folder: Path) -> Path | None:
-    for name in ("README.md", "Readme.md", "readme.md"):
-        p = folder / name
-        if p.exists():
-            return p
-    return None
-
-
-def remove_first_h1(md: str) -> tuple[str, str | None]:
-    """
-    Remove the first H1 from markdown and return (modified_md, h1_title).
-    The H1 is removed since we use the frontmatter title instead.
-    """
-    lines = md.splitlines()
-    result = []
-    h1_title = None
-    skipped_first = False
-
-    for line in lines:
-        if not skipped_first and line.startswith("# "):
-            h1_title = line[2:].strip()
-            skipped_first = True
-            continue
-        result.append(line)
-
-    return "\n".join(result), h1_title
-
-
-IMG_RE = re.compile(r"!\[[^\]]*\]\(([^)]+)\)")
-LINK_RE = re.compile(r"\[([^\]]+)\]\(([^)]+)\)")
-
-
-def rewrite_and_copy_assets(md: str, src_dir: Path, dest_assets_root: Path) -> str:
-    """
-    Copy local image assets referenced in markdown to
-    docs/examples/assets/... and rewrite the links.
-    """
-    dest_assets = dest_assets_root / "assets"
-
-    def repl(m):
-        url = m.group(1).strip()
-        if re.match(r"^(https?:)?//", url):
-            return m.group(0)  # leave remote URLs
-        src_path = (src_dir / url).resolve()
-        if not src_path.exists():
-            return m.group(0)  # leave as-is if not found
-        rel = src_path.relative_to(src_dir)
-        # Create a unique asset path based on source directory name
-        asset_name = src_dir.name.replace("/", "-")
-        dest_path = dest_assets / asset_name / rel
-        dest_path.parent.mkdir(parents=True, exist_ok=True)
-        shutil.copy2(src_path, dest_path)
-        new_rel = f"assets/{asset_name}/{rel.as_posix()}"
-        return m.group(0).replace(url, new_rel)
-
-    return IMG_RE.sub(repl, md)
-
-
-def rewrite_readme_links(
-    md: str,
-    src_dir: Path,
-    examples_dir: Path,
-    parent_index_only: set,
-    current_src_path: str,
-    allowlist_entries: set,
-    current_output_path: str,
-) -> str:
-    """
-    Rewrite links between README.md files to point to the correct .qmd files.
-    """
-
-    def repl(m):
-        text = m.group(1)
-        url = m.group(2).strip()
-
-        # Skip remote URLs and anchor links
-        if re.match(r"^(https?:)?//", url) or url.startswith("#"):
-            return m.group(0)
-
-        # Skip non-markdown files
-        if not url.lower().endswith(".md"):
-            return m.group(0)
-
-        # Resolve the target path
-        try:
-            target_path = (src_dir / url).resolve()
-
-            # Check if target is outside examples_dir
-            try:
-                rel_path = target_path.relative_to(examples_dir)
-            except ValueError:
-                # Target is outside examples_dir, leave as-is
-                return m.group(0)
-
-            parts = list(rel_path.parts)
-
-            # Determine the output path for the target
-            if len(parts) > 0 and parts[-1].lower() in ("readme.md", "readme"):
-                # This is a README link
-                if len(parts) == 1:
-                    # Link to root README -> index.qmd
-                    target_output = "index.qmd"
-                elif len(parts) == 2:
-                    if parts[0] == ".":
-                        # Current directory README
-                        target_output = "index.qmd"
-                    else:
-                        # subdir/README.md
-                        parent_dir = parts[0]
-                        if parent_dir in parent_index_only:
-                            target_output = f"{parent_dir}/index.qmd"
-                        else:
-                            target_output = f"{parent_dir}.qmd"
-                else:
-                    # Deeper nesting: parent/subdir/README.md
-                    # Build the full path like "parent/subdir"
-                    full_path = "/".join(parts[:-1])  # Remove README.md
-                    # Check if this exact path is in allowlist
-                    if full_path in allowlist_entries:
-                        # This is a sub-entry with its own entry -> use .qmd
-                        target_output = f"{full_path}.qmd"
-                    elif parts[0] == ".":
-                        # ./subdir/README.md -> check if subdir has own entry
-                        subdir = parts[1]
-                        if subdir in parent_index_only:
-                            target_output = f"{subdir}/index.qmd"
-                        else:
-                            target_output = f"{subdir}.qmd"
-                    else:
-                        # parent/subdir where parent doesn't have own entry
-                        target_output = f"{full_path}/index.qmd"
-            else:
-                # Regular .md file -> convert to .qmd, keep path structure
-                target_output = "/".join(parts)[:-2] + "qmd"
-
-            # Compute relative path from current output file to target
-            current_parts = current_output_path.split("/")
-            target_parts = target_output.split("/")
-
-            # Special case: if current is a subdir file and target is a single-component file at root
-            # Example: current="magistral/vision", target="magistral.qmd"
-            if len(current_parts) > 1 and len(target_parts) == 1:
-                # Current is in subdir, target is at root level
-                # Go up to root: ../ for each level
-                up_count = len(current_parts) - 1
-                rel_parts = [".."] * up_count + [target_parts[0]]
-                new_url = "/".join(rel_parts)
-            else:
-                # Find common prefix
-                i = 0
-                while (
-                    i < min(len(current_parts) - 1, len(target_parts))
-                    and current_parts[i] == target_parts[i]
-                ):
-                    i += 1
-
-                # Build relative path: go up (../) then down to target
-                up_count = len(current_parts) - 1 - i
-                rel_parts = [".."] * up_count + target_parts[i:]
-
-                if not rel_parts or rel_parts == [".."]:
-                    # Points to same directory or parent
-                    new_url = "/".join(rel_parts) if rel_parts else "."
-                else:
-                    new_url = "/".join(rel_parts)
-
-            return f"[{text}]({new_url})"
-        except (ValueError, IndexError):
-            return m.group(0)
-
-    return LINK_RE.sub(repl, md)
-
-
-def write_qmd(out_path: Path, title: str, body_md: str):
-    out_path.parent.mkdir(parents=True, exist_ok=True)
-    fm = f"---\ntitle: {title!r}\nexecute:\n  eval: false\nformat:\n  html:\n    toc: true\n---\n\n"
-    out_path.write_text(fm + body_md, encoding="utf-8")
-
-
-def update_quarto_yml(generated: list[tuple[str, str, str]]):
-    """
-    Update _quarto.yml with the generated example files in the correct order.
-    This keeps the sidebar in sync with the allowlist.
-
-    Model Guides is now nested under "Getting Started" section.
-    Creates nested sections for models with sub-entries (e.g., magistral, ministral3).
-    Parent pages are now flat files (e.g., ministral3.qmd) with sub-pages in subdirs.
-    """
-    quarto_yml = ROOT / "_quarto.yml"
-    if not quarto_yml.exists():
-        print(f"[WARN] {quarto_yml} not found, skipping update", file=sys.stderr)
-        return
-
-    content = quarto_yml.read_text(encoding="utf-8")
-
-    # First pass: find all parents that have sub-entries
-    parents_with_subs = set()
-    for path, _name, _title in generated:
-        if "/" in path:
-            parent = path.split("/")[0]
-            parents_with_subs.add(parent)
-
-    # Build the YAML contents while preserving allowlist order
-    lines = []
-    processed_sections = set()
-
-    for path, _name, title in generated:
-        # Check if this is a parent page that has sub-pages
-        if path in parents_with_subs:
-            # This is a parent page with sub-pages - create a nested section
-            if path not in processed_sections:
-                processed_sections.add(path)
-                section_title = (
-                    title or path.replace("-", " ").replace("_", " ").title()
-                )
-                lines.append(f'                - section: "{section_title}"')
-                lines.append("                  contents:")
-                # Add the parent page first
-                lines.append(f"                    - docs/models/{path}.qmd")
-                # Then add all sub-pages
-                for sub_path, _sub_name, _sub_title in generated:
-                    if "/" in sub_path and sub_path.split("/")[0] == path:
-                        lines.append(
-                            f"                    - docs/models/{sub_path}.qmd"
-                        )
-        elif "/" not in path:
-            # This is a flat item with no sub-pages
-            # Skip if it was already included as part of a parent section
-            if path not in processed_sections:
-                lines.append(f"                - docs/models/{path}.qmd")
-
-    yaml_content = "\n".join(lines) + "\n"
-
-    # Pattern to match only the Model Guides contents, stopping at the next item
-    # in Getting Started (lines starting with 12 spaces: same level as the section)
-    pattern = r'(            - section: "Model Guides"\n              contents:)([^\n]*|.*?)(?=\n            - |\n        - section:|\n\nformat:)'
-
-    def replacement(match):
-        prefix = match.group(1)
-        return prefix + "\n" + yaml_content
-
-    new_content = re.sub(pattern, replacement, content, flags=re.DOTALL)
-
-    if new_content != content:
-        quarto_yml.write_text(new_content, encoding="utf-8")
-        print(f"Updated {quarto_yml}")
-    else:
-        print(f"No changes needed for {quarto_yml}")
-
-
-def main():
-    allow = read_allowlist()
-    if not EXAMPLES_DIR.exists():
-        print(f"[WARN] {EXAMPLES_DIR} not found", file=sys.stderr)
-        return
-
-    (OUTPUT_DIR / "assets").mkdir(parents=True, exist_ok=True)
-
-    # First pass: identify which parents have their own entry vs only sub-entries
-    parent_entries = set()  # Parents that have their own entry
-    parent_with_subs = set()  # Parents that have sub-entries
-    allowlist_entries = set()  # All entries in allowlist
-
-    for item in allow:
-        if isinstance(item, str):
-            name = item
-        else:
-            name = item.get("name")
-
-        allowlist_entries.add(name)
-
-        if "/" in name:
-            parent = name.split("/")[0]
-            parent_with_subs.add(parent)
-        else:
-            parent_entries.add(name)
-
-    # Parents with subs that DON'T have their own entry -> use index.qmd
-    parent_index_only = parent_with_subs - parent_entries
-
-    generated = []
-    seen_dirs = set()  # Track which parent directories we've created index for
-
-    for item in allow:
-        if isinstance(item, str):
-            name = item
-            title = None
-        else:
-            name = item.get("name")
-            title = item.get("title")
-
-        if not name:
-            print(f"[WARN] Skipping item without name: {item}", file=sys.stderr)
-            continue
-
-        src_dir = EXAMPLES_DIR / name
-        if not src_dir.exists() or not src_dir.is_dir():
-            print(f"[WARN] Skipping {name} (not a directory)", file=sys.stderr)
-            continue
-
-        readme = find_readme(src_dir)
-        if not readme:
-            print(f"[WARN] Skipping {name} (no README.md)", file=sys.stderr)
-            continue
-
-        md = readme.read_text(encoding="utf-8")
-
-        # Determine output path first (needed for link rewriting)
-        parts = name.split("/")
-        if len(parts) == 1:
-            # Simple case: no subdirectory
-            out_path = OUTPUT_DIR / f"{parts[0]}.qmd"
-            sidebar_path = parts[0]
-        else:
-            # Has subdirectory: e.g., magistral/think
-            parent = parts[0]
-            child = "-".join(parts[1:])  # handle nested subdirs
-            out_path = OUTPUT_DIR / parent / f"{child}.qmd"
-            sidebar_path = f"{parent}/{child}"
-
-        # Remove the first H1 (we use frontmatter title instead)
-        md, _ = remove_first_h1(md)
-        # Rewrite links between README files
-        md = rewrite_readme_links(
-            md,
-            src_dir,
-            EXAMPLES_DIR,
-            parent_index_only,
-            name,
-            allowlist_entries,
-            sidebar_path,
-        )
-        md = rewrite_and_copy_assets(md, src_dir, OUTPUT_DIR)
-
-        # Handle parent page generation for sub-entries
-        if len(parts) > 1:
-            # Has subdirectory: e.g., magistral/think
-            parent = parts[0]
-
-            # Create parent.qmd if not already done and parent doesn't have own entry
-            if parent not in seen_dirs and parent in parent_index_only:
-                parent_readme = find_readme(EXAMPLES_DIR / parent)
-                if parent_readme:
-                    parent_md = parent_readme.read_text(encoding="utf-8")
-                    parent_md, _ = remove_first_h1(parent_md)
-                    parent_md = rewrite_readme_links(
-                        parent_md,
-                        EXAMPLES_DIR / parent,
-                        EXAMPLES_DIR,
-                        parent_index_only,
-                        parent,
-                        allowlist_entries,
-                        parent,
-                    )
-                    parent_md = rewrite_and_copy_assets(
-                        parent_md, EXAMPLES_DIR / parent, OUTPUT_DIR
-                    )
-                    parent_title = parent.replace("-", " ").replace("_", " ").title()
-                    write_qmd(OUTPUT_DIR / f"{parent}.qmd", parent_title, parent_md)
-                    generated.append((parent, parent, parent_title))
-                    seen_dirs.add(parent)
-
-        if not title:
-            title = name.replace("/", " ").replace("-", " ").title()
-
-        write_qmd(out_path, title, md)
-        generated.append((sidebar_path, name, title))
-
-    # Index page - preserve allowlist order
-    if generated:
-        listing = "\n".join(
-            [f"- [{title}]({path}.qmd)" for path, name, title in generated]
-        )
-        index_md = (
-            "# Model Guides\n\nBelow are the curated examples for training various model architectures:\n\n"
-            + listing
-            + "\n"
-        )
-        index_fm = (
-            "---\nexecute:\n  eval: false\nformat:\n  html:\n    toc: true\n---\n\n"
-        )
-        (OUTPUT_DIR / "index.qmd").write_text(index_fm + index_md, encoding="utf-8")
-
-        # Auto-update _quarto.yml to keep sidebar in sync
-        update_quarto_yml(generated)
-
-
-if __name__ == "__main__":
-    main()

docs/sequence_parallelism.qmd

@@ -22,7 +22,7 @@ To enable sequence parallelism, add the following to your configuration file:
 
 ```yaml
 # Set to a divisor (> 1) of the number of GPUs available
-context_parallel_size: 4  # Split sequences across 4 GPUs
+sequence_parallel_degree: 4  # Split sequences across 4 GPUs
 # Optional; strides across the key dimension. Larger values use more memory but should make training faster.
 heads_k_stride: 1
 # Optional; one of "varlen_llama3" or "batch_ring". Defaults to
@@ -30,7 +30,7 @@ heads_k_stride: 1
 ring_attn_func:
 ```
 
-The `context_parallel_size` should be a divisor of the total number of GPUs. For example:
+The `sequence_parallel_degree` should be a divisor of the total number of GPUs. For example:
 
 - With 8 GPUs, valid values would be 2, 4, or 8
 - With 4 GPUs, valid values would be 2 or 4
@@ -66,7 +66,7 @@ sequence_len: 8192
 
 ...
 
-context_parallel_size: 4  # Split each sequence into 4 parts, one per GPU
+sequence_parallel_degree: 4  # Split each sequence into 4 parts, one per GPU
 # Optional; strides across the key dimension. Larger values use more memory but should make training faster.
 heads_k_stride: 1
 # Optional; one of "varlen_llama3" or "batch_ring". Defaults to
@@ -89,12 +89,12 @@ Sequence parallelism is compatible with Axolotl's sample packing functionality.
 
 ## Effect on Batch Size
 
-When using sequence parallelism, your effective global batch size is **divided** by the `context_parallel_size`. This happens because:
+When using sequence parallelism, your effective global batch size is **divided** by the `sequence_parallel_degree`. This happens because:
 
-- Each group of `context_parallel_size` GPUs works on the same batch (just different parts of each sequence)
+- Each group of `sequence_parallel_degree` GPUs works on the same batch (just different parts of each sequence)
 - The number of batches processed per step decreases
 
 For example:
 - With 8 GPUs and no sequence parallelism: 8 different batches processed per step
-- With 8 GPUs and `context_parallel_size=4`: Only 2 different batches processed per step (each split across 4 GPUs)
+- With 8 GPUs and `sequence_parallel_degree=4`: Only 2 different batches processed per step (each split across 4 GPUs)
 - If your per-GPU `micro_batch_size` is 2, the global batch size decreases from 16 to 4

docs/streaming.qmd

@@ -1,120 +0,0 @@
----
-title: Streaming Datasets
-description: How to use streaming mode for large-scale datasets and memory-efficient training
-order: 10
----
-
-Streaming enables memory-efficient training with large datasets by loading data
-incrementally rather than loading the entire dataset into memory at once.
-
-Use streaming when:
-
-- Your dataset is too large to fit in memory (e.g. when you're doing pretraining with massive text corpora)
-- You want to start training immediately without preprocessing the entire dataset
-
-Streaming works with both remote and locally stored datasets!
-
-::: {.callout-note}
-Streaming currently only supports a single dataset. Multi-dataset support will be added soon.
-:::
-
-
-## Configuration
-
-### Basic Streaming
-
-Enable streaming mode by setting the `streaming` flag:
-
-```yaml
-streaming: true
-```
-
-### Pretraining with Streaming
-
-For pretraining tasks, streaming is automatically enabled when using `pretraining_dataset`:
-
-```yaml
-pretraining_dataset:
-  - path: HuggingFaceFW/fineweb-edu
-    type: pretrain
-    text_column: text
-    split: train
-
-# Optionally, enable sample packing
-streaming_multipack_buffer_size: 10000
-sample_packing: true
-```
-
-### SFT with Streaming
-
-For supervised fine-tuning with streaming:
-
-```yaml
-streaming: true
-datasets:
-  - path: tatsu-lab/alpaca
-    type: alpaca
-    split: train
-
-# Optionally, enable sample packing
-streaming_multipack_buffer_size: 10000
-sample_packing: true
-```
-
-## Configuration Options
-
-### `streaming_multipack_buffer_size`
-
-Controls the buffer size for multipack streaming (default: 10,000). This determines how
-many samples are buffered before packing. Larger buffers can improve packing efficiency
-but use more memory.
-
-### `shuffle_merged_datasets`
-
-When enabled, shuffles the streaming dataset using the buffer. This requires additional
-memory for the shuffle buffer.
-
-## Sample Packing with Streaming
-
-Sample packing is supported for streaming datasets. When enabled, multiple samples are
-packed into a single sequence to maximize GPU utilization:
-
-```yaml
-sample_packing: true
-streaming_multipack_buffer_size: 10000
-
-# For SFT: attention is automatically isolated between packed samples
-# For pretraining: control with pretrain_multipack_attn
-pretrain_multipack_attn: true  # prevent cross-attention between packed samples
-```
-
-For more information, see our [documentation](multipack.qmd) on multipacking.
-
-## Important Considerations
-
-### Memory Usage
-
-While streaming reduces memory usage compared to loading entire datasets, you still need
-to consider:
-
-- You can control the memory usage by adjusting `streaming_multipack_buffer_size`
-- Sample packing requires buffering multiple samples
-- Shuffling requires additional memory for the shuffle buffer
-
-### Performance
-
-- Streaming may have slightly higher latency compared to preprocessed datasets, as samples are processed on-the-fly
-- Network speed and disk read speed are important when streaming from remote sources or a local dataset, respectively
-- Consider using `axolotl preprocess` for smaller or more frequently used datasets
-
-### Evaluation Datasets
-
-Evaluation datasets are not streamed to ensure consistent evaluation metrics. They're
-loaded normally even when training uses streaming.
-
-## Examples
-
-See the `examples/streaming/` directory for complete configuration examples:
-
-- `pretrain.yaml`: Pretraining with streaming dataset
-- `sft.yaml`: Supervised fine-tuning with streaming

docs/telemetry.qmd

@@ -1,61 +0,0 @@
----
-title: Telemetry
-description: A description of the telemetry implementation in Axolotl.
----
-
-# Telemetry in Axolotl
-
-Axolotl implements anonymous telemetry to help maintainers understand how the library
-is used and where users encounter issues. This data helps prioritize features, optimize
-performance, and fix bugs.
-
-## Data Collection
-
-We collect:
-
-- System info: OS, Python version, Axolotl version, PyTorch version, Transformers
-version, etc.
-- Hardware info: CPU count, memory, GPU count and models
-- Runtime metrics: Training progress, memory usage, timing information
-- Usage patterns: Models (from a whitelist) and configurations used
-- Error tracking: Stack traces and error messages (sanitized to remove personal
-information)
-
-Personally identifiable information (PII) is not collected.
-
-## Implementation
-
-Telemetry is implemented using PostHog and consists of:
-
-- `axolotl.telemetry.TelemetryManager`: A singleton class that initializes the
-telemetry system and provides methods for tracking events.
-- `axolotl.telemetry.errors.send_errors`: A decorator that captures exceptions and
-sends sanitized stack traces.
-- `axolotl.telemetry.runtime_metrics.RuntimeMetricsTracker`: A class that tracks
-runtime metrics during training.
-- `axolotl.telemetry.callbacks.TelemetryCallback`: A Trainer callback that sends
-runtime metrics telemetry.
-
-The telemetry system will block training startup for 10 seconds to ensure users are
-aware of data collection, unless telemetry is explicitly enabled or disabled.
-
-## Opt-Out Mechanism
-
-Telemetry is **enabled by default** on an opt-out basis. To disable it, set
-`AXOLOTL_DO_NOT_TRACK=1` or `DO_NOT_TRACK=1`.
-
-A warning message will be logged on start to clearly inform users about telemetry.
-We will remove this after some period.
-
-To hide the warning message about telemetry that is displayed on train, etc. startup,
-explicitly set: `AXOLOTL_DO_NOT_TRACK=0` (enable telemetry) or `AXOLOTL_DO_NOT_TRACK=1`
-(explicitly disable telemetry).
-
-## Privacy
-
-- All path-like config information is automatically redacted from telemetry data
-- Model information is only collected for whitelisted organizations
-    - See `axolotl/telemetry/whitelist.yaml` for the set of whitelisted organizations
-- Each run generates a unique anonymous ID
-    - This allows us to link different telemetry events in a single same training run
-- Telemetry is only sent from the main process to avoid duplicate events

examples/LiquidAI/README.md

@@ -1,67 +0,0 @@
-# Finetune Liquid Foundation Models 2 (LFM2) with Axolotl
-
-[Liquid Foundation Models 2 (LFM2)](https://huggingface.co/collections/LiquidAI/lfm2-686d721927015b2ad73eaa38) are a family of small, open-weight models from [Liquid AI](https://www.liquid.ai/) focused on quality, speed, and memory efficiency. Liquid AI released text-only [LFM2](https://huggingface.co/collections/LiquidAI/lfm2-686d721927015b2ad73eaa38) and text+vision [LFM2-VL](https://huggingface.co/collections/LiquidAI/lfm2-vl-68963bbc84a610f7638d5ffa) models.
-
-LFM2 features a new hybrid Liquid architecture with multiplicative gates, short-range convolutions, and grouped query attention, enabling fast training and inference.
-
-This guide shows how to fine-tune both the LFM2 and LFM2-VL models with Axolotl.
-
-Thanks to the team at LiquidAI for giving us early access to prepare for these releases.
-
-## Getting Started
-
-1.  Install Axolotl following the [installation guide](https://docs.axolotl.ai/docs/installation.html).
-
-    Here is an example of how to install from pip:
-    ```bash
-    # Ensure you have a compatible version of Pytorch installed
-    pip3 install packaging setuptools wheel ninja
-    pip3 install --no-build-isolation 'axolotl[flash-attn]>=0.12.0'
-    ```
-
-2.  Run one of the finetuning examples below.
-
-    **LFM2**
-    ```bash
-    # FFT SFT (1x48GB @ 25GiB)
-    axolotl train examples/LiquidAI/lfm2-350m-fft.yaml
-    ```
-
-    **LFM2-VL**
-    ```bash
-    # LoRA SFT (1x48GB @ 2.7GiB)
-    axolotl train examples/LiquidAI/lfm2-vl-lora.yaml
-    ```
-
-    **LFM2-MoE**
-    ```bash
-    pip install git+https://github.com/huggingface/transformers.git@0c9a72e4576fe4c84077f066e585129c97bfd4e6
-
-    # LoRA SFT (1x48GB @ 16.2GiB)
-    axolotl train examples/LiquidAI/lfm2-8b-a1b-lora.yaml
-    ```
-
-### TIPS
-
-- **Installation Error**: If you encounter `ImportError: ... undefined symbol ...` or `ModuleNotFoundError: No module named 'causal_conv1d_cuda'`, the `causal-conv1d` package may have been installed incorrectly. Try uninstalling it:
-  ```bash
-  pip uninstall -y causal-conv1d
-  ```
-
-- **Dataset Loading**: Read more on how to load your own dataset in our [documentation](https://docs.axolotl.ai/docs/dataset_loading.html).
-- **Dataset Formats**:
-  - For LFM2 models, the dataset format follows the OpenAI Messages format as seen [here](https://docs.axolotl.ai/docs/dataset-formats/conversation.html#chat_template).
-  - For LFM2-VL models, Axolotl follows the multi-content Messages format. See our [Multimodal docs](https://docs.axolotl.ai/docs/multimodal.html#dataset-format) for details.
-
-## Optimization Guides
-
-- [Optimizations Guide](https://docs.axolotl.ai/docs/optimizations.html)
-
-## Related Resources
-
-- [LFM2 Blog](https://www.liquid.ai/blog/liquid-foundation-models-v2-our-second-series-of-generative-ai-models)
-- [LFM2-VL Blog](https://www.liquid.ai/blog/lfm2-vl-efficient-vision-language-models)
-- [LFM2-MoE Blog](https://www.liquid.ai/blog/lfm2-8b-a1b-an-efficient-on-device-mixture-of-experts)
-- [Axolotl Docs](https://docs.axolotl.ai)
-- [Axolotl GitHub](https://github.com/axolotl-ai-cloud/axolotl)
-- [Axolotl Discord](https://discord.gg/7m9sfhzaf3)

examples/LiquidAI/lfm2-8b-a1b-lora.yaml

@@ -1,59 +0,0 @@
-base_model: LiquidAI/LFM2-8B-A1B
-
-plugins:
-  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
-
-load_in_8bit: true
-
-eot_tokens:
-  - "<|im_end|>"
-datasets:
-  - path: mlabonne/FineTome-100k
-    type: chat_template
-    split: train[:20%]
-    field_messages: conversations
-    message_field_role: from
-    message_field_content: value
-dataset_prepared_path: last_run_prepared
-val_set_size: 0.05
-output_dir: ./outputs/out
-
-sequence_len: 4096
-sample_packing: true
-
-adapter: lora
-lora_model_dir:
-
-lora_r: 32
-lora_alpha: 16
-lora_dropout: 0.05
-lora_target_modules: 'model.layers.[\d]+.(mlp|cross_attn|self_attn).(up|down|gate|q|k|v|o)_proj'
-
-wandb_project:
-wandb_entity:
-wandb_watch:
-wandb_name:
-wandb_log_model:
-
-gradient_accumulation_steps: 2
-micro_batch_size: 4
-num_epochs: 1
-optimizer: adamw_torch_fused
-lr_scheduler: cosine
-learning_rate: 5e-5
-
-bf16: true
-tf32: true
-
-gradient_checkpointing: true
-resume_from_checkpoint:
-logging_steps: 1
-flash_attention: true
-
-warmup_ratio: 0.1
-evals_per_epoch: 2
-saves_per_epoch: 1
-
-weight_decay: 0.0
-
-# save_first_step: true  # uncomment this to validate checkpoint saving works with your config

examples/LiquidAI/lfm2-vl-lora.yaml

@@ -1,61 +0,0 @@
-base_model: LiquidAI/LFM2-VL-450M
-trust_remote_code: true
-model_type: AutoModelForImageTextToText
-processor_type: AutoProcessor
-
-plugins:
-  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
-
-# these 3 lines are needed for now to handle vision chat templates w images
-skip_prepare_dataset: true
-remove_unused_columns: false
-sample_packing: false
-
-datasets:
-  - path: HuggingFaceH4/llava-instruct-mix-vsft
-    type: chat_template
-    split: train[:1%]
-
-dataset_prepared_path: last_run_prepared
-val_set_size: 0.0
-output_dir: ./outputs/out
-
-adapter: lora
-lora_model_dir:
-
-sequence_len: 8192
-pad_to_sequence_len: false
-
-lora_r: 32
-lora_alpha: 16
-lora_dropout: 0.05
-lora_target_modules: 'model.language_model.layers.[\d]+.(mlp|cross_attn|self_attn).(up|down|gate|q|k|v|o)_proj'
-
-wandb_project:
-wandb_entity:
-wandb_watch:
-wandb_name:
-wandb_log_model:
-
-gradient_accumulation_steps: 4
-micro_batch_size: 1
-num_epochs: 1
-optimizer: adamw_bnb_8bit
-lr_scheduler: cosine
-learning_rate: 0.0002
-
-bf16: true
-fp16:
-tf32: true
-
-gradient_checkpointing: true
-logging_steps: 1
-flash_attention: true
-eager_attention:
-
-warmup_ratio: 0.1
-evals_per_epoch: 1
-saves_per_epoch: 1
-weight_decay: 0.0
-
-# save_first_step: true  # uncomment this to validate checkpoint saving works with your config

examples/alst/README.md

@@ -1,30 +0,0 @@
-# Arctic Long Sequence Training (ALST)
-
-Artic Long Sequence Training (ALST) is a technique for training long context models using a variety of optimization
-techniques. It is a combination of:
-- TiledMLP: Leverage tiling over the sequence dimension on MLP layers to reduce memory usage
-- Tiled Loss: Using optimized loss functions like Liger-Kernel or Cut Cross Entropy to reduce memory usage
-- Activation Offloading: Offload activations to CPU RAM to reduce memory usage
-
-For more information, you can check out the ALST paper [here](https://www.arxiv.org/abs/2506.13996).
-
-## Usage
-
-```yaml
-tiled_mlp: true
-
-# See Sequence Parallelism docs
-# https://docs.axolotl.ai/docs/sequence_parallelism.html
-context_parallel_size: int
-
-plugins:
-# See Cut Cross Entropy docs
-# https://docs.axolotl.ai/docs/custom_integrations.html#cut-cross-entropy
-  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
-
-# or Liger Kernel docs
-# https://docs.axolotl.ai/docs/custom_integrations.html#liger-kernels
-  - axolotl.integrations.liger.LigerPlugin
-# ...
-
-```

examples/alst/llama3-8b-deepspeed-alst.yaml

@@ -1,53 +0,0 @@
-base_model: meta-llama/Llama-3.1-8B
-# Automatically upload checkpoint and final model to HF
-# hub_model_id: username/custom_model_name
-
-datasets:
-  - path: togethercomputer/Long-Data-Collections
-    type: completion
-    field: text
-    data_files:
-      - pretrain/rp_sub.jsonl.zst
-  - path: princeton-nlp/TextbookChapters
-    type: completion
-    field: chapter
-dataset_prepared_path: last_run_prepared
-val_set_size: 0.0
-output_dir: ./outputs/out
-
-sequence_len: 500_000
-min_sample_len: 200_000
-sample_packing: true
-
-tiled_mlp: true
-context_parallel_size: 8
-plugins:
-  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
-
-gradient_accumulation_steps: 1
-micro_batch_size: 1
-num_epochs: 1
-optimizer: adamw_torch_8bit
-lr_scheduler: cosine
-learning_rate: 2e-5
-
-bf16: auto
-tf32: true
-
-gradient_checkpointing: true
-activation_offloading: legacy
-
-resume_from_checkpoint:
-logging_steps: 1
-flash_attention: true
-
-warmup_steps: 100
-saves_per_epoch: 1
-evals_per_epoch: 2
-weight_decay: 0.0
-special_tokens:
-  pad_token: <|end_of_text|>
-
-deepspeed: deepspeed_configs/zero3_bf16_cpuoffload_all.json
-
-# save_first_step: true  # uncomment this to validate checkpoint saving works with your config

examples/alst/llama3-8b-fsdp2-alst.yaml

@@ -1,59 +0,0 @@
-base_model: meta-llama/Llama-3.1-8B
-# Automatically upload checkpoint and final model to HF
-# hub_model_id: username/custom_model_name
-
-datasets:
-  - path: togethercomputer/Long-Data-Collections
-    type: completion
-    field: text
-    data_files:
-      - pretrain/rp_sub.jsonl.zst
-  - path: princeton-nlp/TextbookChapters
-    type: completion
-    field: chapter
-dataset_prepared_path: last_run_prepared
-val_set_size: 0.0
-output_dir: ./outputs/out
-
-sequence_len: 500_000
-min_sample_len: 200_000
-sample_packing: true
-
-tiled_mlp: true
-context_parallel_size: 8
-plugins:
-  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
-
-gradient_accumulation_steps: 1
-micro_batch_size: 1
-num_epochs: 1
-optimizer: adamw_torch_8bit
-lr_scheduler: cosine
-learning_rate: 2e-5
-
-bf16: auto
-tf32: true
-
-gradient_checkpointing: true
-activation_offloading: legacy
-
-resume_from_checkpoint:
-logging_steps: 1
-flash_attention: true
-
-warmup_steps: 100
-saves_per_epoch: 1
-evals_per_epoch: 2
-weight_decay: 0.0
-special_tokens:
-  pad_token: <|end_of_text|>
-
-fsdp_version: 2
-fsdp_config:
-  offload_params: false  # offloading is currently not compatible with SP + torchao optimizer
-  state_dict_type: SHARDED_STATE_DICT
-  auto_wrap_policy: TRANSFORMER_BASED_WRAP
-  transformer_layer_cls_to_wrap: LlamaDecoderLayer
-  reshard_after_forward: true
-
-# save_first_step: true  # uncomment this to validate checkpoint saving works with your config

examples/apertus/README.md

@@ -1,110 +0,0 @@
-# Finetune Swiss-AI's Apertus with Axolotl
-
-[Apertus](https://huggingface.co/collections/swiss-ai/apertus-llm-68b699e65415c231ace3b059) is a family of opensource models trained by Swiss-ai.
-
-This guide shows how to fine-tune it with Axolotl with multi-turn conversations and proper masking.
-
-## Getting started
-
-1. Install Axolotl following the [installation guide](https://docs.axolotl.ai/docs/installation.html). You need to install from main as Apertus is only on nightly or use our latest [Docker images](https://docs.axolotl.ai/docs/docker.html).
-
-    Here is an example of how to install from main for pip:
-
-```bash
-# Ensure you have Pytorch installed (Pytorch 2.6.0 min)
-git clone https://github.com/axolotl-ai-cloud/axolotl.git
-cd axolotl
-
-pip3 install packaging==23.2 setuptools==75.8.0 wheel ninja
-pip3 install --no-build-isolation -e '.[flash-attn]'
-
-# Install CCE https://docs.axolotl.ai/docs/custom_integrations.html#cut-cross-entropy
-python scripts/cutcrossentropy_install.py | sh
-```
-
-2. (Optional, highly recommended) Install XIELU CUDA
-
-```bash
-## Recommended for reduced VRAM and faster speeds
-
-# Point to CUDA toolkit directory
-# For those using our Docker image, use the below path.
-export CUDA_HOME=/usr/local/cuda
-
-pip3 install git+https://github.com/nickjbrowning/XIELU@59d6031 --no-build-isolation --no-deps
-```
-
-For any installation errors, see [XIELU Installation Issues](#xielu-installation-issues)
-
-3. Run the finetuning example:
-
-```bash
-axolotl train examples/apertus/apertus-8b-qlora.yaml
-```
-
-This config uses about 8.7 GiB VRAM.
-
-Let us know how it goes. Happy finetuning! 🚀
-
-### Tips
-
-- For inference, the official Apertus team recommends `top_p=0.9` and `temperature=0.8`.
-- You can instead use full paremter fine-tuning by removing the `adapter: qlora` and `load_in_4bit: true` from the config.
-- Read more on how to load your own dataset at [docs](https://docs.axolotl.ai/docs/dataset_loading.html).
-- The dataset format follows the OpenAI Messages format as seen [here](https://docs.axolotl.ai/docs/dataset-formats/conversation.html#chat_template).
-
-### XIELU Installation Issues
-
-#### `ModuleNotFoundError: No module named 'torch'`
-
-Please check these one by one:
-- Running in correct environment
-- Env has PyTorch installed
-- CUDA toolkit is at `CUDA_HOME`
-
-If those didn't help, please try the below solutions:
-
-1. Pass env for CMAKE and try install again:
-
-    ```bash
-    Python_EXECUTABLE=$(which python) pip3 install git+https://github.com/nickjbrowning/XIELU@59d6031 --no-build-isolation --no-deps
-    ```
-
-2. Git clone the repo and manually hardcode python path:
-
-    ```bash
-    git clone https://github.com/nickjbrowning/XIELU
-    cd xielu
-    git checkout 59d6031
-
-    cd xielu
-    nano CMakeLists.txt  # or vi depending on your preference
-    ```
-
-    ```diff
-    execute_process(
-    -    COMMAND ${Python_EXECUTABLE} -c "import torch.utils; print(torch.utils.cmake_prefix_path)"
-    +    COMMAND /root/miniconda3/envs/py3.11/bin/python -c "import torch.utils; print(torch.utils.cmake_prefix_path)"
-        RESULT_VARIABLE TORCH_CMAKE_PATH_RESULT
-        OUTPUT_VARIABLE TORCH_CMAKE_PATH_OUTPUT
-        ERROR_VARIABLE TORCH_CMAKE_PATH_ERROR
-    )
-    ```
-
-    ```bash
-    pip3 install . --no-build-isolation --no-deps
-    ```
-
-## Optimization Guides
-
-- [Multi-GPU Training](https://docs.axolotl.ai/docs/multi-gpu.html)
-- [Multi-Node Training](https://docs.axolotl.ai/docs/multi-node.html)
-- [LoRA Optimizations](https://docs.axolotl.ai/docs/lora_optims.html)
-
-## Related Resources
-
-- [Apertus Tech Report](https://github.com/swiss-ai/apertus-tech-report/blob/main/Apertus_Tech_Report.pdf)
-- [Axolotl Docs](https://docs.axolotl.ai)
-- [Axolotl Website](https://axolotl.ai)
-- [Axolotl GitHub](https://github.com/axolotl-ai-cloud/axolotl)
-- [Axolotl Discord](https://discord.gg/7m9sfhzaf3)

examples/apertus/apertus-8b-qlora.yaml

@@ -1,64 +0,0 @@
-base_model: swiss-ai/Apertus-8B-Instruct-2509
-
-# Automatically upload checkpoint and final model to HF
-# hub_model_id: username/custom_model_name
-
-plugins:
-  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
-
-load_in_8bit: false
-load_in_4bit: true
-
-datasets:
-  - path: fozziethebeat/alpaca_messages_2k_test
-    type: chat_template
-
-dataset_prepared_path: last_run_prepared
-val_set_size: 0.1
-output_dir: ./outputs/lora-out
-
-adapter: qlora
-lora_model_dir:
-
-sequence_len: 2048
-sample_packing: true
-
-lora_r: 32
-lora_alpha: 16
-lora_dropout: 0.05
-lora_target_linear: true
-lora_target_modules:
-  - gate_proj
-  - down_proj
-  - up_proj
-  - q_proj
-  - v_proj
-  - k_proj
-  - o_proj
-
-wandb_project:
-wandb_entity:
-wandb_watch:
-wandb_name:
-wandb_log_model:
-
-gradient_accumulation_steps: 4
-micro_batch_size: 2
-num_epochs: 1
-optimizer: adamw_bnb_8bit
-lr_scheduler: cosine
-learning_rate: 0.0002
-
-bf16: auto
-tf32: false
-
-gradient_checkpointing: true
-resume_from_checkpoint:
-logging_steps: 1
-flash_attention: true
-
-warmup_ratio: 0.1
-evals_per_epoch: 1
-saves_per_epoch: 1
-
-# save_first_step: true  # uncomment this to validate checkpoint saving works with your config

examples/arcee/README.md

@@ -1,56 +0,0 @@
-# Finetune ArceeAI's AFM with Axolotl
-
-[Arcee Foundation Models (AFM)](https://huggingface.co/collections/arcee-ai/afm-45b-68823397c351603014963473) are a family of 4.5B parameter open weight models trained by Arcee.ai.
-
-This guide shows how to fine-tune it with Axolotl with multi-turn conversations and proper masking.
-
-Thanks to the team at Arcee.ai for using Axolotl in supervised fine-tuning the AFM model.
-
-## Getting started
-
-1. Install Axolotl following the [installation guide](https://docs.axolotl.ai/docs/installation.html). You need to install from main as AFM is only on nightly or use our latest [Docker images](https://docs.axolotl.ai/docs/docker.html).
-
-    Here is an example of how to install from main for pip:
-
-```bash
-# Ensure you have Pytorch installed (Pytorch 2.6.0 min)
-git clone https://github.com/axolotl-ai-cloud/axolotl.git
-cd axolotl
-
-pip3 install packaging==23.2 setuptools==75.8.0 wheel ninja
-pip3 install --no-build-isolation -e '.[flash-attn]'
-
-# Install CCE https://docs.axolotl.ai/docs/custom_integrations.html#cut-cross-entropy
-python scripts/cutcrossentropy_install.py | sh
-```
-
-2. Run the finetuning example:
-
-```bash
-axolotl train examples/arcee/afm-4.5b-qlora.yaml
-```
-
-This config uses about 7.8GiB VRAM.
-
-Let us know how it goes. Happy finetuning! 🚀
-
-### TIPS
-
-- For inference, the official Arcee.ai team recommends `top_p: 0.95`, `temperature: 0.5`, `top_k: 50`, and `repeat_penalty: 1.1`.
-- You can run a full finetuning by removing the `adapter: qlora` and `load_in_4bit: true` from the config.
-- Read more on how to load your own dataset at [docs](https://docs.axolotl.ai/docs/dataset_loading.html).
-- The dataset format follows the OpenAI Messages format as seen [here](https://docs.axolotl.ai/docs/dataset-formats/conversation.html#chat_template).
-
-## Optimization Guides
-
-- [Multi-GPU Training](https://docs.axolotl.ai/docs/multi-gpu.html)
-- [Multi-Node Training](https://docs.axolotl.ai/docs/multi-node.html)
-- [LoRA Optimizations](https://docs.axolotl.ai/docs/lora_optims.html)
-
-## Related Resources
-
-- [AFM Blog](https://docs.arcee.ai/arcee-foundation-models/introduction-to-arcee-foundation-models)
-- [Axolotl Docs](https://docs.axolotl.ai)
-- [Axolotl Website](https://axolotl.ai)
-- [Axolotl GitHub](https://github.com/axolotl-ai-cloud/axolotl)
-- [Axolotl Discord](https://discord.gg/7m9sfhzaf3)

examples/arcee/afm-4.5b-qlora.yaml

@@ -1,64 +0,0 @@
-base_model: arcee-ai/AFM-4.5B
-
-# Automatically upload checkpoint and final model to HF
-# hub_model_id: username/custom_model_name
-
-plugins:
-  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
-
-load_in_8bit: false
-load_in_4bit: true
-
-datasets:
-  - path: fozziethebeat/alpaca_messages_2k_test
-    type: chat_template
-
-dataset_prepared_path: last_run_prepared
-val_set_size: 0.1
-output_dir: ./outputs/lora-out
-
-adapter: qlora
-lora_model_dir:
-
-sequence_len: 2048
-sample_packing: true
-
-lora_r: 32
-lora_alpha: 16
-lora_dropout: 0.05
-lora_target_linear: true
-lora_target_modules:
-  - gate_proj
-  - down_proj
-  - up_proj
-  - q_proj
-  - v_proj
-  - k_proj
-  - o_proj
-
-wandb_project:
-wandb_entity:
-wandb_watch:
-wandb_name:
-wandb_log_model:
-
-gradient_accumulation_steps: 4
-micro_batch_size: 2
-num_epochs: 1
-optimizer: adamw_bnb_8bit
-lr_scheduler: cosine
-learning_rate: 0.0002
-
-bf16: auto
-tf32: false
-
-gradient_checkpointing: true
-resume_from_checkpoint:
-logging_steps: 1
-flash_attention: true
-
-warmup_ratio: 0.1
-evals_per_epoch: 1
-saves_per_epoch: 1
-
-# save_first_step: true  # uncomment this to validate checkpoint saving works with your config

examples/archived/cerebras/btlm-ft.yml

@@ -66,7 +66,7 @@ flash_optimum:
 gptq_groupsize:
 gptq_model_v1:
 
-warmup_ratio: 0.1
+warmup_steps: 32
 evals_per_epoch: 4
 saves_per_epoch: 1
 save_total_limit:

examples/archived/cerebras/qlora.yml

@@ -43,7 +43,7 @@ xformers_attention: true
 flash_attention:
 gptq_groupsize:
 gptq_model_v1:
-warmup_ratio: 0.1
+warmup_steps: 10
 evals_per_epoch: 4
 saves_per_epoch: 1
 weight_decay: 0.1

examples/archived/code-llama/13b/lora.yml

@@ -17,7 +17,7 @@ output_dir: ./outputs/lora-out
 
 sequence_len: 4096
 sample_packing: true
-
+pad_to_sequence_len: true
 
 adapter: lora
 lora_model_dir:
@@ -47,7 +47,7 @@ resume_from_checkpoint:
 logging_steps: 1
 flash_attention: true
 
-warmup_ratio: 0.1
+warmup_steps: 10
 evals_per_epoch: 4
 saves_per_epoch: 1
 weight_decay: 0.0

examples/archived/code-llama/13b/qlora.yml

@@ -20,7 +20,7 @@ lora_model_dir:
 
 sequence_len: 4096
 sample_packing: true
-
+pad_to_sequence_len: true
 
 lora_r: 32
 lora_alpha: 16
@@ -48,7 +48,7 @@ resume_from_checkpoint:
 logging_steps: 1
 flash_attention: true
 
-warmup_ratio: 0.1
+warmup_steps: 10
 evals_per_epoch: 4
 saves_per_epoch: 1
 weight_decay: 0.0

examples/archived/code-llama/34b/lora.yml

@@ -17,7 +17,7 @@ output_dir: ./outputs/lora-out
 
 sequence_len: 4096
 sample_packing: true
-
+pad_to_sequence_len: true
 
 adapter: lora
 lora_model_dir:
@@ -47,7 +47,7 @@ resume_from_checkpoint:
 logging_steps: 1
 flash_attention: true
 
-warmup_ratio: 0.1
+warmup_steps: 10
 evals_per_epoch: 4
 saves_per_epoch: 1
 weight_decay: 0.0

examples/archived/code-llama/34b/qlora.yml

@@ -20,7 +20,7 @@ lora_model_dir:
 
 sequence_len: 4096
 sample_packing: true
-
+pad_to_sequence_len: true
 
 lora_r: 32
 lora_alpha: 16
@@ -48,7 +48,7 @@ resume_from_checkpoint:
 logging_steps: 1
 flash_attention: true
 
-warmup_ratio: 0.1
+warmup_steps: 10
 evals_per_epoch: 4
 saves_per_epoch: 1
 weight_decay: 0.0

examples/archived/code-llama/7b/lora.yml

@@ -17,7 +17,7 @@ output_dir: ./outputs/lora-out
 
 sequence_len: 4096
 sample_packing: true
-
+pad_to_sequence_len: true
 
 adapter: lora
 lora_model_dir:
@@ -47,7 +47,7 @@ resume_from_checkpoint:
 logging_steps: 1
 flash_attention: true
 
-warmup_ratio: 0.1
+warmup_steps: 10
 evals_per_epoch: 4
 saves_per_epoch: 1
 weight_decay: 0.0

examples/archived/code-llama/7b/qlora.yml

@@ -20,7 +20,7 @@ lora_model_dir:
 
 sequence_len: 4096
 sample_packing: true
-
+pad_to_sequence_len: true
 
 lora_r: 32
 lora_alpha: 16
@@ -48,7 +48,7 @@ resume_from_checkpoint:
 logging_steps: 1
 flash_attention: true
 
-warmup_ratio: 0.1
+warmup_steps: 10
 evals_per_epoch: 4
 saves_per_epoch: 1
 weight_decay: 0.0

examples/archived/dbrx/16bit-lora.yaml

@@ -54,7 +54,7 @@ resume_from_checkpoint:
 logging_steps: 1
 flash_attention: true
 
-warmup_ratio: 0.1
+warmup_steps: 10
 evals_per_epoch:
 saves_per_epoch: 1
 

examples/archived/dbrx/8bit-lora.yaml

@@ -57,7 +57,7 @@ resume_from_checkpoint:
 logging_steps: 1
 flash_attention: true
 
-warmup_ratio: 0.1
+warmup_steps: 10
 evals_per_epoch:
 saves_per_epoch: 1
 

examples/archived/dbrx/fft-ds-zero3.yaml

@@ -41,7 +41,7 @@ resume_from_checkpoint:
 logging_steps: 1
 flash_attention: true
 
-warmup_ratio: 0.1
+warmup_steps: 10
 evals_per_epoch:
 saves_per_epoch: 1
 

examples/archived/deepcoder/deepcoder-14B-preview-lora.yml

@@ -9,6 +9,10 @@ strict: false
 datasets:
   - path: fozziethebeat/alpaca_messages_2k_test
     type: chat_template
+    field_messages: messages
+    message_property_mappings:
+      role: role
+      content: content
 
 dataset_prepared_path:
 val_set_size: 0.05
@@ -17,7 +21,7 @@ output_dir: ./outputs/lora-out
 sequence_len: 4096
 sample_packing: true
 eval_sample_packing: false
-
+pad_to_sequence_len: true
 
 adapter: lora
 lora_model_dir:
@@ -47,7 +51,7 @@ resume_from_checkpoint:
 logging_steps: 1
 flash_attention: true
 
-warmup_ratio: 0.1
+warmup_steps: 10
 evals_per_epoch: 1
 saves_per_epoch: 1
 weight_decay: 0.0

examples/archived/falcon/config-7b-lora.yml

@@ -47,7 +47,7 @@ xformers_attention: true
 flash_attention:
 gptq_groupsize:
 gptq_model_v1:
-warmup_ratio: 0.1
+warmup_steps: 40
 evals_per_epoch: 4
 saves_per_epoch: 1
 weight_decay: 0.0

examples/archived/falcon/config-7b-qlora.yml

@@ -77,7 +77,7 @@ xformers_attention: true
 flash_attention:
 gptq_groupsize:
 gptq_model_v1:
-warmup_ratio: 0.1
+warmup_steps: 10
 evals_per_epoch: 4
 saves_per_epoch: 1
 weight_decay: 0.000001

examples/archived/falcon/config-7b.yml

@@ -44,7 +44,7 @@ xformers_attention: true
 flash_attention:
 gptq_groupsize:
 gptq_model_v1:
-warmup_ratio: 0.1
+warmup_steps: 40
 evals_per_epoch: 4
 saves_per_epoch: 1
 weight_decay: 0.0

examples/archived/gemma/qlora.yml

@@ -25,7 +25,7 @@ lora_target_linear: true
 sequence_len: 4096
 sample_packing: true
 eval_sample_packing: false
-
+pad_to_sequence_len: true
 
 wandb_project:
 wandb_entity: