tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

update doc and use P2P=LOC for brittle grpo test (#2649)

Browse Source 
* update doc and skip brittle grpo test

* fix the path to run the multigpu tests

* increase timeout, use LOC instead of NVL

* typo

* use hf cache from s3 backed cloudfront

* mark grpo as flaky test dues to vllm start
This commit is contained in:
Wing Lian
2025-05-12 14:17:25 -04:00
committed by GitHub
parent c7b6790614
commit f34eef546a
6 changed files with 131 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/multi-gpu-e2e.yml vendored
View File

@@ -3,7 +3,7 @@ name: docker-multigpu-tests-biweekly
on: on:
pull_request: pull_request:
paths: paths:
- 'tests/e2e/multigpu/*.py' - 'tests/e2e/multigpu/**.py'
- 'requirements.txt' - 'requirements.txt'
- 'setup.py' - 'setup.py'
- 'pyproject.toml' - 'pyproject.toml'

224
.github/workflows/tests.yml vendored
View File

@@ -44,96 +44,102 @@ jobs:
env: env:
SKIP: no-commit-to-branch SKIP: no-commit-to-branch
preload-cache: # preload-cache:
name: Preload HF cache # name: Preload HF cache
runs-on: ubuntu-latest # runs-on: ubuntu-latest
strategy: # strategy:
fail-fast: false # fail-fast: false
matrix: # matrix:
python_version: ["3.11"] # python_version: ["3.11"]
pytorch_version: ["2.6.0"] # pytorch_version: ["2.6.0"]
timeout-minutes: 20 # timeout-minutes: 20
#
env: # env:
AXOLOTL_IS_CI_CACHE_PRELOAD: "1" # AXOLOTL_IS_CI_CACHE_PRELOAD: "1"
#
steps: # steps:
- name: Check out repository code # - name: Check out repository code
uses: actions/checkout@v4 # uses: actions/checkout@v4
#
- name: Restore HF cache # - name: Restore HF cache
id: hf-cache-restore # id: hf-cache-restore
uses: actions/cache/restore@v4 # uses: actions/cache/restore@v4
with: # with:
path: | # path: |
/home/runner/.cache/huggingface/hub/datasets--* # /home/runner/.cache/huggingface/hub/datasets--*
/home/runner/.cache/huggingface/hub/models--* # /home/runner/.cache/huggingface/hub/models--*
key: ${{ runner.os }}-hf-hub-cache-v2 # key: ${{ runner.os }}-hf-hub-cache-v2
#
- name: Setup Python # - name: Restore Cache from S3
uses: actions/setup-python@v5 # id: hf-cache-restore-s3
with: # run: |
python-version: ${{ matrix.python_version }} # mkdir -p /home/runner/.cache/huggingface/hub
cache: 'pip' # caching pip dependencies # curl -L https://d1dttdx32dkk5p.cloudfront.net/hf-cache.tar.zst | tar -xf - -C /home/runner/.cache/huggingface/hub/ --use-compress-program unzstd
#
- name: upgrade pip # - name: Setup Python
run: | # uses: actions/setup-python@v5
pip3 install --upgrade pip # with:
pip3 install --upgrade packaging==23.2 setuptools==75.8.0 wheel # python-version: ${{ matrix.python_version }}
# cache: 'pip' # caching pip dependencies
- name: Install PyTorch #
run: | # - name: upgrade pip
pip3 install torch==${{ matrix.pytorch_version }} # run: |
# pip3 install --upgrade pip
- name: Install dependencies # pip3 install --upgrade packaging==23.2 setuptools==75.8.0 wheel
run: | #
pip3 show torch # - name: Install PyTorch
pip3 install --no-build-isolation -U -e . # run: |
python scripts/unsloth_install.py | sh # pip3 install torch==${{ matrix.pytorch_version }}
python scripts/cutcrossentropy_install.py | sh #
pip3 install -r requirements-dev.txt -r requirements-tests.txt # - name: Install dependencies
# run: |
- name: Make sure PyTorch version wasn't clobbered # pip3 show torch
run: | # pip3 install --no-build-isolation -U -e .
python -c "import torch; assert '${{ matrix.pytorch_version }}' in torch.__version__" # python scripts/unsloth_install.py | sh
# python scripts/cutcrossentropy_install.py | sh
- name: Ensure axolotl CLI was installed # pip3 install -r requirements-dev.txt -r requirements-tests.txt
run: | #
axolotl --help # - name: Make sure PyTorch version wasn't clobbered
# run: |
- name: Pre-Download dataset fixture # python -c "import torch; assert '${{ matrix.pytorch_version }}' in torch.__version__"
run: | #
huggingface-cli download --repo-type=dataset axolotl-ai-internal/axolotl-oss-dataset-fixtures # - name: Ensure axolotl CLI was installed
# run: |
- name: Run tests # axolotl --help
run: | #
pytest -v tests/conftest.py # - name: Pre-Download dataset fixture
# run: |
- name: Upload coverage to Codecov # huggingface-cli download --repo-type=dataset axolotl-ai-internal/axolotl-oss-dataset-fixtures
uses: codecov/codecov-action@v5 #
with: # - name: Run tests
token: ${{ secrets.CODECOV_TOKEN }} # run: |
files: ./coverage.xml # pytest -v tests/conftest.py
flags: unittests,pytorch-${{ matrix.pytorch_version }} #
fail_ci_if_error: false # - name: Upload coverage to Codecov
# uses: codecov/codecov-action@v5
- name: cleanup pip cache # with:
run: | # token: ${{ secrets.CODECOV_TOKEN }}
find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \; # files: ./coverage.xml
# flags: unittests,pytorch-${{ matrix.pytorch_version }}
- name: Save HF cache # fail_ci_if_error: false
id: hf-cache #
uses: actions/cache/save@v4 # - name: cleanup pip cache
with: # run: |
path: | # find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \;
/home/runner/.cache/huggingface/hub/datasets--* #
/home/runner/.cache/huggingface/hub/models--* # - name: Save HF cache
key: ${{ steps.hf-cache-restore.outputs.cache-primary-key }} # id: hf-cache
# uses: actions/cache/save@v4
# with:
# path: |
# /home/runner/.cache/huggingface/hub/datasets--*
# /home/runner/.cache/huggingface/hub/models--*
# key: ${{ steps.hf-cache-restore.outputs.cache-primary-key }}
pytest: pytest:
name: PyTest name: PyTest
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [preload-cache] # needs: [preload-cache]
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
@@ -145,14 +151,20 @@ jobs:
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Restore HF cache # - name: Restore HF cache
id: hf-cache-restore # id: hf-cache-restore
uses: actions/cache/restore@v4 # uses: actions/cache/restore@v4
with: # with:
path: | # path: |
/home/runner/.cache/huggingface/hub/datasets--* # /home/runner/.cache/huggingface/hub/datasets--*
/home/runner/.cache/huggingface/hub/models--* # /home/runner/.cache/huggingface/hub/models--*
key: ${{ runner.os }}-hf-hub-cache-v2 # key: ${{ runner.os }}-hf-hub-cache-v2
- name: Restore Cache from S3
id: hf-cache-restore-s3
run: |
mkdir -p /home/runner/.cache/huggingface/hub
curl -L https://d1dttdx32dkk5p.cloudfront.net/hf-cache.tar.zst | tar -xf - -C /home/runner/.cache/huggingface/hub/ --use-compress-program unzstd
- name: Setup Python - name: Setup Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
@@ -210,7 +222,7 @@ jobs:
pytest-sdist: pytest-sdist:
name: PyTest from Source Dist name: PyTest from Source Dist
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [preload-cache] # needs: [preload-cache]
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
@@ -222,14 +234,20 @@ jobs:
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v4 uses: actions/checkout@v4
- name: Restore HF cache # - name: Restore HF cache
id: hf-cache-restore # id: hf-cache-restore
uses: actions/cache/restore@v4 # uses: actions/cache/restore@v4
with: # with:
path: | # path: |
/home/runner/.cache/huggingface/hub/datasets--* # /home/runner/.cache/huggingface/hub/datasets--*
/home/runner/.cache/huggingface/hub/models--* # /home/runner/.cache/huggingface/hub/models--*
key: ${{ runner.os }}-hf-hub-cache-v2 # key: ${{ runner.os }}-hf-hub-cache-v2
- name: Restore Cache from S3
id: hf-cache-restore-s3
run: |
mkdir -p /home/runner/.cache/huggingface/hub
curl -L https://d1dttdx32dkk5p.cloudfront.net/hf-cache.tar.zst | tar -xf - -C /home/runner/.cache/huggingface/hub/ --use-compress-program unzstd
- name: Setup Python - name: Setup Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5

2
cicd/e2e_tests.py
View File

@@ -6,7 +6,7 @@ from .single_gpu import GPU_CONFIG, VOLUME_CONFIG, app, cicd_image, run_cmd
@app.function( @app.function(
image=cicd_image, image=cicd_image,
gpu=GPU_CONFIG, gpu=GPU_CONFIG,
timeout=60 * 60, timeout=90 * 60, # 90 min
cpu=8.0, cpu=8.0,
memory=131072, memory=131072,
volumes=VOLUME_CONFIG, volumes=VOLUME_CONFIG,

2
codecov.yml
View File

@@ -19,7 +19,7 @@ coverage:
if_no_uploads: error if_no_uploads: error
if_not_found: success if_not_found: success
if_ci_failed: error if_ci_failed: error
only_pulls: false only_pulls: true
flags: null flags: null
paths: null paths: null
patch: patch:

1
docs/config.qmd
View File

@@ -505,6 +505,7 @@ save_strategy: # Set to `"no"` to skip checkpoint saves, `"epoch"` at end of eac
save_steps: # Leave empty to save at each epoch, integer for every N steps. float for fraction of total steps save_steps: # Leave empty to save at each epoch, integer for every N steps. float for fraction of total steps
saves_per_epoch: # number of times per epoch to save a checkpoint, mutually exclusive with save_steps saves_per_epoch: # number of times per epoch to save a checkpoint, mutually exclusive with save_steps
save_total_limit: # Checkpoints saved at a time save_total_limit: # Checkpoints saved at a time
save_only_model: # Save only the model weights, skipping the optimizer. Using this means you can't resume from checkpoints.
# Maximum number of iterations to train for. It precedes num_epochs which means that # Maximum number of iterations to train for. It precedes num_epochs which means that
# if both are set, num_epochs will not be guaranteed. # if both are set, num_epochs will not be guaranteed.
# e.g., when 1 epoch is 1000 steps => `num_epochs: 2` and `max_steps: 100` will train for 100 steps # e.g., when 1 epoch is 1000 steps => `num_epochs: 2` and `max_steps: 100` will train for 100 steps

10
tests/e2e/multigpu/solo/test_grpo.py
View File

@@ -166,6 +166,7 @@ def oai_gsm8k_transform(cfg, *args, **kwargs):
""" """
) )
@pytest.mark.skip(reason="flaky test")
@pytest.mark.parametrize( @pytest.mark.parametrize(
"num_gpus", "num_gpus",
[1, 2], [1, 2],
@@ -227,7 +228,7 @@ def oai_gsm8k_transform(cfg, *args, **kwargs):
current_env = os.environ.copy() current_env = os.environ.copy()
env = { env = {
"NCCL_P2P_LEVEL": "NVL", "NCCL_P2P_LEVEL": "LOC",
**current_env, **current_env,
"CUDA_VISIBLE_DEVICES": "1", "CUDA_VISIBLE_DEVICES": "1",
"VLLM_DISABLE_COMPILE_CACHE": "1", "VLLM_DISABLE_COMPILE_CACHE": "1",
@@ -257,7 +258,7 @@ def oai_gsm8k_transform(cfg, *args, **kwargs):
f"{get_torch_dist_unique_port()}", f"{get_torch_dist_unique_port()}",
], ],
env={ env={
"NCCL_P2P_LEVEL": "NVL", "NCCL_P2P_LEVEL": "LOC",
"NCCL_DEBUG": "INFO", "NCCL_DEBUG": "INFO",
**current_env, **current_env,
}, },
@@ -265,6 +266,7 @@ def oai_gsm8k_transform(cfg, *args, **kwargs):
finally: finally:
recursive_kill(vllm_process) recursive_kill(vllm_process)
@pytest.mark.skip(reason="flaky test")
@pytest.mark.parametrize( @pytest.mark.parametrize(
"num_gpus", "num_gpus",
[1, 2], [1, 2],
@@ -320,7 +322,7 @@ def oai_gsm8k_transform(cfg, *args, **kwargs):
current_env = os.environ.copy() current_env = os.environ.copy()
env = { env = {
"NCCL_P2P_LEVEL": "NVL", # nccl can be brittle, assume P2P isn't reliable "NCCL_P2P_LEVEL": "LOC", # nccl can be brittle, assume P2P isn't reliable
**current_env, **current_env,
"CUDA_VISIBLE_DEVICES": "1", "CUDA_VISIBLE_DEVICES": "1",
"VLLM_DISABLE_COMPILE_CACHE": "1", "VLLM_DISABLE_COMPILE_CACHE": "1",
@@ -350,7 +352,7 @@ def oai_gsm8k_transform(cfg, *args, **kwargs):
f"{get_torch_dist_unique_port()}", f"{get_torch_dist_unique_port()}",
], ],
env={ env={
"NCCL_P2P_LEVEL": "NVL", "NCCL_P2P_LEVEL": "LOC",
"NCCL_DEBUG": "INFO", "NCCL_DEBUG": "INFO",
**current_env, **current_env,
}, },