update doc and use P2P=LOC for brittle grpo test (#2649)
* update doc and skip brittle grpo test * fix the path to run the multigpu tests * increase timeout, use LOC instead of NVL * typo * use hf cache from s3 backed cloudfront * mark grpo as flaky test dues to vllm start
This commit is contained in:
Wing Lian
GitHub
2
.github/workflows/multi-gpu-e2e.yml
vendored
2
.github/workflows/multi-gpu-e2e.yml
vendored
@@ -3,7 +3,7 @@ name: docker-multigpu-tests-biweekly
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'tests/e2e/multigpu/*.py'
|
||||
- 'tests/e2e/multigpu/**.py'
|
||||
- 'requirements.txt'
|
||||
- 'setup.py'
|
||||
- 'pyproject.toml'
|
||||
|
||||
224
.github/workflows/tests.yml
vendored
224
.github/workflows/tests.yml
vendored
@@ -44,96 +44,102 @@ jobs:
|
||||
env:
|
||||
SKIP: no-commit-to-branch
|
||||
|
||||
preload-cache:
|
||||
name: Preload HF cache
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python_version: ["3.11"]
|
||||
pytorch_version: ["2.6.0"]
|
||||
timeout-minutes: 20
|
||||
|
||||
env:
|
||||
AXOLOTL_IS_CI_CACHE_PRELOAD: "1"
|
||||
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Restore HF cache
|
||||
id: hf-cache-restore
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: |
|
||||
/home/runner/.cache/huggingface/hub/datasets--*
|
||||
/home/runner/.cache/huggingface/hub/models--*
|
||||
key: ${{ runner.os }}-hf-hub-cache-v2
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python_version }}
|
||||
cache: 'pip' # caching pip dependencies
|
||||
|
||||
- name: upgrade pip
|
||||
run: |
|
||||
pip3 install --upgrade pip
|
||||
pip3 install --upgrade packaging==23.2 setuptools==75.8.0 wheel
|
||||
|
||||
- name: Install PyTorch
|
||||
run: |
|
||||
pip3 install torch==${{ matrix.pytorch_version }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip3 show torch
|
||||
pip3 install --no-build-isolation -U -e .
|
||||
python scripts/unsloth_install.py | sh
|
||||
python scripts/cutcrossentropy_install.py | sh
|
||||
pip3 install -r requirements-dev.txt -r requirements-tests.txt
|
||||
|
||||
- name: Make sure PyTorch version wasn't clobbered
|
||||
run: |
|
||||
python -c "import torch; assert '${{ matrix.pytorch_version }}' in torch.__version__"
|
||||
|
||||
- name: Ensure axolotl CLI was installed
|
||||
run: |
|
||||
axolotl --help
|
||||
|
||||
- name: Pre-Download dataset fixture
|
||||
run: |
|
||||
huggingface-cli download --repo-type=dataset axolotl-ai-internal/axolotl-oss-dataset-fixtures
|
||||
|
||||
- name: Run tests
|
||||
run: |
|
||||
pytest -v tests/conftest.py
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v5
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
files: ./coverage.xml
|
||||
flags: unittests,pytorch-${{ matrix.pytorch_version }}
|
||||
fail_ci_if_error: false
|
||||
|
||||
- name: cleanup pip cache
|
||||
run: |
|
||||
find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \;
|
||||
|
||||
- name: Save HF cache
|
||||
id: hf-cache
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: |
|
||||
/home/runner/.cache/huggingface/hub/datasets--*
|
||||
/home/runner/.cache/huggingface/hub/models--*
|
||||
key: ${{ steps.hf-cache-restore.outputs.cache-primary-key }}
|
||||
# preload-cache:
|
||||
# name: Preload HF cache
|
||||
# runs-on: ubuntu-latest
|
||||
# strategy:
|
||||
# fail-fast: false
|
||||
# matrix:
|
||||
# python_version: ["3.11"]
|
||||
# pytorch_version: ["2.6.0"]
|
||||
# timeout-minutes: 20
|
||||
#
|
||||
# env:
|
||||
# AXOLOTL_IS_CI_CACHE_PRELOAD: "1"
|
||||
#
|
||||
# steps:
|
||||
# - name: Check out repository code
|
||||
# uses: actions/checkout@v4
|
||||
#
|
||||
# - name: Restore HF cache
|
||||
# id: hf-cache-restore
|
||||
# uses: actions/cache/restore@v4
|
||||
# with:
|
||||
# path: |
|
||||
# /home/runner/.cache/huggingface/hub/datasets--*
|
||||
# /home/runner/.cache/huggingface/hub/models--*
|
||||
# key: ${{ runner.os }}-hf-hub-cache-v2
|
||||
#
|
||||
# - name: Restore Cache from S3
|
||||
# id: hf-cache-restore-s3
|
||||
# run: |
|
||||
# mkdir -p /home/runner/.cache/huggingface/hub
|
||||
# curl -L https://d1dttdx32dkk5p.cloudfront.net/hf-cache.tar.zst | tar -xf - -C /home/runner/.cache/huggingface/hub/ --use-compress-program unzstd
|
||||
#
|
||||
# - name: Setup Python
|
||||
# uses: actions/setup-python@v5
|
||||
# with:
|
||||
# python-version: ${{ matrix.python_version }}
|
||||
# cache: 'pip' # caching pip dependencies
|
||||
#
|
||||
# - name: upgrade pip
|
||||
# run: |
|
||||
# pip3 install --upgrade pip
|
||||
# pip3 install --upgrade packaging==23.2 setuptools==75.8.0 wheel
|
||||
#
|
||||
# - name: Install PyTorch
|
||||
# run: |
|
||||
# pip3 install torch==${{ matrix.pytorch_version }}
|
||||
#
|
||||
# - name: Install dependencies
|
||||
# run: |
|
||||
# pip3 show torch
|
||||
# pip3 install --no-build-isolation -U -e .
|
||||
# python scripts/unsloth_install.py | sh
|
||||
# python scripts/cutcrossentropy_install.py | sh
|
||||
# pip3 install -r requirements-dev.txt -r requirements-tests.txt
|
||||
#
|
||||
# - name: Make sure PyTorch version wasn't clobbered
|
||||
# run: |
|
||||
# python -c "import torch; assert '${{ matrix.pytorch_version }}' in torch.__version__"
|
||||
#
|
||||
# - name: Ensure axolotl CLI was installed
|
||||
# run: |
|
||||
# axolotl --help
|
||||
#
|
||||
# - name: Pre-Download dataset fixture
|
||||
# run: |
|
||||
# huggingface-cli download --repo-type=dataset axolotl-ai-internal/axolotl-oss-dataset-fixtures
|
||||
#
|
||||
# - name: Run tests
|
||||
# run: |
|
||||
# pytest -v tests/conftest.py
|
||||
#
|
||||
# - name: Upload coverage to Codecov
|
||||
# uses: codecov/codecov-action@v5
|
||||
# with:
|
||||
# token: ${{ secrets.CODECOV_TOKEN }}
|
||||
# files: ./coverage.xml
|
||||
# flags: unittests,pytorch-${{ matrix.pytorch_version }}
|
||||
# fail_ci_if_error: false
|
||||
#
|
||||
# - name: cleanup pip cache
|
||||
# run: |
|
||||
# find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \;
|
||||
#
|
||||
# - name: Save HF cache
|
||||
# id: hf-cache
|
||||
# uses: actions/cache/save@v4
|
||||
# with:
|
||||
# path: |
|
||||
# /home/runner/.cache/huggingface/hub/datasets--*
|
||||
# /home/runner/.cache/huggingface/hub/models--*
|
||||
# key: ${{ steps.hf-cache-restore.outputs.cache-primary-key }}
|
||||
|
||||
pytest:
|
||||
name: PyTest
|
||||
runs-on: ubuntu-latest
|
||||
needs: [preload-cache]
|
||||
# needs: [preload-cache]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -145,14 +151,20 @@ jobs:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Restore HF cache
|
||||
id: hf-cache-restore
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: |
|
||||
/home/runner/.cache/huggingface/hub/datasets--*
|
||||
/home/runner/.cache/huggingface/hub/models--*
|
||||
key: ${{ runner.os }}-hf-hub-cache-v2
|
||||
# - name: Restore HF cache
|
||||
# id: hf-cache-restore
|
||||
# uses: actions/cache/restore@v4
|
||||
# with:
|
||||
# path: |
|
||||
# /home/runner/.cache/huggingface/hub/datasets--*
|
||||
# /home/runner/.cache/huggingface/hub/models--*
|
||||
# key: ${{ runner.os }}-hf-hub-cache-v2
|
||||
|
||||
- name: Restore Cache from S3
|
||||
id: hf-cache-restore-s3
|
||||
run: |
|
||||
mkdir -p /home/runner/.cache/huggingface/hub
|
||||
curl -L https://d1dttdx32dkk5p.cloudfront.net/hf-cache.tar.zst | tar -xf - -C /home/runner/.cache/huggingface/hub/ --use-compress-program unzstd
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
@@ -210,7 +222,7 @@ jobs:
|
||||
pytest-sdist:
|
||||
name: PyTest from Source Dist
|
||||
runs-on: ubuntu-latest
|
||||
needs: [preload-cache]
|
||||
# needs: [preload-cache]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -222,14 +234,20 @@ jobs:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Restore HF cache
|
||||
id: hf-cache-restore
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: |
|
||||
/home/runner/.cache/huggingface/hub/datasets--*
|
||||
/home/runner/.cache/huggingface/hub/models--*
|
||||
key: ${{ runner.os }}-hf-hub-cache-v2
|
||||
# - name: Restore HF cache
|
||||
# id: hf-cache-restore
|
||||
# uses: actions/cache/restore@v4
|
||||
# with:
|
||||
# path: |
|
||||
# /home/runner/.cache/huggingface/hub/datasets--*
|
||||
# /home/runner/.cache/huggingface/hub/models--*
|
||||
# key: ${{ runner.os }}-hf-hub-cache-v2
|
||||
|
||||
- name: Restore Cache from S3
|
||||
id: hf-cache-restore-s3
|
||||
run: |
|
||||
mkdir -p /home/runner/.cache/huggingface/hub
|
||||
curl -L https://d1dttdx32dkk5p.cloudfront.net/hf-cache.tar.zst | tar -xf - -C /home/runner/.cache/huggingface/hub/ --use-compress-program unzstd
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
|
||||
Reference in New Issue
Block a user