c67910fa6f
* bump hf deps * upgrade liger-kernel too * install cce from fork for transformers fix * fix reference to vocab size in gemma3 patch * use padding_idx instead of pad_token_id * remove fixed gemma3 patch * use updated cce fork * fix local mllama cce patches w docstring * add test for multipack with trainer setup and fix trainer for trainer refactor upstream * bump modal version * guard for iterable datasetS * mllama model arch layout changed in latest transformers * fix batch sampler with drop_last * fix: address upstream vlm changes for lora * fix: update references to old lora target path * fix: remove mllama fa2 patch due to upstream fix * fix: lora kernel patch path for multimodal models * fix: removed mllama from quarto * run test for came optim on 2.6.0+ * fix fsdp2 patch and remove deprecated patch * make sure to set sequence_parallel_degree for grpo * Add SP test for GRPO * add sp to grpo config for trainer * use reward_funcs as kwarg to grpo trainer * fix the comprehension for reward funcs * reward funcs already passed in as args * init sp_group right before training * fix check for adding models to SP context * make sure to pass args to super * upgrade deepspeed * use updated trl and add reasoning flags for vllm * patch the worker --------- Co-authored-by: NanoCode012 <nano@axolotl.ai>
76 lines
2.5 KiB
YAML
76 lines
2.5 KiB
YAML
name: docker-multigpu-tests-biweekly
|
|
|
|
on:
|
|
pull_request:
|
|
paths:
|
|
- 'tests/e2e/multigpu/**.py'
|
|
- 'requirements.txt'
|
|
- 'setup.py'
|
|
- 'pyproject.toml'
|
|
- '.github/workflows/multi-gpu-e2e.yml'
|
|
- 'src/axolotl/core/trainers/mixins/sequence_parallel.py'
|
|
- 'src/axolotl/utils/distributed.py'
|
|
workflow_dispatch:
|
|
schedule:
|
|
- cron: '0 0 * * 1,4' # Runs at 00:00 UTC every monday & thursday
|
|
|
|
# Cancel jobs on the same ref if a new one is triggered
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
|
|
|
|
jobs:
|
|
test-axolotl-multigpu:
|
|
if: ${{ ! contains(github.event.commits[0].message, '[skip e2e]') && github.repository_owner == 'axolotl-ai-cloud' }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- cuda: 124
|
|
cuda_version: 12.4.1
|
|
python_version: "3.11"
|
|
pytorch: 2.6.0
|
|
axolotl_extras: vllm
|
|
num_gpus: 2
|
|
nightly_build: "true"
|
|
- cuda: 124
|
|
cuda_version: 12.4.1
|
|
python_version: "3.11"
|
|
pytorch: 2.5.1
|
|
axolotl_extras:
|
|
num_gpus: 2
|
|
nightly_build: "true"
|
|
- cuda: 126
|
|
cuda_version: 12.6.3
|
|
python_version: "3.11"
|
|
pytorch: 2.7.0
|
|
axolotl_extras:
|
|
num_gpus: 2
|
|
nightly_build: "true"
|
|
runs-on: [self-hosted, modal]
|
|
timeout-minutes: 120
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
- name: Install Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.11"
|
|
- name: Install Modal
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install modal==1.0.2 jinja2
|
|
- name: Update env vars
|
|
run: |
|
|
echo "BASE_TAG=main-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}" >> $GITHUB_ENV
|
|
echo "PYTORCH_VERSION=${{ matrix.pytorch}}" >> $GITHUB_ENV
|
|
echo "AXOLOTL_ARGS=${{ matrix.axolotl_args}}" >> $GITHUB_ENV
|
|
echo "AXOLOTL_EXTRAS=${{ matrix.axolotl_extras}}" >> $GITHUB_ENV
|
|
echo "CUDA=${{ matrix.cuda }}" >> $GITHUB_ENV
|
|
echo "N_GPUS=${{ matrix.num_gpus }}" >> $GITHUB_ENV
|
|
echo "NIGHTLY_BUILD=${{ matrix.nightly_build }}" >> $GITHUB_ENV
|
|
echo "CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }}" >> $GITHUB_ENV
|
|
- name: Run tests job on Modal
|
|
run: |
|
|
modal run cicd.multigpu
|