54392ac8a6
* Attempt to run multigpu in PR CI for now to ensure it works * fix yaml file * forgot to include multigpu tests * fix call to cicd.multigpu * dump dictdefault to dict for yaml conversion * use to_dict instead of casting * 16bit-lora w flash attention, 8bit lora seems problematic * add llama fsdp test * more tests * Add test for qlora + fsdp with prequant * limit accelerate to 2 processes and disable broken qlora+fsdp+bnb test * move multigpu tests to biweekly
45 lines
1.5 KiB
YAML
45 lines
1.5 KiB
YAML
name: docker-multigpu-tests-biweekly
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
schedule:
|
|
- cron: '0 0 * * 1,4' # Runs at 00:00 UTC every monday & thursday
|
|
|
|
jobs:
|
|
test-axolotl-multigpu:
|
|
if: ${{ ! contains(github.event.commits[0].message, '[skip docker]]') && github.repository_owner == 'axolotl-ai-cloud' }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- cuda: 121
|
|
cuda_version: 12.1.1
|
|
python_version: "3.11"
|
|
pytorch: 2.3.1
|
|
axolotl_extras:
|
|
num_gpus: 2
|
|
runs-on: [self-hosted, modal]
|
|
timeout-minutes: 120
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
- name: Install Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: "3.10"
|
|
- name: Install Modal
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
pip install modal==0.63.64 jinja2
|
|
- name: Update env vars
|
|
run: |
|
|
echo "BASE_TAG=main-base-py${{ matrix.python_version }}-cu${{ matrix.cuda }}-${{ matrix.pytorch }}" >> $GITHUB_ENV
|
|
echo "PYTORCH_VERSION=${{ matrix.pytorch}}" >> $GITHUB_ENV
|
|
echo "AXOLOTL_ARGS=${{ matrix.axolotl_args}}" >> $GITHUB_ENV
|
|
echo "AXOLOTL_EXTRAS=${{ matrix.axolotl_extras}}" >> $GITHUB_ENV
|
|
echo "CUDA=${{ matrix.cuda }}" >> $GITHUB_ENV
|
|
echo "N_GPUS=${{ matrix.num_gpus }}" >> $GITHUB_ENV
|
|
- name: Run tests job on Modal
|
|
run: |
|
|
modal run cicd.multigpu
|