From c4053481ff0f6739f1f9d513f1d02c55e6746a38 Mon Sep 17 00:00:00 2001 From: Dan Saunders Date: Wed, 23 Apr 2025 10:33:30 -0400 Subject: [PATCH] Codecov fixes / improvements (#2549) * adding codecov reporting * random change * codecov fixes * adding missing dependency * fix --------- Co-authored-by: Dan Saunders --- .github/workflows/multi-gpu-e2e.yml | 1 + .github/workflows/tests-nightly.yml | 1 + .github/workflows/tests.yml | 3 +++ cicd/cicd.sh | 12 +++--------- cicd/e2e_tests.py | 1 + cicd/multigpu.py | 1 + cicd/multigpu.sh | 16 ++++++---------- requirements-tests.txt | 1 + 8 files changed, 17 insertions(+), 19 deletions(-) diff --git a/.github/workflows/multi-gpu-e2e.yml b/.github/workflows/multi-gpu-e2e.yml index f89de494d..fcc6ee021 100644 --- a/.github/workflows/multi-gpu-e2e.yml +++ b/.github/workflows/multi-gpu-e2e.yml @@ -67,6 +67,7 @@ jobs: echo "CUDA=${{ matrix.cuda }}" >> $GITHUB_ENV echo "N_GPUS=${{ matrix.num_gpus }}" >> $GITHUB_ENV echo "NIGHTLY_BUILD=${{ matrix.nightly_build }}" >> $GITHUB_ENV + echo "CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }}" >> $GITHUB_ENV - name: Run tests job on Modal run: | modal run cicd.multigpu diff --git a/.github/workflows/tests-nightly.yml b/.github/workflows/tests-nightly.yml index 9d3b056e1..77cefc992 100644 --- a/.github/workflows/tests-nightly.yml +++ b/.github/workflows/tests-nightly.yml @@ -147,6 +147,7 @@ jobs: echo "CUDA=${{ matrix.cuda }}" >> $GITHUB_ENV echo "N_GPUS=${{ matrix.num_gpus }}" >> $GITHUB_ENV echo "NIGHTLY_BUILD=${{ matrix.nightly_build }}" >> $GITHUB_ENV + echo "CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }}" >> $GITHUB_ENV - name: Run tests job on Modal run: | modal run cicd.e2e_tests diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 863687128..ba1c837cb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -109,6 +109,7 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 with: + token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml flags: unittests,pytorch-${{ matrix.pytorch_version }} fail_ci_if_error: false @@ -241,6 +242,7 @@ jobs: echo "CUDA=${{ matrix.cuda }}" >> $GITHUB_ENV echo "MODAL_IMAGE_BUILDER_VERSION=2024.10" >> $GITHUB_ENV echo "N_GPUS=${{ matrix.num_gpus }}" >> $GITHUB_ENV + echo "CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }}" >> $GITHUB_ENV - name: Run tests job on Modal run: | modal run cicd.e2e_tests @@ -288,6 +290,7 @@ jobs: echo "CUDA=${{ matrix.cuda }}" >> $GITHUB_ENV echo "MODAL_IMAGE_BUILDER_VERSION=2024.10" >> $GITHUB_ENV echo "N_GPUS=${{ matrix.num_gpus }}" >> $GITHUB_ENV + echo "CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }}" >> $GITHUB_ENV - name: Run tests job on Modal run: | modal run cicd.e2e_tests diff --git a/cicd/cicd.sh b/cicd/cicd.sh index 3ae4896ac..89d746e5b 100755 --- a/cicd/cicd.sh +++ b/cicd/cicd.sh @@ -9,8 +9,7 @@ pytest -v --durations=10 -n8 \ --ignore=tests/patched/ \ --ignore=tests/cli \ /workspace/axolotl/tests/ \ - --cov=axolotl \ - --cov-report=xml:coverage.xml + --cov=axolotl # Run lora kernels tests with coverage append pytest -v --durations=10 \ @@ -51,11 +50,6 @@ pytest -v --durations=10 \ /workspace/axolotl/tests/e2e/ \ --cov=axolotl \ --cov-append \ - --cov-report=xml:coverage.xml + --cov-report=xml:e2e-coverage.xml -# Upload coverage to Codecov -if [ -f e2e-coverage.xml ]; then - codecov -f e2e-coverage.xml -F e2e,pytorch-${PYTORCH_VERSION} -else - echo "Coverage file not found. Coverage report may have failed." -fi +codecov upload-process -t $CODECOV_TOKEN -f e2e-coverage.xml -F e2e,pytorch-${PYTORCH_VERSION} diff --git a/cicd/e2e_tests.py b/cicd/e2e_tests.py index d7cdcc473..998f8c35d 100644 --- a/cicd/e2e_tests.py +++ b/cicd/e2e_tests.py @@ -28,6 +28,7 @@ df_args = { "GITHUB_REF": os.environ.get("GITHUB_REF", "refs/heads/main"), "GITHUB_SHA": os.environ.get("GITHUB_SHA", ""), "NIGHTLY_BUILD": os.environ.get("NIGHTLY_BUILD", ""), + "CODECOV_TOKEN": os.environ.get("CODECOV_TOKEN", ""), "HF_HOME": "/workspace/data/huggingface-cache/hub", } diff --git a/cicd/multigpu.py b/cicd/multigpu.py index 66d4d9990..90d4ce1ee 100644 --- a/cicd/multigpu.py +++ b/cicd/multigpu.py @@ -29,6 +29,7 @@ df_args = { "CUDA": os.environ.get("CUDA", "121"), "GITHUB_REF": os.environ.get("GITHUB_REF", "refs/heads/main"), "GITHUB_SHA": os.environ.get("GITHUB_SHA", ""), + "CODECOV_TOKEN": os.environ.get("CODECOV_TOKEN", ""), "HF_HOME": "/workspace/data/huggingface-cache/hub", } diff --git a/cicd/multigpu.sh b/cicd/multigpu.sh index 519ac0133..1e6f01471 100755 --- a/cicd/multigpu.sh +++ b/cicd/multigpu.sh @@ -6,13 +6,13 @@ pytest -v -n2 \ --ignore=/workspace/axolotl/tests/e2e/multigpu/solo/ \ --ignore=/workspace/axolotl/tests/e2e/multigpu/patched/ \ /workspace/axolotl/tests/e2e/multigpu/ \ - --cov=axolotl \ - --cov-report=xml:multigpu-coverage.xml + --cov=axolotl -pytest -v --durations=10 -n1 /workspace/axolotl/tests/e2e/multigpu/solo/ \ +# Run solo tests with coverage append +pytest -v --durations=10 -n1 \ + /workspace/axolotl/tests/e2e/multigpu/solo/ \ --cov=axolotl \ - --cov-append \ - --cov-report=xml:multigpu-coverage.xml + --cov-append pytest -v --durations=10 -n1 /workspace/axolotl/tests/e2e/multigpu/patched/ \ --cov=axolotl \ @@ -20,8 +20,4 @@ pytest -v --durations=10 -n1 /workspace/axolotl/tests/e2e/multigpu/patched/ \ --cov-report=xml:multigpu-coverage.xml # Upload coverage to Codecov -if [ -f multigpu-coverage.xml ]; then - codecov -f multigpu-coverage.xml -F multigpu,docker-tests,pytorch-${PYTORCH_VERSION} -else - echo "Coverage file not found. Coverage report may have failed." -fi +codecov upload-process -t $CODECOV_TOKEN -f multigpu-coverage.xml -F multigpu,docker-tests,pytorch-${PYTORCH_VERSION} diff --git a/requirements-tests.txt b/requirements-tests.txt index 43e3f88f4..93b2ceee5 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,4 +1,5 @@ codecov +codecov-cli pytest pytest-cov pytest-retry