From f776f889a164b4440debd3c7d2f35f8356aaea44 Mon Sep 17 00:00:00 2001 From: Dan Saunders Date: Wed, 16 Apr 2025 18:02:17 -0400 Subject: [PATCH] adding codecov reporting (#2372) [skip ci] * adding codecov reporting * update codecov-action to v5 * fix --------- Co-authored-by: Dan Saunders --- .coveragerc | 14 +++++++++ .github/workflows/tests.yml | 13 ++++++-- README.md | 1 + cicd/cicd.sh | 63 ++++++++++++++++++++++++++++++++----- cicd/multigpu.sh | 19 +++++++++++ codecov.yml | 51 ++++++++++++++++++++++++++++++ requirements-dev.txt | 2 +- requirements-tests.txt | 4 ++- 8 files changed, 155 insertions(+), 12 deletions(-) create mode 100644 .coveragerc create mode 100644 codecov.yml diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..c5670410c --- /dev/null +++ b/.coveragerc @@ -0,0 +1,14 @@ +[run] +source = axolotl +omit = + */tests/* + setup.py + +[report] +exclude_lines = + pragma: no cover + def __repr__ + raise NotImplementedError + if __name__ == .__main__.: + pass + raise ImportError diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9eb85a5b1..863687128 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -102,9 +102,16 @@ jobs: - name: Run tests run: | - pytest -v -n8 --dist loadfile --ignore=tests/e2e/ --ignore=tests/patched/ --ignore=tests/cli/ tests/ - pytest -v tests/patched/ - pytest -v tests/cli/ + pytest -v -n8 --dist loadfile --ignore=tests/e2e/ --ignore=tests/patched/ --ignore=tests/cli/ tests/ --cov=axolotl --cov-report=xml + pytest -v tests/patched/ --cov=axolotl --cov-append --cov-report=xml + pytest -v tests/cli/ --cov=axolotl --cov-append --cov-report=xml + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + files: ./coverage.xml + flags: unittests,pytorch-${{ matrix.pytorch_version }} + fail_ci_if_error: false - name: cleanup pip cache run: | diff --git a/README.md b/README.md index 0f2907df4..56e45e3fe 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@

GitHub License tests + codecov Releases
contributors diff --git a/cicd/cicd.sh b/cicd/cicd.sh index 1d9ea7fbe..3ae4896ac 100755 --- a/cicd/cicd.sh +++ b/cicd/cicd.sh @@ -3,10 +3,59 @@ set -e python -c "import torch; assert '$PYTORCH_VERSION' in torch.__version__" -pytest -v --durations=10 -n8 --ignore=tests/e2e/ --ignore=tests/patched/ --ignore=tests/cli /workspace/axolotl/tests/ -pytest -v --durations=10 /workspace/axolotl/tests/e2e/patched/lora_kernels # running these with the other patches causes a failure -pytest -v --durations=10 --ignore=tests/e2e/patched/lora_kernels /workspace/axolotl/tests/e2e/patched -pytest -v --durations=10 -n1 /workspace/axolotl/tests/e2e/solo/ -pytest -v --durations=10 /workspace/axolotl/tests/e2e/integrations/ -pytest -v --durations=10 /workspace/axolotl/tests/cli -pytest -v --durations=10 --ignore=tests/e2e/solo/ --ignore=tests/e2e/patched/ --ignore=tests/e2e/multigpu/ --ignore=tests/e2e/integrations/ --ignore=tests/cli /workspace/axolotl/tests/e2e/ +# Run unit tests with initial coverage report +pytest -v --durations=10 -n8 \ + --ignore=tests/e2e/ \ + --ignore=tests/patched/ \ + --ignore=tests/cli \ + /workspace/axolotl/tests/ \ + --cov=axolotl \ + --cov-report=xml:coverage.xml + +# Run lora kernels tests with coverage append +pytest -v --durations=10 \ + /workspace/axolotl/tests/e2e/patched/lora_kernels \ + --cov=axolotl \ + --cov-append + +# Run patched tests excluding lora kernels with coverage append +pytest -v --durations=10 \ + --ignore=tests/e2e/patched/lora_kernels \ + /workspace/axolotl/tests/e2e/patched \ + --cov=axolotl \ + --cov-append + +# Run solo tests with coverage append +pytest -v --durations=10 -n1 \ + /workspace/axolotl/tests/e2e/solo/ \ + --cov=axolotl \ + --cov-append + +# Run integration tests with coverage append +pytest -v --durations=10 \ + /workspace/axolotl/tests/e2e/integrations/ \ + --cov=axolotl \ + --cov-append + +pytest -v --durations=10 /workspace/axolotl/tests/cli \ + --cov=axolotl \ + --cov-append + +# Run remaining e2e tests with coverage append and final report +pytest -v --durations=10 \ + --ignore=tests/e2e/solo/ \ + --ignore=tests/e2e/patched/ \ + --ignore=tests/e2e/multigpu/ \ + --ignore=tests/e2e/integrations/ \ + --ignore=tests/cli \ + /workspace/axolotl/tests/e2e/ \ + --cov=axolotl \ + --cov-append \ + --cov-report=xml:coverage.xml + +# Upload coverage to Codecov +if [ -f e2e-coverage.xml ]; then + codecov -f e2e-coverage.xml -F e2e,pytorch-${PYTORCH_VERSION} +else + echo "Coverage file not found. Coverage report may have failed." +fi diff --git a/cicd/multigpu.sh b/cicd/multigpu.sh index 008a74bff..898cf1155 100755 --- a/cicd/multigpu.sh +++ b/cicd/multigpu.sh @@ -4,3 +4,22 @@ set -e # only run one test at a time so as not to OOM the GPU pytest -v --durations=10 -n2 /workspace/axolotl/tests/e2e/multigpu/ --ignore=/workspace/axolotl/tests/e2e/multigpu/solo/ pytest -v --durations=10 -n1 /workspace/axolotl/tests/e2e/multigpu/solo/ + +# Only run two tests at a time to avoid OOM on GPU (with coverage collection) +pytest -v -n2 \ + --ignore=/workspace/axolotl/tests/e2e/multigpu/solo/ + /workspace/axolotl/tests/e2e/multigpu/ \ + --cov=axolotl \ + --cov-report=xml:multigpu-coverage.xml + +pytest -v --durations=10 -n1 /workspace/axolotl/tests/e2e/multigpu/solo/ \ + --cov=axolotl \ + --cov-append \ + --cov-report=xml:multigpu-coverage.xml + +# Upload coverage to Codecov +if [ -f multigpu-coverage.xml ]; then + codecov -f multigpu-coverage.xml -F multigpu,docker-tests,pytorch-${PYTORCH_VERSION} +else + echo "Coverage file not found. Coverage report may have failed." +fi diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..6d47379e8 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,51 @@ +codecov: + require_ci_to_pass: yes + +coverage: + precision: 2 + round: down + range: "70...100" + status: + project: + default: + # basic + target: auto + threshold: 0% + base: auto + # advanced + branches: null + if_no_uploads: error + if_not_found: success + if_ci_failed: error + only_pulls: false + flags: null + paths: null + patch: + default: + # basic + target: auto + threshold: 0% + base: auto + # advanced + branches: null + if_no_uploads: error + if_not_found: success + if_ci_failed: error + only_pulls: false + flags: null + paths: null + +parsers: + gcov: + branch_detection: + conditional: yes + loop: yes + method: no + macro: no + +comment: + layout: "reach,diff,flags,files,footer" + behavior: default + require_changes: no + require_base: no + require_head: yes diff --git a/requirements-dev.txt b/requirements-dev.txt index 1dce5df5f..5c42d96d4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,6 @@ -pre-commit black mypy +pre-commit types-requests quartodoc jupyter diff --git a/requirements-tests.txt b/requirements-tests.txt index a13f73923..43e3f88f4 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -1,5 +1,7 @@ +codecov pytest -pytest-xdist +pytest-cov pytest-retry pytest-sugar +pytest-xdist tbparse