Merge branch 'main' into codecov-pulls-only

.github/workflows/tests.yml

@@ -106,13 +106,12 @@ jobs:
           pytest -v tests/patched/ --cov=axolotl --cov-append --cov-report=xml
           pytest -v tests/cli/ --cov=axolotl --cov-append --cov-report=xml
 
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v5
+      - name: Upload coverage artifacts
+        uses: actions/upload-artifact@v4
         with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./coverage.xml
-          flags: unittests,pytorch-${{ matrix.pytorch_version }}
-          fail_ci_if_error: false
+          name: coverage-${{ matrix.pytorch_version }}-${{ github.run_id }}
+          path: ./coverage.xml
+          retention-days: 1
 
       - name: cleanup pip cache
         run: |
@@ -234,6 +233,14 @@ jobs:
         run: |
           modal run cicd.e2e_tests
 
+      - name: Upload coverage artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-e2e-1st-${{ github.run_id }}
+          path: ./e2e-coverage.xml
+          retention-days: 1
+
   docker-e2e-tests:
     if: github.repository_owner == 'axolotl-ai-cloud'
     # this job needs to be run on self-hosted GPU runners...
@@ -297,6 +304,14 @@ jobs:
         run: |
           modal run cicd.e2e_tests
 
+      - name: Upload coverage artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-e2e-${{ matrix.cuda }}-${{ matrix.pytorch }}-${{ github.run_id }}
+          path: ./e2e-coverage.xml
+          retention-days: 1
+
   docker-e2e-cleanup:
     runs-on: [self-hosted, modal]
     timeout-minutes: 90
@@ -336,3 +351,26 @@ jobs:
       - name: Run tests job on Modal
         run: |
           modal run cicd.cleanup
+
+  upload-coverage:
+    name: Upload Coverage to Codecov
+    runs-on: ubuntu-latest
+    needs: [pytest, docker-e2e-tests, docker-e2e-tests-1st]
+    if: github.event_name == 'pull_request' || github.ref == 'refs/heads/main'
+
+    steps:
+      - name: Download coverage reports
+        uses: actions/download-artifact@v4
+        with:
+          path: coverage-reports
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          directory: coverage-reports
+          fail_ci_if_error: false
+          verbose: true
+          name: codecov-umbrella
+          override_commit: ${{ github.event.pull_request.head.sha || github.sha }}
+          override_pr: ${{ github.event.pull_request.number }}

cicd/cicd.sh

@@ -51,5 +51,3 @@ pytest -v --durations=10 \
   --cov=axolotl \
   --cov-append \
   --cov-report=xml:e2e-coverage.xml
-
-codecov upload-process -t $CODECOV_TOKEN -f e2e-coverage.xml -F e2e,pytorch-${PYTORCH_VERSION} || true

cicd/e2e_tests.py

@@ -1,5 +1,7 @@
 """Modal app to run axolotl GPU tests"""
 
+import pathlib
+
 from .single_gpu import GPU_CONFIG, VOLUME_CONFIG, app, cicd_image, run_cmd
 
 
@@ -12,9 +14,21 @@ from .single_gpu import GPU_CONFIG, VOLUME_CONFIG, app, cicd_image, run_cmd
     volumes=VOLUME_CONFIG,
 )
 def cicd_pytest():
+
     run_cmd("./cicd/cicd.sh", "/workspace/axolotl")
 
+    # Read the coverage file if it exists
+    coverage_file = pathlib.Path("/workspace/axolotl/e2e-coverage.xml")
+    if coverage_file.exists():
+        return coverage_file.read_text(encoding="utf-8")
+    return None
+
 
 @app.local_entrypoint()
 def main():
-    cicd_pytest.remote()
+    coverage = cicd_pytest.remote()
+
+    # Save the coverage file to the local filesystem if it was generated
+    if coverage:
+        with open("e2e-coverage.xml", "w", encoding="utf-8") as f:
+            f.write(coverage)

cicd/multigpu.py

@@ -77,7 +77,18 @@ def run_cmd(cmd: str, run_folder: str):
 def cicd_pytest():
     run_cmd("./cicd/multigpu.sh", "/workspace/axolotl")
 
+    # Read the coverage file if it exists
+    coverage_file = pathlib.Path("/workspace/axolotl/multigpu-coverage.xml")
+    if coverage_file.exists():
+        return coverage_file.read_text(encoding="utf-8")
+    return None
+
 
 @app.local_entrypoint()
 def main():
-    cicd_pytest.remote()
+    coverage = cicd_pytest.remote()
+
+    # Save the coverage file to the local filesystem if it was generated
+    if coverage:
+        with open("multigpu-coverage.xml", "w", encoding="utf-8") as file:
+            file.write(coverage)