diff --git a/cicd/multigpu.sh b/cicd/multigpu.sh
index 898cf1155..519ac0133 100755
--- a/cicd/multigpu.sh
+++ b/cicd/multigpu.sh
@@ -1,13 +1,10 @@
 #!/bin/bash
 set -e
 
-# only run one test at a time so as not to OOM the GPU
-pytest -v  --durations=10 -n2 /workspace/axolotl/tests/e2e/multigpu/ --ignore=/workspace/axolotl/tests/e2e/multigpu/solo/
-pytest -v  --durations=10 -n1 /workspace/axolotl/tests/e2e/multigpu/solo/
-
 # Only run two tests at a time to avoid OOM on GPU (with coverage collection)
 pytest -v -n2 \
-  --ignore=/workspace/axolotl/tests/e2e/multigpu/solo/
+  --ignore=/workspace/axolotl/tests/e2e/multigpu/solo/ \
+  --ignore=/workspace/axolotl/tests/e2e/multigpu/patched/ \
   /workspace/axolotl/tests/e2e/multigpu/ \
   --cov=axolotl \
   --cov-report=xml:multigpu-coverage.xml
@@ -17,6 +14,11 @@ pytest -v  --durations=10 -n1 /workspace/axolotl/tests/e2e/multigpu/solo/ \
   --cov-append \
   --cov-report=xml:multigpu-coverage.xml
 
+pytest -v  --durations=10 -n1 /workspace/axolotl/tests/e2e/multigpu/patched/ \
+  --cov=axolotl \
+  --cov-append \
+  --cov-report=xml:multigpu-coverage.xml
+
 # Upload coverage to Codecov
 if [ -f multigpu-coverage.xml ]; then
   codecov -f multigpu-coverage.xml -F multigpu,docker-tests,pytorch-${PYTORCH_VERSION}
diff --git a/tests/e2e/multigpu/patched/__init__.py b/tests/e2e/multigpu/patched/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/e2e/multigpu/test_sp.py b/tests/e2e/multigpu/patched/test_sp.py
similarity index 99%
rename from tests/e2e/multigpu/test_sp.py
rename to tests/e2e/multigpu/patched/test_sp.py
index 72e5cb88c..1667408f4 100644
--- a/tests/e2e/multigpu/test_sp.py
+++ b/tests/e2e/multigpu/patched/test_sp.py
@@ -10,7 +10,7 @@ from transformers.testing_utils import get_torch_dist_unique_port
 
 from axolotl.utils.dict import DictDefault
 
-from ..utils import check_tensorboard
+from ...utils import check_tensorboard
 
 os.environ["WANDB_DISABLED"] = "true"