diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 56370035c..e8ae97f7a 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -8,7 +8,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04-4core
+    runs-on: self-hosted
     steps:
       - name: Checkout
         uses: actions/checkout@v3
diff --git a/Dockerfile b/Dockerfile
index 824a04560..f29b9adf9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,17 +3,20 @@ FROM huggingface/transformers-pytorch-deepspeed-latest-gpu:latest
 RUN export TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6+PTX"
 RUN apt-get update && \
     apt-get install -y build-essential ninja-build vim git-lfs && \
-    git lfs install && \
-    pip3 install --force-reinstall https://download.pytorch.org/whl/nightly/cu117/torch-2.0.0.dev20230301%2Bcu117-cp38-cp38-linux_x86_64.whl --index-url https://download.pytorch.org/whl/nightly/cu117 && \
-    mdir /tmp/wheels && \
+    git lfs install --skip-repo && \
+    mkdir /tmp/wheels && \
     cd /tmp/wheels && \
     curl -L -O https://github.com/winglian/axolotl/raw/wheels/wheels/deepspeed-0.9.2%2B7ddc3b01-cp38-cp38-linux_x86_64.whl && \
     curl -L -O https://github.com/winglian/axolotl/raw/wheels/wheels/flash_attn-1.0.4-cp38-cp38-linux_x86_64.whl && \
     pip install deepspeed-0.9.2%2B7ddc3b01-cp38-cp38-linux_x86_64.whl && \
     pip install flash_attn-1.0.4-cp38-cp38-linux_x86_64.whl && \
+    pip install wheel && \
     pip install "peft @ git+https://github.com/huggingface/peft.git@main" --force-reinstall --no-dependencies
 
 WORKDIR /workspace
 ARG REF=main
-RUN git clone https://github.com/winglian/axolotl && cd axolotl && git checkout $REF
-RUN pip install -e .[int4]
+RUN git clone https://github.com/winglian/axolotl && cd axolotl && git checkout $REF && \
+    pip install -e .[int4]
+
+RUN pip3 install --force-reinstall https://download.pytorch.org/whl/nightly/cu117/torch-2.0.0.dev20230301%2Bcu117-cp38-cp38-linux_x86_64.whl --index-url https://download.pytorch.org/whl/nightly/cu117
+