diff --git a/scripts/setup-runpod.sh b/scripts/setup-runpod.sh
index cc1212dc3..d6ce6c280 100644
--- a/scripts/setup-runpod.sh
+++ b/scripts/setup-runpod.sh
@@ -26,6 +26,14 @@ if [ -z "${TORCH_CUDA_ARCH_LIST}" ]; then # only set this if not set yet
     export TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6+PTX"
 fi
 
+# install flash-attn and deepspeed from pre-built wheels for this specific container b/c these take forever to install
+mkdir -p /workspace/wheels
+cd /workspace/wheels
+curl -L -O https://github.com/winglian/axolotl/raw/wheels/wheels/deepspeed-0.9.2%2B7ddc3b01-cp38-cp38-linux_x86_64.whl
+curl -L -O https://github.com/winglian/axolotl/raw/wheels/wheels/flash_attn-1.0.4-cp38-cp38-linux_x86_64.whl
+pip install deepspeed-0.9.2%2B7ddc3b01-cp38-cp38-linux_x86_64.whl
+pip install flash_attn-1.0.4-cp38-cp38-linux_x86_64.whl
+
 cd /workspace/
 git clone https://github.com/winglian/axolotl.git
 cd axolotl