diff --git a/docker/Dockerfile-base b/docker/Dockerfile-base
index 486d17d2c..4fd65724e 100644
--- a/docker/Dockerfile-base
+++ b/docker/Dockerfile-base
@@ -51,8 +51,15 @@ RUN git clone https://github.com/microsoft/DeepSpeed.git && \
 
 FROM base-builder
 
+# recompile apex
+RUN python3 -m pip uninstall -y apex
+RUN git clone https://github.com/NVIDIA/apex
+#  `MAX_JOBS=1` disables parallel building to avoid cpu memory OOM when building image on GitHub Action (standard) runners
+RUN cd apex && MAX_JOBS=1 python3 -m pip install --global-option="--cpp_ext" --global-option="--cuda_ext" --no-cache -v --disable-pip-version-check .
+
 RUN mkdir /workspace/wheels
 COPY --from=deepspeed-builder /workspace/DeepSpeed/dist/deepspeed-*.whl wheels
 COPY --from=flash-attn-builder /workspace/flash-attention/dist/flash_attn-*.whl wheels
 
-RUN pip3 install.sh wheels/deepspeed-*.whl wheels/flash_attn-*.whl
+RUN pip3 install wheels/deepspeed-*.whl wheels/flash_attn-*.whl
+RUN git lfs install --skip-repo