diff --git a/docker/Dockerfile-base b/docker/Dockerfile-base
index ca43c9652..e8f3c7f4e 100644
--- a/docker/Dockerfile-base
+++ b/docker/Dockerfile-base
@@ -29,7 +29,7 @@ ENV PATH="/root/miniconda3/envs/py${PYTHON_VERSION}/bin:${PATH}"
 WORKDIR /workspace
 
 RUN python3 -m pip install --upgrade pip && pip3 install packaging && \
-    python3 -m pip install --no-cache-dir -U torch==${PYTORCH_VERSION}+cu${CUDA} deepspeed-kernels --extra-index-url https://download.pytorch.org/whl/cu$CUDA
+    python3 -m pip install --no-cache-dir -U torch==${PYTORCH_VERSION}+cu${CUDA} --extra-index-url https://download.pytorch.org/whl/cu$CUDA
 
 RUN git lfs install --skip-repo && \
     pip3 install awscli && \
diff --git a/requirements.txt b/requirements.txt
index 41cc8e105..1dab2b5cb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,7 +5,7 @@ transformers==4.37.0
 tokenizers==0.15.0
 bitsandbytes>=0.41.1
 accelerate==0.26.1
-deepspeed
+deepspeed>=0.13.1
 addict
 fire
 PyYAML>=6.0
diff --git a/setup.py b/setup.py
index 235018dcc..5a0a231d9 100644
--- a/setup.py
+++ b/setup.py
@@ -56,7 +56,8 @@ setup(
             "fused-dense-lib  @ git+https://github.com/Dao-AILab/flash-attention@v2.3.3#subdirectory=csrc/fused_dense_lib",
         ],
         "deepspeed": [
-            "deepspeed",
+            "deepspeed>=0.13.1",
+            "deepspeed-kernels",
         ],
         "mamba-ssm": [
             "mamba-ssm==1.0.1",