From 312b8d51d6986d563c0aa396e7b5cee46875cb3b Mon Sep 17 00:00:00 2001
From: Wing Lian <wing.lian@gmail.com>
Date: Sat, 27 May 2023 12:36:53 -0400
Subject: [PATCH 1/2] update docker to compile latest bnb to properly support
 qlora

---
 .github/workflows/base.yml |  3 +++
 docker/Dockerfile-base     | 13 ++++++++++++-
 requirements.txt           |  6 ++++++
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml
index 449adec35..571faf771 100644
--- a/.github/workflows/base.yml
+++ b/.github/workflows/base.yml
@@ -16,9 +16,11 @@ jobs:
         include:
           - cuda: cu118
             cuda_version: 11.8.0
+            cuda_version_bnb: 118
             pytorch: 2.0.0
           - cuda: cu117
             cuda_version: 11.7.0
+            cuda_version_bnb: 117
             pytorch: 1.13.1
     steps:
       - name: Checkout
@@ -47,5 +49,6 @@ jobs:
           cache-to: type=gha,mode=max
           build-args: |
             CUDA_VERSION=${{ matrix.cuda_version }}
+            CUDA_VERSION_BNB=${{ matrix.cuda_version_bnb }}
             CUDA=${{ matrix.cuda }}
             PYTORCH_VERSION=${{ matrix.pytorch }}
diff --git a/docker/Dockerfile-base b/docker/Dockerfile-base
index d2abd9bca..f35dbfc2e 100644
--- a/docker/Dockerfile-base
+++ b/docker/Dockerfile-base
@@ -1,4 +1,5 @@
 ARG CUDA_VERSION="11.8.0"
+ARG CUDA_VERSION_BNB="118"
 ARG CUDNN_VERSION="8"
 ARG UBUNTU_VERSION="22.04"
 ARG MAX_JOBS=4
@@ -58,6 +59,15 @@ RUN git clone https://github.com/microsoft/DeepSpeed.git && \
     cd DeepSpeed && \
     MAX_CONCURRENCY=8 DS_BUILD_SPARSE_ATTN=0 DS_BUILD_OPS=1 python3 setup.py bdist_wheel
 
+FROM base-builder AS bnb-builder
+
+WORKDIR /workspace
+
+RUN git clone https://github.com/TimDettmers/bitsandbytes.git && \
+    cd bitsandbytes && \
+    CUDA_VERSION=$CUDA_VERSION_BNB make cuda11x \
+    python setup.py bdist_wheel
+
 FROM base-builder
 
 # recompile apex
@@ -68,13 +78,14 @@ RUN cd apex && MAX_JOBS=1 python3 -m pip install --global-option="--cpp_ext" --g
 
 RUN mkdir /workspace/wheels
 COPY --from=deepspeed-builder /workspace/DeepSpeed/dist/deepspeed-*.whl wheels
+COPY --from=bnb-builder /workspace/bitsandbytes/dist/bitsandbytes-*.whl wheels
 COPY --from=flash-attn-builder /workspace/flash-attention/dist/flash_attn-*.whl wheels
 COPY --from=flash-attn-builder /workspace/flash-attention/csrc/fused_dense_lib/dist/fused_dense_lib-*.whl wheels
 COPY --from=flash-attn-builder /workspace/flash-attention/csrc/xentropy/dist/xentropy_cuda_lib-*.whl wheels
 COPY --from=flash-attn-builder /workspace/flash-attention/csrc/rotary/dist/rotary_emb-*.whl wheels
 COPY --from=flash-attn-builder /workspace/flash-attention/csrc/layer_norm/dist/dropout_layer_norm-*.whl wheels
 
-RUN pip3 install wheels/deepspeed-*.whl wheels/flash_attn-*.whl wheels/fused_dense_lib-*.whl wheels/xentropy_cuda_lib-*.whl wheels/rotary_emb-*.whl wheels/dropout_layer_norm-*.whl
+RUN pip3 install wheels/deepspeed-*.whl wheels/flash_attn-*.whl wheels/fused_dense_lib-*.whl wheels/xentropy_cuda_lib-*.whl wheels/rotary_emb-*.whl wheels/dropout_layer_norm-*.whl wheels/bitsandbytes-*.whl
 RUN git lfs install --skip-repo
 RUN pip3 install "peft @ git+https://github.com/huggingface/peft.git@main" \
         "accelerate @ git+https://github.com/huggingface/accelerate.git@main" \
diff --git a/requirements.txt b/requirements.txt
index 9703000eb..1af103e17 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,3 +11,9 @@ sentencepiece
 wandb
 einops
 xformers
+# qlora things
+bert-score==0.3.13
+evaluate==0.4.0
+rouge-score==0.1.2
+scipy
+scikit-learn==1.2.2

From cf379803958ed3580fd1c52e6837983a8e69a675 Mon Sep 17 00:00:00 2001
From: Wing Lian <wing.lian@gmail.com>
Date: Sat, 27 May 2023 15:28:54 -0400
Subject: [PATCH 2/2] fix missing run coninuation

---
 docker/Dockerfile-base | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/Dockerfile-base b/docker/Dockerfile-base
index f35dbfc2e..b775808bf 100644
--- a/docker/Dockerfile-base
+++ b/docker/Dockerfile-base
@@ -65,7 +65,7 @@ WORKDIR /workspace
 
 RUN git clone https://github.com/TimDettmers/bitsandbytes.git && \
     cd bitsandbytes && \
-    CUDA_VERSION=$CUDA_VERSION_BNB make cuda11x \
+    CUDA_VERSION=$CUDA_VERSION_BNB make cuda11x && \
     python setup.py bdist_wheel
 
 FROM base-builder