diff --git a/docker/Dockerfile b/docker/Dockerfile index ff47548bc..6eea7322c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,9 +21,9 @@ WORKDIR /workspace/axolotl # If AXOLOTL_EXTRAS is set, append it in brackets RUN sed -i "s/torch==.*/torch==$PYTORCH_VERSION/" requirements.txt RUN if [ "$AXOLOTL_EXTRAS" != "" ] ; then \ - pip install -e .[flash-attn,$AXOLOTL_EXTRAS]; \ + pip install -e .[deepspeed,flash-attn,$AXOLOTL_EXTRAS]; \ else \ - pip install -e .[flash-attn]; \ + pip install -e .[deepspeed,flash-attn]; \ fi # fix so that git fetch/pull from remote works diff --git a/docker/Dockerfile-base b/docker/Dockerfile-base index bb28dd839..ca43c9652 100644 --- a/docker/Dockerfile-base +++ b/docker/Dockerfile-base @@ -10,8 +10,10 @@ ENV PATH="/root/miniconda3/bin:${PATH}" ARG PYTHON_VERSION="3.9" ARG PYTORCH_VERSION="2.0.1" ARG CUDA="118" +ARG TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 9.0+PTX" ENV PYTHON_VERSION=$PYTHON_VERSION +ENV TORCH_CUDA_ARCH_LIST=$TORCH_CUDA_ARCH_LIST RUN apt-get update \ && apt-get install -y wget git build-essential ninja-build git-lfs libaio-dev && rm -rf /var/lib/apt/lists/* \ @@ -29,45 +31,7 @@ WORKDIR /workspace RUN python3 -m pip install --upgrade pip && pip3 install packaging && \ python3 -m pip install --no-cache-dir -U torch==${PYTORCH_VERSION}+cu${CUDA} deepspeed-kernels --extra-index-url https://download.pytorch.org/whl/cu$CUDA -FROM base-builder AS deepspeed-builder - -ARG TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 9.0+PTX" - -WORKDIR /workspace - -RUN git clone https://github.com/microsoft/DeepSpeed.git && \ - cd DeepSpeed && \ - MAX_CONCURRENCY=8 DS_BUILD_SPARSE_ATTN=0 DS_BUILD_OPS=1 DS_BUILD_EVOFORMER_ATTN=0 python3 setup.py bdist_wheel - -FROM base-builder AS bnb-builder - -WORKDIR /workspace -ARG CUDA="118" -ENV CUDA=$CUDA -ARG MAX_JOBS="-1" -ENV MAX_JOBS=$MAX_JOBS - -RUN git clone https://github.com/TimDettmers/bitsandbytes.git && \ - cd bitsandbytes && \ - CUDA_VERSION=$CUDA make cuda11x && \ - python setup.py bdist_wheel - -FROM base-builder - -ARG TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 9.0+PTX" -ENV TORCH_CUDA_ARCH_LIST=$TORCH_CUDA_ARCH_LIST - -RUN mkdir -p /workspace/builds -COPY --from=bnb-builder /workspace/bitsandbytes /workspace/builds/bitsandbytes - -RUN mkdir -p /workspace/wheels/bitsandbytes -COPY --from=deepspeed-builder /workspace/DeepSpeed/dist/deepspeed-*.whl wheels -COPY --from=bnb-builder /workspace/bitsandbytes/dist/bitsandbytes-*.whl wheels -COPY --from=bnb-builder /workspace/bitsandbytes/bitsandbytes/libbitsandbytes*.so wheels/bitsandbytes - -RUN pip3 install wheels/deepspeed-*.whl -RUN cd /workspace/builds/bitsandbytes && python3 setup.py install -RUN git lfs install --skip-repo -RUN pip3 install awscli && \ +RUN git lfs install --skip-repo && \ + pip3 install awscli && \ # The base image ships with `pydantic==1.8.2` which is not working pip3 install -U --no-cache-dir pydantic==1.10.10 diff --git a/examples/cerebras/btlm-ft.yml b/examples/cerebras/btlm-ft.yml index e598fc865..1fea9915e 100644 --- a/examples/cerebras/btlm-ft.yml +++ b/examples/cerebras/btlm-ft.yml @@ -14,7 +14,7 @@ datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca dataset_prepared_path: last_prepared_run -val_set_size: 0.01 +val_set_size: 0.05 adapter: lora_model_dir: diff --git a/examples/cerebras/qlora.yml b/examples/cerebras/qlora.yml index 2dc4160da..9f1dcc852 100644 --- a/examples/cerebras/qlora.yml +++ b/examples/cerebras/qlora.yml @@ -7,7 +7,7 @@ datasets: - path: teknium/GPT4-LLM-Cleaned type: alpaca dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 adapter: qlora lora_model_dir: sequence_len: 2048 diff --git a/examples/code-llama/13b/lora.yml b/examples/code-llama/13b/lora.yml index 26c5cae95..f3df1a1e2 100644 --- a/examples/code-llama/13b/lora.yml +++ b/examples/code-llama/13b/lora.yml @@ -11,7 +11,7 @@ datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 output_dir: ./lora-out sequence_len: 4096 diff --git a/examples/code-llama/13b/qlora.yml b/examples/code-llama/13b/qlora.yml index f6c1be56d..8bcd0dc78 100644 --- a/examples/code-llama/13b/qlora.yml +++ b/examples/code-llama/13b/qlora.yml @@ -11,7 +11,7 @@ datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 output_dir: ./qlora-out adapter: qlora diff --git a/examples/code-llama/34b/lora.yml b/examples/code-llama/34b/lora.yml index 1996e6c04..2eb9df481 100644 --- a/examples/code-llama/34b/lora.yml +++ b/examples/code-llama/34b/lora.yml @@ -11,7 +11,7 @@ datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 output_dir: ./lora-out sequence_len: 4096 diff --git a/examples/code-llama/34b/qlora.yml b/examples/code-llama/34b/qlora.yml index c854ceab7..3093ec01f 100644 --- a/examples/code-llama/34b/qlora.yml +++ b/examples/code-llama/34b/qlora.yml @@ -11,7 +11,7 @@ datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 output_dir: ./qlora-out adapter: qlora diff --git a/examples/code-llama/7b/lora.yml b/examples/code-llama/7b/lora.yml index a53123219..422351d9a 100644 --- a/examples/code-llama/7b/lora.yml +++ b/examples/code-llama/7b/lora.yml @@ -11,7 +11,7 @@ datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 output_dir: ./lora-out sequence_len: 4096 diff --git a/examples/code-llama/7b/qlora.yml b/examples/code-llama/7b/qlora.yml index 5ad53063c..f5712c009 100644 --- a/examples/code-llama/7b/qlora.yml +++ b/examples/code-llama/7b/qlora.yml @@ -11,7 +11,7 @@ datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 output_dir: ./qlora-out adapter: qlora diff --git a/examples/falcon/config-7b-lora.yml b/examples/falcon/config-7b-lora.yml index d46702152..25884410a 100644 --- a/examples/falcon/config-7b-lora.yml +++ b/examples/falcon/config-7b-lora.yml @@ -12,7 +12,7 @@ datasets: - path: teknium/GPT4-LLM-Cleaned type: alpaca:chat dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 adapter: lora lora_model_dir: sequence_len: 2048 diff --git a/examples/falcon/config-7b-qlora.yml b/examples/falcon/config-7b-qlora.yml index 78ffb43f6..8e90e6614 100644 --- a/examples/falcon/config-7b-qlora.yml +++ b/examples/falcon/config-7b-qlora.yml @@ -18,7 +18,7 @@ datasets: - Chain-of-Thought/formatted_cot_data/gsm8k_train.json type: "alpaca:chat" dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 # enable QLoRA adapter: qlora lora_model_dir: diff --git a/examples/falcon/config-7b.yml b/examples/falcon/config-7b.yml index 96039db60..fd5f63ccc 100644 --- a/examples/falcon/config-7b.yml +++ b/examples/falcon/config-7b.yml @@ -12,7 +12,7 @@ datasets: - path: teknium/GPT4-LLM-Cleaned type: alpaca:chat dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 adapter: lora_model_dir: sequence_len: 2048 diff --git a/examples/gptj/qlora.yml b/examples/gptj/qlora.yml index e887e15d5..57f132047 100644 --- a/examples/gptj/qlora.yml +++ b/examples/gptj/qlora.yml @@ -7,7 +7,7 @@ datasets: - path: teknium/GPT4-LLM-Cleaned type: alpaca dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 adapter: qlora lora_model_dir: sequence_len: 2048 diff --git a/examples/llama-2/fft_optimized.yml b/examples/llama-2/fft_optimized.yml index e7bbfc1c9..cf86a3e5c 100644 --- a/examples/llama-2/fft_optimized.yml +++ b/examples/llama-2/fft_optimized.yml @@ -11,7 +11,7 @@ datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca dataset_prepared_path: last_run_prepared -val_set_size: 0.01 +val_set_size: 0.05 output_dir: ./out sequence_len: 4096 diff --git a/examples/llama-2/gptq-lora.yml b/examples/llama-2/gptq-lora.yml index 2bce70f35..61b00992f 100644 --- a/examples/llama-2/gptq-lora.yml +++ b/examples/llama-2/gptq-lora.yml @@ -15,7 +15,7 @@ datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 adapter: lora lora_model_dir: sequence_len: 4096 diff --git a/examples/llama-2/lora.yml b/examples/llama-2/lora.yml index 8e1047c0b..7d50877c7 100644 --- a/examples/llama-2/lora.yml +++ b/examples/llama-2/lora.yml @@ -11,7 +11,7 @@ datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 output_dir: ./lora-out sequence_len: 4096 diff --git a/examples/llama-2/qlora.yml b/examples/llama-2/qlora.yml index afeb981ac..29b756ce5 100644 --- a/examples/llama-2/qlora.yml +++ b/examples/llama-2/qlora.yml @@ -11,7 +11,7 @@ datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 output_dir: ./qlora-out adapter: qlora diff --git a/examples/llama-2/relora.yml b/examples/llama-2/relora.yml index ffccb7c0f..8a7243d6f 100644 --- a/examples/llama-2/relora.yml +++ b/examples/llama-2/relora.yml @@ -11,7 +11,7 @@ datasets: - path: teknium/GPT4-LLM-Cleaned type: alpaca dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 output_dir: ./relora-out adapter: qlora diff --git a/examples/llama-2/tiny-llama.yml b/examples/llama-2/tiny-llama.yml index b249e9434..b91877e97 100644 --- a/examples/llama-2/tiny-llama.yml +++ b/examples/llama-2/tiny-llama.yml @@ -12,7 +12,7 @@ datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 output_dir: ./lora-out sequence_len: 4096 diff --git a/examples/mistral/config.yml b/examples/mistral/config.yml index 40a7e2f4e..f0f7dad0a 100644 --- a/examples/mistral/config.yml +++ b/examples/mistral/config.yml @@ -11,7 +11,7 @@ datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 output_dir: ./out sequence_len: 8192 diff --git a/examples/mistral/qlora.yml b/examples/mistral/qlora.yml index ff929dcb6..28c5ed242 100644 --- a/examples/mistral/qlora.yml +++ b/examples/mistral/qlora.yml @@ -11,7 +11,7 @@ datasets: - path: mhenrichsen/alpaca_2k_test type: alpaca dataset_prepared_path: last_run_prepared -val_set_size: 0.01 +val_set_size: 0.05 output_dir: ./qlora-out adapter: qlora diff --git a/examples/openllama-3b/qlora.yml b/examples/openllama-3b/qlora.yml index 9fe5968d2..89fbecde3 100644 --- a/examples/openllama-3b/qlora.yml +++ b/examples/openllama-3b/qlora.yml @@ -9,7 +9,7 @@ datasets: - path: teknium/GPT4-LLM-Cleaned type: alpaca dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 adapter: qlora lora_model_dir: sequence_len: 1024 diff --git a/examples/xgen-7b/xgen-7b-8k-qlora.yml b/examples/xgen-7b/xgen-7b-8k-qlora.yml index 40a62e6c9..524f4e993 100644 --- a/examples/xgen-7b/xgen-7b-8k-qlora.yml +++ b/examples/xgen-7b/xgen-7b-8k-qlora.yml @@ -16,7 +16,7 @@ datasets: - openassistant_best_replies_train.jsonl type: "completion" dataset_prepared_path: -val_set_size: 0.01 +val_set_size: 0.05 # enable QLoRA adapter: qlora lora_model_dir: diff --git a/requirements.txt b/requirements.txt index f478481e9..f3692468e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ torch==2.0.1 auto-gptq==0.4.2 packaging -peft @ git+https://github.com/huggingface/peft.git +peft==0.6.0 transformers @ git+https://github.com/huggingface/transformers.git@acc394c4f5e1283c19783581790b3dc3105a3697 bitsandbytes>=0.41.1 accelerate @ git+https://github.com/huggingface/accelerate@80da9cfb09bb3cc9f1b385cb55d6b90d025a5fd9