From 2f20cb7ebfc6cdc100196c7a72288470320df009 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Fri, 15 Nov 2024 19:08:38 -0500 Subject: [PATCH] upgrade datasets==3.1.0 and add upstream check (#2067) [skip ci] --- .github/workflows/tests-nightly.yml | 1 + cicd/Dockerfile.jinja | 1 + requirements.txt | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests-nightly.yml b/.github/workflows/tests-nightly.yml index 52c28e919..bb9927888 100644 --- a/.github/workflows/tests-nightly.yml +++ b/.github/workflows/tests-nightly.yml @@ -48,6 +48,7 @@ jobs: sed -i 's#^peft.*#peft @ git+https://github.com/huggingface/peft.git@main#' requirements.txt sed -i 's#^accelerate.*#accelerate @ git+https://github.com/huggingface/accelerate.git@main#' requirements.txt sed -i 's#^trl.*#trl @ git+https://github.com/huggingface/trl.git@main#' requirements.txt + sed -i 's#^datasets.*#datasets @ git+https://github.com/huggingface/datasets.git@main#' requirements.txt - name: Install dependencies run: | diff --git a/cicd/Dockerfile.jinja b/cicd/Dockerfile.jinja index 8fe84ac64..8fd040d77 100644 --- a/cicd/Dockerfile.jinja +++ b/cicd/Dockerfile.jinja @@ -28,6 +28,7 @@ RUN if [ "$NIGHTLY_BUILD" = "true" ] ; then \ sed -i 's#^peft.*#peft @ git+https://github.com/huggingface/peft.git@main#' requirements.txt; \ sed -i 's#^accelerate.*#accelerate @ git+https://github.com/huggingface/accelerate.git@main#' requirements.txt; \ sed -i 's#^trl.*#trl @ git+https://github.com/huggingface/trl.git@main#' requirements.txt; \ + sed -i 's#^datasets.*#datasets @ git+https://github.com/huggingface/datasets.git@main#' requirements.txt; \ fi RUN if [ "$AXOLOTL_EXTRAS" != "" ] ; then \ diff --git a/requirements.txt b/requirements.txt index 76c3273fc..aab4d0ead 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ transformers==4.46.1 tokenizers>=0.20.1 bitsandbytes==0.44.1 accelerate==1.1.0 -datasets==3.0.1 +datasets==3.1.0 deepspeed==0.15.3 pydantic==2.6.3 addict