tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

upgrade datasets to 4.4.1 (#3266)

Browse Source 
* upgrade datasets

* cleanup pip cache earlier

* cleanup unused things from worker

* also cleanup sdist
This commit is contained in:
Wing Lian
2025-11-14 12:52:14 -05:00
committed by GitHub
parent 0fbde69e9c
commit a6bafb55cb
2 changed files with 17 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
.github/workflows/tests.yml vendored
View File

@@ -59,6 +59,10 @@ jobs:
timeout-minutes: 20 timeout-minutes: 20
steps: steps:
- name: cleanup node
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v4 uses: actions/checkout@v4
@@ -91,6 +95,10 @@ jobs:
python scripts/cutcrossentropy_install.py | sh python scripts/cutcrossentropy_install.py | sh
pip3 install -r requirements-dev.txt -r requirements-tests.txt pip3 install -r requirements-dev.txt -r requirements-tests.txt
- name: cleanup pip cache
run: |
find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \;
- name: Make sure PyTorch version wasn't clobbered - name: Make sure PyTorch version wasn't clobbered
run: | run: |
python -c "import torch; assert '${{ matrix.pytorch_version }}' in torch.__version__" python -c "import torch; assert '${{ matrix.pytorch_version }}' in torch.__version__"
@@ -118,10 +126,6 @@ jobs:
flags: unittests,pytorch-${{ matrix.pytorch_version }} flags: unittests,pytorch-${{ matrix.pytorch_version }}
fail_ci_if_error: false fail_ci_if_error: false
- name: cleanup pip cache
run: |
find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \;
pytest-sdist: pytest-sdist:
name: PyTest from Source Dist name: PyTest from Source Dist
runs-on: ubuntu-latest runs-on: ubuntu-latest
@@ -134,6 +138,10 @@ jobs:
timeout-minutes: 20 timeout-minutes: 20
steps: steps:
- name: cleanup node
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
- name: Check out repository code - name: Check out repository code
uses: actions/checkout@v4 uses: actions/checkout@v4
@@ -167,6 +175,10 @@ jobs:
python scripts/cutcrossentropy_install.py | sh python scripts/cutcrossentropy_install.py | sh
pip3 install -r requirements-dev.txt -r requirements-tests.txt pip3 install -r requirements-dev.txt -r requirements-tests.txt
- name: cleanup pip cache
run: |
find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \;
- name: Make sure PyTorch version wasn't clobbered - name: Make sure PyTorch version wasn't clobbered
run: | run: |
python -c "import torch; assert '${{ matrix.pytorch_version }}' in torch.__version__" python -c "import torch; assert '${{ matrix.pytorch_version }}' in torch.__version__"
@@ -184,10 +196,6 @@ jobs:
pytest -v --durations=10 tests/monkeypatch/ --cov=axolotl --cov-append --cov-report=xml pytest -v --durations=10 tests/monkeypatch/ --cov=axolotl --cov-append --cov-report=xml
pytest -v --durations=10 tests/cli/ pytest -v --durations=10 tests/cli/
- name: cleanup pip cache
run: |
find "$(pip cache dir)/http-v2" -type f -mtime +14 -exec rm {} \;
gate-skip-e2e: gate-skip-e2e:
needs: [pre-commit, pytest, pytest-sdist] needs: [pre-commit, pytest, pytest-sdist]
runs-on: ubuntu-latest runs-on: ubuntu-latest

2
requirements.txt
View File

@@ -15,7 +15,7 @@ peft>=0.17.1
tokenizers>=0.22.1 tokenizers>=0.22.1
transformers==4.57.1 transformers==4.57.1
accelerate==1.11.0 accelerate==1.11.0
datasets==4.3.0 datasets==4.4.1
deepspeed>=0.17.0 deepspeed>=0.17.0
trl==0.25.0 trl==0.25.0
hf_xet==1.2.0 hf_xet==1.2.0