tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix zero3 integration (#1897)

Browse Source 
* fix zero3 integration

* bump transformers and accelerate too
This commit is contained in:
Wing Lian
2024-09-05 10:58:50 -04:00
committed by GitHub
parent ab461d83c4
commit 6e354682e3
2 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
requirements.txt
View File

@@ -1,10 +1,10 @@
--extra-index-url https://huggingface.github.io/autogptq-index/whl/cu118/ --extra-index-url https://huggingface.github.io/autogptq-index/whl/cu118/
packaging==23.2 packaging==23.2
peft==0.12.0 peft==0.12.0
transformers==4.44.0 transformers==4.44.2
tokenizers>=0.19.1 tokenizers>=0.19.1
bitsandbytes==0.43.3 bitsandbytes==0.43.3
accelerate==0.33.0 accelerate==0.34.0
datasets==2.20.0 datasets==2.20.0
deepspeed==0.14.4 deepspeed==0.14.4
pydantic==2.6.3 pydantic==2.6.3

3
src/axolotl/utils/trainer.py
View File

@@ -425,7 +425,8 @@ def setup_deepspeed_env(cfg, stage=None):
os.environ["ACCELERATE_DEEPSPEED_ZERO_STAGE"] = str(stage) os.environ["ACCELERATE_DEEPSPEED_ZERO_STAGE"] = str(stage)
if stage == 3: if stage == 3:
os.environ["ACCELERATE_DEEPSPEED_ZERO3_INIT"] = "true" os.environ["ACCELERATE_DEEPSPEED_ZERO3_INIT"] = "true"
HfTrainerDeepSpeedConfig(cfg.deepspeed) # If we don't assign this, it doesn't actually get set in the accelerate weakref
_ = HfTrainerDeepSpeedConfig(cfg.deepspeed)
def setup_fsdp_envs(cfg): def setup_fsdp_envs(cfg):