tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

zero3 can'y use 8bit optimizer

Browse Source
This commit is contained in:
Wing Lian
2024-10-31 12:12:25 -04:00
committed by Wing Lian
parent 920ea77bdf
commit 127953af4e
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/e2e/multigpu/test_llama.py
View File

@@ -637,7 +637,7 @@ class TestMultiGPULlama:
"gradient_accumulation_steps": 4, "gradient_accumulation_steps": 4,
"output_dir": temp_dir, "output_dir": temp_dir,
"learning_rate": 0.00001, "learning_rate": 0.00001,
"optimizer": "adamw_8bit", "optimizer": "adamw_torch",
"lr_scheduler": "cosine", "lr_scheduler": "cosine",
"flash_attention": True, "flash_attention": True,
"deepspeed": "deepspeed_configs/zero3_bf16_cpuoffload_all.json", "deepspeed": "deepspeed_configs/zero3_bf16_cpuoffload_all.json",