Mixtral fixes 20240124 (#1192) [skip ci]

* mixtral nccl fixes * make sure to patch for z3
2024-01-24 14:59:57 -05:00
parent af0243021c
commit 54d2ac155b
14 changed files with 71 additions and 13 deletions
--- a/examples/mistral/Mistral-7b-example/config.yml
+++ b/examples/mistral/Mistral-7b-example/config.yml
@@ -65,7 +65,7 @@ eval_table_max_new_tokens: 128
 saves_per_epoch: 1
 debug:
 #default deepspeed, can use more aggresive if needed like zero2, zero3
-deepspeed: deepspeed/zero1.json
+deepspeed: deepspeed_configs/zero1.json
 weight_decay: 0.0
 fsdp:
 fsdp_config: