add a basic ds zero3 config (#347)

better defaults for ds
2023-08-06 17:19:51 -04:00
parent 10405b9995
commit bb53a165f5
1 changed files with 5 additions and 5 deletions
--- a/deepspeed/zero3.json
+++ b/deepspeed/zero3.json
@@ -37,18 +37,18 @@
      "lr": "auto",
      "betas": [
        0.9,
-        0.999
+        0.95
      ],
      "eps": 1e-8,
      "weight_decay": "auto"
    }
  },
  "scheduler": {
-    "type": "OneCycle",
+    "type": "WarmupLR",
    "params": {
-      "cycle_min_lr": 0.00001,
-      "cycle_max_lr": 0.00003,
-      "cycle_first_step_size": 120
+      "warmup_min_lr": "auto",
+      "warmup_max_lr": "auto",
+      "warmup_num_steps": "auto"
    }
  },
  "train_batch_size": "auto",