From bb53a165f5f2805b78f4d7c252c6978e24d81cd2 Mon Sep 17 00:00:00 2001
From: Wing Lian <wing.lian@gmail.com>
Date: Sun, 6 Aug 2023 17:19:51 -0400
Subject: [PATCH] add a basic ds zero3 config (#347)

better defaults for ds
---
 ds_config.json => deepspeed/zero3.json | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
 rename ds_config.json => deepspeed/zero3.json (89%)

diff --git a/ds_config.json b/deepspeed/zero3.json
similarity index 89%
rename from ds_config.json
rename to deepspeed/zero3.json
index 1e150ba95..69ad19259 100644
--- a/ds_config.json
+++ b/deepspeed/zero3.json
@@ -37,18 +37,18 @@
       "lr": "auto",
       "betas": [
         0.9,
-        0.999
+        0.95
       ],
       "eps": 1e-8,
       "weight_decay": "auto"
     }
   },
   "scheduler": {
-    "type": "OneCycle",
+    "type": "WarmupLR",
     "params": {
-      "cycle_min_lr": 0.00001,
-      "cycle_max_lr": 0.00003,
-      "cycle_first_step_size": 120
+      "warmup_min_lr": "auto",
+      "warmup_max_lr": "auto",
+      "warmup_num_steps": "auto"
     }
   },
   "train_batch_size": "auto",