8bit and deepspeed changes

2023-04-30 06:50:35 -04:00
parent 4dbef0941f
commit 9190ada23a
2 changed files with 11 additions and 16 deletions
--- a/ds_config.json
+++ b/ds_config.json
@@ -20,10 +20,12 @@
    }
  },
  "scheduler": {
-    "type": "OneCycle",
+    "type": "WarmupDecayLR",
    "params": {
-      "cycle_min_lr": 1e-7,
-      "cycle_max_lr": 1e-4
+      "warmup_min_lr": "auto",
+      "warmup_max_lr": "auto",
+      "warmup_num_steps": "auto",
+      "total_num_steps": "auto"
    }
  },
  "zero_optimization": {