tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

add a basic ds zero3 config (#347)

Browse Source 
better defaults for ds
This commit is contained in:
Wing Lian
2023-08-06 17:19:51 -04:00
committed by GitHub
parent 10405b9995
commit bb53a165f5
1 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
ds_config.json → deepspeed/zero3.json
View File

@@ -37,18 +37,18 @@
"lr": "auto", "lr": "auto",
"betas": [ "betas": [
0.9, 0.9,
0.999 0.95
], ],
"eps": 1e-8, "eps": 1e-8,
"weight_decay": "auto" "weight_decay": "auto"
} }
}, },
"scheduler": { "scheduler": {
"type": "OneCycle", "type": "WarmupLR",
"params": { "params": {
"cycle_min_lr": 0.00001, "warmup_min_lr": "auto",
"cycle_max_lr": 0.00003, "warmup_max_lr": "auto",
"cycle_first_step_size": 120 "warmup_num_steps": "auto"
} }
}, },
"train_batch_size": "auto", "train_batch_size": "auto",