tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

make one cycle lr div factor configurable

Browse Source
This commit is contained in:
Wing Lian
2023-05-21 20:25:06 -04:00
parent 0f74464652
commit 99383f14a3
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/axolotl/utils/trainer.py
View File

@@ -157,7 +157,7 @@ def setup_trainer(cfg, train_dataset, eval_dataset, model, tokenizer):
cfg.learning_rate, cfg.learning_rate,
total_steps=total_num_steps, total_steps=total_num_steps,
epochs=cfg.num_epochs, epochs=cfg.num_epochs,
div_factor=10, div_factor=cfg.lr_div_factor if cfg.lr_div_factor else 6,
**lr_scheduler_kwargs, **lr_scheduler_kwargs,
) )
elif cfg.lr_scheduler == "log_sweep": elif cfg.lr_scheduler == "log_sweep":
@@ -182,7 +182,7 @@ def setup_trainer(cfg, train_dataset, eval_dataset, model, tokenizer):
cfg.early_stopping_patience, cfg.early_stopping_patience,
) )
callbacks.append(early_stop_cb) callbacks.append(early_stop_cb)
if cfg.local_rank == 0 and cfg.adapter == 'lora': # only save in rank 0 if cfg.local_rank == 0 and cfg.adapter == 'lora': # only save in rank 0
callbacks.append(SavePeftModelCallback) callbacks.append(SavePeftModelCallback)