diff --git a/scripts/finetune.py b/scripts/finetune.py
index 47aada411..cd9234334 100644
--- a/scripts/finetune.py
+++ b/scripts/finetune.py
@@ -261,7 +261,6 @@ def train(
         model.save_pretrained(cfg.output_dir)
         return
 
-    model.train()
 
     trainer = setup_trainer(cfg, train_dataset, eval_dataset, model, tokenizer)