diff --git a/scripts/finetune.py b/scripts/finetune.py
index 3255a623f..d02448ec2 100644
--- a/scripts/finetune.py
+++ b/scripts/finetune.py
@@ -82,6 +82,8 @@ def do_inference(cfg, model, tokenizer, prompter: Optional[str]):
             max_seq_len=255, mem_freq=50, top_k=5, max_cache_size=None
         )
 
+    model = model.to(cfg.device)
+
     while True:
         print("=" * 80)
         # support for multiline inputs