From 17605b85d8046b7dee53289175dea17b8700fe0b Mon Sep 17 00:00:00 2001
From: Maxime <672982+maximegmd@users.noreply.github.com>
Date: Sat, 26 Aug 2023 22:40:56 +0200
Subject: [PATCH] fix: inference did not move the model to the correct device
 (#483)

---
 scripts/finetune.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/finetune.py b/scripts/finetune.py
index 3255a623f..d02448ec2 100644
--- a/scripts/finetune.py
+++ b/scripts/finetune.py
@@ -82,6 +82,8 @@ def do_inference(cfg, model, tokenizer, prompter: Optional[str]):
             max_seq_len=255, mem_freq=50, top_k=5, max_cache_size=None
         )
 
+    model = model.to(cfg.device)
+
     while True:
         print("=" * 80)
         # support for multiline inputs