improve inference

scripts/finetune.py

@@ -79,10 +79,10 @@ def do_inference(cfg, model, tokenizer):
 
     from axolotl.prompters import ReflectAlpacaPrompter
 
+    while True:
         instruction = str(input("Give me an instruction: "))
-    instruction = (
-        instruction if not instruction else "Tell me a joke about dromedaries."
-    )
+        if not instruction:
+            return
         prompt = ReflectAlpacaPrompter().build_prompt(instruction=instruction)
         batch = tokenizer(prompt, return_tensors="pt", add_special_tokens=True)
 

src/axolotl/utils/models.py

@@ -66,6 +66,7 @@ def load_model(
             from alpaca_lora_4bit.autograd_4bit import load_llama_model_4bit_low_ram
             from huggingface_hub import snapshot_download
 
+            try:
                 snapshot_download_kwargs = {}
                 if cfg.base_model_ignore_patterns:
                     snapshot_download_kwargs["ignore_patterns"] = cfg.base_model_ignore_patterns
@@ -82,6 +83,8 @@ def load_model(
                         "unable to find a cached model file, this will likely fail..."
                     )
                     model_path = str(cache_model_path)
+            except:
+                model_path = cfg.base_model
             model, tokenizer = load_llama_model_4bit_low_ram(
                 base_model_config if base_model_config else base_model,
                 model_path,