tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

improve inference

Browse Source
This commit is contained in:
Wing Lian
2023-04-19 12:57:27 -04:00
parent 5749eb0a1c
commit d65385912e
2 changed files with 42 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
scripts/finetune.py
View File

@@ -79,10 +79,10 @@ def do_inference(cfg, model, tokenizer):
from axolotl.prompters import ReflectAlpacaPrompter from axolotl.prompters import ReflectAlpacaPrompter
while True:
instruction = str(input("Give me an instruction: ")) instruction = str(input("Give me an instruction: "))
instruction = ( if not instruction:
instruction if not instruction else "Tell me a joke about dromedaries." return
)
prompt = ReflectAlpacaPrompter().build_prompt(instruction=instruction) prompt = ReflectAlpacaPrompter().build_prompt(instruction=instruction)
batch = tokenizer(prompt, return_tensors="pt", add_special_tokens=True) batch = tokenizer(prompt, return_tensors="pt", add_special_tokens=True)

3
src/axolotl/utils/models.py
View File

@@ -66,6 +66,7 @@ def load_model(
from alpaca_lora_4bit.autograd_4bit import load_llama_model_4bit_low_ram from alpaca_lora_4bit.autograd_4bit import load_llama_model_4bit_low_ram
from huggingface_hub import snapshot_download from huggingface_hub import snapshot_download
try:
snapshot_download_kwargs = {} snapshot_download_kwargs = {}
if cfg.base_model_ignore_patterns: if cfg.base_model_ignore_patterns:
snapshot_download_kwargs["ignore_patterns"] = cfg.base_model_ignore_patterns snapshot_download_kwargs["ignore_patterns"] = cfg.base_model_ignore_patterns
@@ -82,6 +83,8 @@ def load_model(
"unable to find a cached model file, this will likely fail..." "unable to find a cached model file, this will likely fail..."
) )
model_path = str(cache_model_path) model_path = str(cache_model_path)
except:
model_path = cfg.base_model
model, tokenizer = load_llama_model_4bit_low_ram( model, tokenizer = load_llama_model_4bit_low_ram(
base_model_config if base_model_config else base_model, base_model_config if base_model_config else base_model,
model_path, model_path,