From c6ae5c43cbd900e72222f1515adfb081a3500b6a Mon Sep 17 00:00:00 2001 From: NanoCode012 Date: Thu, 4 Sep 2025 03:25:09 +0700 Subject: [PATCH] fix: chat template jinja file not being loaded during inference (#3112) * fix: chat template jinja file not being loaded during inference * fix: bot comment --- src/axolotl/cli/inference.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/axolotl/cli/inference.py b/src/axolotl/cli/inference.py index 06b64292f..debe57167 100644 --- a/src/axolotl/cli/inference.py +++ b/src/axolotl/cli/inference.py @@ -14,10 +14,7 @@ from transformers import GenerationConfig, TextIteratorStreamer, TextStreamer from axolotl.cli.args import InferenceCliArgs from axolotl.cli.config import load_cfg from axolotl.cli.utils import load_model_and_tokenizer -from axolotl.utils.chat_templates import ( - get_chat_template, - get_chat_template_from_config, -) +from axolotl.utils.chat_templates import get_chat_template_from_config from axolotl.utils.dict import DictDefault from axolotl.utils.logging import get_logger @@ -64,7 +61,9 @@ def do_inference( importlib.import_module("axolotl.prompters"), prompter ) elif cfg.chat_template: - chat_template_str = get_chat_template(cfg.chat_template, tokenizer=tokenizer) + chat_template_str = get_chat_template_from_config( + cfg, ds_cfg=None, tokenizer=tokenizer + ) elif cfg.datasets[0].type == "chat_template": chat_template_str = get_chat_template_from_config( cfg=cfg, ds_cfg=cfg.datasets[0], tokenizer=tokenizer @@ -159,7 +158,13 @@ def do_inference_gradio( importlib.import_module("axolotl.prompters"), prompter ) elif cfg.chat_template: - chat_template_str = get_chat_template(cfg.chat_template, tokenizer=tokenizer) + chat_template_str = get_chat_template_from_config( + cfg, ds_cfg=None, tokenizer=tokenizer + ) + elif cfg.datasets[0].type == "chat_template": + chat_template_str = get_chat_template_from_config( + cfg=cfg, ds_cfg=cfg.datasets[0], tokenizer=tokenizer + ) model = model.to(cfg.device, dtype=cfg.torch_dtype)