From e0420b3528ad3120f3bace81d7dff69c657ab91b Mon Sep 17 00:00:00 2001 From: NanoCode012 Date: Thu, 10 Apr 2025 01:01:42 +0700 Subject: [PATCH] fix: allow merge lora on pre-quantized model (#2511) * fix: allow merge lora on pre-quantized model * fix: remove unused sections per comment --- src/axolotl/cli/merge_lora.py | 8 +++++++- src/axolotl/utils/models.py | 6 ------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/axolotl/cli/merge_lora.py b/src/axolotl/cli/merge_lora.py index 1059218fa..c7a3a3225 100644 --- a/src/axolotl/cli/merge_lora.py +++ b/src/axolotl/cli/merge_lora.py @@ -32,7 +32,13 @@ def do_merge_lora(*, cfg: DictDefault) -> None: LOG.info("Running merge of LoRA with base model...") model = model.merge_and_unload(progressbar=True) - model.to(dtype=cfg.torch_dtype) + try: + model.to(dtype=cfg.torch_dtype) + except ValueError as e: + LOG.warning("Failed to convert model to dtype %s", cfg.torch_dtype) + LOG.warning("Ignore this if the base_model is pre-quantized.") + LOG.warning("Error raised: %s", e) + model.generation_config.do_sample = True if cfg.local_rank == 0: diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py index f808f4bdd..c5e569f13 100644 --- a/src/axolotl/utils/models.py +++ b/src/axolotl/utils/models.py @@ -151,12 +151,6 @@ def check_model_config(cfg: DictDefault, model_config: PretrainedConfig): "Please make sure to point to a GPTQ model." ) - if not cfg.gptq and quant_config_exists and not cfg.load_in_4bit: - raise ValueError( - "model_config.quantization_config is set but `gptq` flag is not. " - "Please use the `gptq` flag to train quantized model or point to a non-quantized model." - ) - lora_modules_to_save = get_linear_embedding_layers(model_config.model_type) if ( cfg.adapter