From fcef8c95fe9357867d966c657594dfdbdcaf10bd Mon Sep 17 00:00:00 2001 From: Sunny Liu Date: Mon, 21 Apr 2025 17:17:20 -0400 Subject: [PATCH] skip set_quant_config if quantization not given --- src/axolotl/utils/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py index c5b168ff7..ffdc940fd 100644 --- a/src/axolotl/utils/models.py +++ b/src/axolotl/utils/models.py @@ -887,7 +887,7 @@ class ModelLoader: # but deepspeed needs this still in bfloat16 bnb_config["bnb_4bit_quant_storage"] = torch.float32 - if self.cfg.quantization.bnb_config_kwargs: + if self.cfg.quantization and self.cfg.quantization.bnb_config_kwargs: bnb_config.update(self.cfg.quantization.bnb_config_kwargs) self.model_kwargs["quantization_config"] = BitsAndBytesConfig(