check if self.cfg.quantization exists when directly setting load_in_4bit

2025-04-21 21:42:23 -04:00
parent 136b37e4d4
commit db7e92f6a6
1 changed files with 1 additions and 1 deletions
--- a/src/axolotl/utils/models.py
+++ b/src/axolotl/utils/models.py
@@ -893,7 +893,7 @@ class ModelLoader:
                # but deepspeed needs this still in bfloat16
                bnb_config["bnb_4bit_quant_storage"] = torch.float32

-            if self.cfg.quantization.bnb_config_kwargs:
+            if self.cfg.quantization and self.cfg.quantization.bnb_config_kwargs:
                bnb_config.update(self.cfg.quantization.bnb_config_kwargs)

            self.model_kwargs["quantization_config"] = BitsAndBytesConfig(