diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py
index 93c111a78..939a312d5 100644
--- a/src/axolotl/utils/models.py
+++ b/src/axolotl/utils/models.py
@@ -85,7 +85,7 @@ def load_model(
         raise e
 
     model_kwargs = {}
-    if cfg.adapter == "qlora":
+    if cfg.adapter == "qlora" and cfg.load_in_4bit:
         model_kwargs["quantization_config"] = BitsAndBytesConfig(
             load_in_4bit=True,
             llm_int8_threshold=6.0,