qlora and 4bit check so we are able to merge and unload

2023-05-25 22:55:13 -04:00
parent e7e1a777bd
commit 1987e5cf56
1 changed files with 1 additions and 1 deletions
--- a/src/axolotl/utils/models.py
+++ b/src/axolotl/utils/models.py
@@ -85,7 +85,7 @@ def load_model(
        raise e

    model_kwargs = {}
-    if cfg.adapter == "qlora":
+    if cfg.adapter == "qlora" and cfg.load_in_4bit:
        model_kwargs["quantization_config"] = BitsAndBytesConfig(
            load_in_4bit=True,
            llm_int8_threshold=6.0,