Update src/axolotl/utils/models.py

Co-authored-by: Aman Gupta Karmani <aman@tmm1.net>
2023-08-27 21:01:43 +02:00
parent 9e699683d7
commit 7fd662dd89
1 changed files with 1 additions and 1 deletions
--- a/src/axolotl/utils/models.py
+++ b/src/axolotl/utils/models.py
@@ -368,7 +368,7 @@ def load_model(

    # LlamaRMSNorm layers are in fp32 after kbit_training or full finetune, so we need to
    # convert them back to fp16/bf16 for flash-attn compatibility.
-    if (fix_dtype or not cfg.adapter) and (
+    if fix_dtype and (
        cfg.flash_attention and cfg.is_llama_derived_model
    ):
        for name, module in model.named_modules():