From 7fd662dd89e4fb8e97a7b1fbb4328f33220f60c1 Mon Sep 17 00:00:00 2001 From: Maxime <672982+maximegmd@users.noreply.github.com> Date: Sun, 27 Aug 2023 21:01:43 +0200 Subject: [PATCH] Update src/axolotl/utils/models.py Co-authored-by: Aman Gupta Karmani --- src/axolotl/utils/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py index ed917d963..4575f5966 100644 --- a/src/axolotl/utils/models.py +++ b/src/axolotl/utils/models.py @@ -368,7 +368,7 @@ def load_model( # LlamaRMSNorm layers are in fp32 after kbit_training or full finetune, so we need to # convert them back to fp16/bf16 for flash-attn compatibility. - if (fix_dtype or not cfg.adapter) and ( + if fix_dtype and ( cfg.flash_attention and cfg.is_llama_derived_model ): for name, module in model.named_modules():