From 080612219b8f2b40fe6ae41956d1d72363ba7282 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Fri, 13 Oct 2023 17:54:35 -0400 Subject: [PATCH] use even if not using sample packing --- src/axolotl/utils/models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py index 19b13a342..ad1432583 100644 --- a/src/axolotl/utils/models.py +++ b/src/axolotl/utils/models.py @@ -136,7 +136,11 @@ def load_model( replace_stablelm_attn_with_flash_attn(cfg.base_model) - if cfg.is_llama_derived_model and cfg.flash_attention and cfg.sample_packing: + if ( + cfg.is_llama_derived_model + and cfg.flash_attention + and (cfg.noisy_embeddings_alpha or cfg.sample_packing) + ): if cfg.device not in ["mps", "cpu"] and not inference: from axolotl.monkeypatch.llama_attn_hijack_flash import ( replace_llama_attn_with_flash_attn,