Only fuse if flash_attn_fuse_mlp is True

src/axolotl/utils/models.py

@@ -385,6 +385,7 @@ def load_model(
                 **model_kwargs,
             )
 
+            if cfg.flash_attn_fuse_mlp:
                 LOG.info("Mixtral MoE: Replacing experts with SwiGLU")
                 replace_mixtral_mlp_with_swiglu(model)