tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Only fuse if flash_attn_fuse_mlp is True

Browse Source
This commit is contained in:
Casper
2023-12-10 19:17:12 +01:00
parent 279a1401b5
commit a58a9e5f6c
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/axolotl/utils/models.py
View File

@@ -385,8 +385,9 @@ def load_model(
**model_kwargs, **model_kwargs,
) )
LOG.info("Mixtral MoE: Replacing experts with SwiGLU") if cfg.flash_attn_fuse_mlp:
replace_mixtral_mlp_with_swiglu(model) LOG.info("Mixtral MoE: Replacing experts with SwiGLU")
replace_mixtral_mlp_with_swiglu(model)
elif model_type == "MambaLMHeadModel": elif model_type == "MambaLMHeadModel":
# FIXME this is janky at best and hacked together to make it work # FIXME this is janky at best and hacked together to make it work