diff --git a/src/axolotl/models/mixtral/__init__.py b/src/axolotl/models/mixtral/__init__.py
index d71fab50d..83af3e356 100644
--- a/src/axolotl/models/mixtral/__init__.py
+++ b/src/axolotl/models/mixtral/__init__.py
@@ -3,4 +3,7 @@ Custom modeling code for mixtral
 """
 
 from .configuration_moe_mistral import MixtralConfig  # noqa
-from .modeling_moe_mistral import MixtralForCausalLM  # noqa
+from .modeling_moe_mistral import ( # noqa
+    MixtralForCausalLM,
+    replace_mixtral_mlp_with_swiglu,
+)
\ No newline at end of file