tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Liger GraniteMoE (#2715)

Browse Source
This commit is contained in:
xzuyn
2025-05-23 18:40:43 -04:00
committed by GitHub
parent a535b68043
commit d27c35ac44
1 changed files with 10 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/axolotl/integrations/liger/__init__.py
View File

@@ -175,6 +175,16 @@ class LigerPlugin(BasePlugin):
rms_norm=cfg.liger_rms_norm, rms_norm=cfg.liger_rms_norm,
layer_norm=cfg.liger_layer_norm, layer_norm=cfg.liger_layer_norm,
) )
elif cfg.model_config_type == "granitemoe":
from liger_kernel.transformers import apply_liger_kernel_to_granite
apply_liger_kernel_to_granite(
rope=cfg.liger_rope,
cross_entropy=cfg.liger_cross_entropy,
fused_linear_cross_entropy=cfg.liger_fused_linear_cross_entropy,
rms_norm=cfg.liger_rms_norm,
swiglu=cfg.liger_glu_activation,
)
else: else:
logging.warning( logging.warning(
f"Unsupported model config type: {cfg.model_config_type}. Liger not applied." f"Unsupported model config type: {cfg.model_config_type}. Liger not applied."