tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix liger plugin load issues (#1876)

Browse Source
This commit is contained in:
Aman Gupta Karmani
2024-08-27 20:08:26 -07:00
committed by GitHub
parent 159b8b9a74
commit c1a61ae23c
1 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/axolotl/integrations/liger/__init__.py
View File

@@ -22,8 +22,7 @@ import logging
from liger_kernel.transformers.cross_entropy import LigerCrossEntropyLoss from liger_kernel.transformers.cross_entropy import LigerCrossEntropyLoss
from liger_kernel.transformers.geglu import LigerGEGLUMLP from liger_kernel.transformers.geglu import LigerGEGLUMLP
from liger_kernel.transformers.model.llama import lce_forward from liger_kernel.transformers.model.llama import lce_forward as llama_lce_forward
from liger_kernel.transformers.model.qwen2 import lce_forward as qwen2_lce_forward
from liger_kernel.transformers.rms_norm import LigerRMSNorm from liger_kernel.transformers.rms_norm import LigerRMSNorm
from liger_kernel.transformers.rope import liger_rotary_pos_emb from liger_kernel.transformers.rope import liger_rotary_pos_emb
from liger_kernel.transformers.swiglu import LigerSwiGLUMLP from liger_kernel.transformers.swiglu import LigerSwiGLUMLP
@@ -54,7 +53,7 @@ class LigerPlugin(BasePlugin):
if cfg.liger_cross_entropy: if cfg.liger_cross_entropy:
modeling_llama.CrossEntropyLoss = LigerCrossEntropyLoss modeling_llama.CrossEntropyLoss = LigerCrossEntropyLoss
elif cfg.liger_fused_linear_cross_entropy: elif cfg.liger_fused_linear_cross_entropy:
modeling_llama.LlamaForCausalLM.forward = lce_forward modeling_llama.LlamaForCausalLM.forward = llama_lce_forward
elif cfg.model_config_type == "mistral": elif cfg.model_config_type == "mistral":
from transformers.models.mistral import modeling_mistral from transformers.models.mistral import modeling_mistral
@@ -105,6 +104,9 @@ class LigerPlugin(BasePlugin):
modeling_jamba.JambaForCausalLM.forward = jamba_lce_forward modeling_jamba.JambaForCausalLM.forward = jamba_lce_forward
elif cfg.model_config_type == "qwen2": elif cfg.model_config_type == "qwen2":
from liger_kernel.transformers.model.qwen2 import (
lce_forward as qwen2_lce_forward,
)
from transformers.models.qwen2 import modeling_qwen2 from transformers.models.qwen2 import modeling_qwen2
if cfg.liger_rope: if cfg.liger_rope: