tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

multipack support for phi moe

Browse Source
This commit is contained in:
Wing Lian
2024-11-15 22:56:25 -05:00
parent 0c8b1d824a
commit cb8bfab9cc
2 changed files with 2 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/axolotl/common/architectures.py
View File

@@ -10,6 +10,7 @@ MOE_ARCH_BLOCK = {
"JetMoeMoE", "JetMoeMoE",
], ],
"mixtral": "MixtralSparseMoeBlock", "mixtral": "MixtralSparseMoeBlock",
"phimoe": "PhiMoESparseMoeBlock",
"qwen2_moe": "Qwen2MoeSparseMoeBlock", "qwen2_moe": "Qwen2MoeSparseMoeBlock",
"deepseek_v2": "DeepseekV2MoE", "deepseek_v2": "DeepseekV2MoE",
} }

1
src/axolotl/monkeypatch/multipack.py
View File

@@ -20,6 +20,7 @@ SUPPORTED_MULTIPACK_MODEL_TYPES = [
"falcon", "falcon",
"phi", "phi",
"phi3", "phi3",
"phimoe",
"gemma", "gemma",
"gemma2", "gemma2",
"gemmoe", "gemmoe",