qwen3.5 configs (#3554) [skip ci]

* qwen3.5  configs

* update shared experts readme
This commit is contained in:
VED
2026-04-01 18:49:31 +05:30
committed by GitHub
parent 5e5603c9aa
commit 9e64c76326
5 changed files with 30 additions and 20 deletions

View File

@@ -31,11 +31,11 @@ lora_target_modules:
- k_proj
- v_proj
- o_proj
# Add gate_up_proj and down_proj to also target shared experts (nn.Linear):
# - gate_up_proj
# - down_proj
# Regex matching to target shared experts too
# lora_target_modules: 'model\.(language_model\.)?layers\.[\d]+\.(mlp|self_attn)\.(shared_expert\.)?(up|down|gate|gate_up|q|k|v|o)_proj'
# Target experts
# Target routed experts (3D nn.Parameter tensors, not nn.Linear — use lora_target_parameters):
# lora_target_parameters:
# - mlp.experts.gate_up_proj
# - mlp.experts.down_proj