qwen3.5 configs (#3554) [skip ci]
* qwen3.5 configs * update shared experts readme
This commit is contained in:
@@ -31,11 +31,11 @@ lora_target_modules:
|
||||
- k_proj
|
||||
- v_proj
|
||||
- o_proj
|
||||
# Add gate_up_proj and down_proj to also target shared experts (nn.Linear):
|
||||
# - gate_up_proj
|
||||
# - down_proj
|
||||
|
||||
# Regex matching to target shared experts too
|
||||
# lora_target_modules: 'model\.(language_model\.)?layers\.[\d]+\.(mlp|self_attn)\.(shared_expert\.)?(up|down|gate|gate_up|q|k|v|o)_proj'
|
||||
|
||||
# Target experts
|
||||
# Target routed experts (3D nn.Parameter tensors, not nn.Linear — use lora_target_parameters):
|
||||
# lora_target_parameters:
|
||||
# - mlp.experts.gate_up_proj
|
||||
# - mlp.experts.down_proj
|
||||
|
||||
Reference in New Issue
Block a user