diff --git a/examples/qwen3-next/README.md b/examples/qwen3-next/README.md index eb0d5fd28..678175fd4 100644 --- a/examples/qwen3-next/README.md +++ b/examples/qwen3-next/README.md @@ -38,7 +38,7 @@ pip3 uninstall -y causal-conv1d && pip3 install flash-linear-attention==0.3.2 axolotl train examples/qwen3-next/qwen3-next-80b-a3b-qlora.yaml ``` -This config uses about 41.7 GiB VRAM. +This config uses about 45.62 GiB VRAM. Let us know how it goes. Happy finetuning! 🚀 diff --git a/examples/qwen3-next/qwen3-next-80b-a3b-qlora.yaml b/examples/qwen3-next/qwen3-next-80b-a3b-qlora.yaml index 11481dcd3..db841beab 100644 --- a/examples/qwen3-next/qwen3-next-80b-a3b-qlora.yaml +++ b/examples/qwen3-next/qwen3-next-80b-a3b-qlora.yaml @@ -27,6 +27,14 @@ lora_r: 16 lora_alpha: 8 lora_dropout: 0.05 lora_target_modules: + - linear_attn.in_proj_ba + - linear_attn.in_proj_qkvz + - linear_attn.out_proj + - shared_expert.up_proj + - shared_expert.down_proj + - shared_expert.gate_proj + - shared_expert_gate + - mlp.gate - q_proj - v_proj - k_proj