From b986f7c7cb3e7a8a43bf4136686118a4c7e4a669 Mon Sep 17 00:00:00 2001 From: NanoCode012 Date: Sun, 20 Jul 2025 00:54:43 +0700 Subject: [PATCH] fix: return proper attention for llama4 lora kernel and fsdp2 llama4 example fix (#2943) * fix: return proper attention for llama4 lora optim * fix: update fsdp2 llama4 config --- examples/llama-4/scout-qlora-flexattn-fsdp2.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llama-4/scout-qlora-flexattn-fsdp2.yaml b/examples/llama-4/scout-qlora-flexattn-fsdp2.yaml index b319349c4..b3e8c328c 100644 --- a/examples/llama-4/scout-qlora-flexattn-fsdp2.yaml +++ b/examples/llama-4/scout-qlora-flexattn-fsdp2.yaml @@ -74,7 +74,7 @@ fsdp: fsdp_config: fsdp_version: 2 fsdp_offload_params: false - fsdp_cpu_ram_efficient_loading: true + # fsdp_cpu_ram_efficient_loading: true # does not work with load_in_8bit/4bit fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP fsdp_transformer_layer_cls_to_wrap: Llama4TextDecoderLayer fsdp_state_dict_type: SHARDED_STATE_DICT