From 2eb7ff95af21eff41ec5a421cb9319daa5d987bb Mon Sep 17 00:00:00 2001 From: Vincenzo di Cicco <112694549+v-dicicco@users.noreply.github.com> Date: Wed, 30 Jul 2025 12:38:13 +0200 Subject: [PATCH] Use '<|finetune_right_pad|>' as padding token for LLama4 (#2988) [skip ci] --- examples/llama-4/do-no-use-fa2/maverick-qlora-fsdp1.yaml | 2 +- examples/llama-4/do-no-use-fa2/scout-qlora-fsdp1.yaml | 2 +- examples/llama-4/do-no-use-fa2/scout-qlora-single-h100.yaml | 2 +- examples/llama-4/do-no-use-fa2/scout-vision-qlora-fsdp.yaml | 2 +- examples/llama-4/scout-qlora-flexattn-fsdp2.yaml | 2 +- examples/llama-4/scout-qlora-single-h100-flex.yaml | 2 +- examples/llama-4/scout-vision-qlora-fsdp2-flex.yaml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/llama-4/do-no-use-fa2/maverick-qlora-fsdp1.yaml b/examples/llama-4/do-no-use-fa2/maverick-qlora-fsdp1.yaml index 3bd05b5ba..5770ce947 100644 --- a/examples/llama-4/do-no-use-fa2/maverick-qlora-fsdp1.yaml +++ b/examples/llama-4/do-no-use-fa2/maverick-qlora-fsdp1.yaml @@ -84,7 +84,7 @@ fsdp_config: fsdp_state_dict_type: FULL_STATE_DICT fsdp_sharding_strategy: FULL_SHARD special_tokens: - pad_token: <|finetune_right_pad_id|> + pad_token: <|finetune_right_pad|> eos_token: <|eot|> # save_first_step: true # uncomment this to validate checkpoint saving works with your config diff --git a/examples/llama-4/do-no-use-fa2/scout-qlora-fsdp1.yaml b/examples/llama-4/do-no-use-fa2/scout-qlora-fsdp1.yaml index 1c6ba1410..7cd8032d2 100644 --- a/examples/llama-4/do-no-use-fa2/scout-qlora-fsdp1.yaml +++ b/examples/llama-4/do-no-use-fa2/scout-qlora-fsdp1.yaml @@ -88,7 +88,7 @@ fsdp_config: fsdp_sharding_strategy: FULL_SHARD fsdp_activation_checkpointing: true special_tokens: - pad_token: <|finetune_right_pad_id|> + pad_token: <|finetune_right_pad|> eos_token: <|eot|> # save_first_step: true # uncomment this to validate checkpoint saving works with your config diff --git a/examples/llama-4/do-no-use-fa2/scout-qlora-single-h100.yaml b/examples/llama-4/do-no-use-fa2/scout-qlora-single-h100.yaml index 081089555..03acdc234 100644 --- a/examples/llama-4/do-no-use-fa2/scout-qlora-single-h100.yaml +++ b/examples/llama-4/do-no-use-fa2/scout-qlora-single-h100.yaml @@ -81,7 +81,7 @@ evals_per_epoch: 1 saves_per_epoch: 1 weight_decay: 0.0 special_tokens: - pad_token: <|finetune_right_pad_id|> + pad_token: <|finetune_right_pad|> eos_token: <|eot|> # save_first_step: true # uncomment this to validate checkpoint saving works with your config diff --git a/examples/llama-4/do-no-use-fa2/scout-vision-qlora-fsdp.yaml b/examples/llama-4/do-no-use-fa2/scout-vision-qlora-fsdp.yaml index 390be5af7..d9255ea16 100644 --- a/examples/llama-4/do-no-use-fa2/scout-vision-qlora-fsdp.yaml +++ b/examples/llama-4/do-no-use-fa2/scout-vision-qlora-fsdp.yaml @@ -84,7 +84,7 @@ fsdp_config: fsdp_sharding_strategy: FULL_SHARD fsdp_activation_checkpointing: true special_tokens: - pad_token: <|finetune_right_pad_id|> + pad_token: <|finetune_right_pad|> eos_token: <|eot|> # save_first_step: true # uncomment this to validate checkpoint saving works with your config diff --git a/examples/llama-4/scout-qlora-flexattn-fsdp2.yaml b/examples/llama-4/scout-qlora-flexattn-fsdp2.yaml index 6193e4ed5..4cda4949e 100644 --- a/examples/llama-4/scout-qlora-flexattn-fsdp2.yaml +++ b/examples/llama-4/scout-qlora-flexattn-fsdp2.yaml @@ -82,7 +82,7 @@ fsdp_config: fsdp_reshard_after_forward: true fsdp_activation_checkpointing: true special_tokens: - pad_token: <|finetune_right_pad_id|> + pad_token: <|finetune_right_pad|> eos_token: <|eot|> # save_first_step: true # uncomment this to validate checkpoint saving works with your config diff --git a/examples/llama-4/scout-qlora-single-h100-flex.yaml b/examples/llama-4/scout-qlora-single-h100-flex.yaml index c3bbfe56a..518cfa57c 100644 --- a/examples/llama-4/scout-qlora-single-h100-flex.yaml +++ b/examples/llama-4/scout-qlora-single-h100-flex.yaml @@ -80,7 +80,7 @@ saves_per_epoch: 1 weight_decay: 0.0 special_tokens: - pad_token: <|finetune_right_pad_id|> + pad_token: <|finetune_right_pad|> eos_token: <|eot|> # save_first_step: true # uncomment this to validate checkpoint saving works with your config diff --git a/examples/llama-4/scout-vision-qlora-fsdp2-flex.yaml b/examples/llama-4/scout-vision-qlora-fsdp2-flex.yaml index a67936cf1..28da15084 100644 --- a/examples/llama-4/scout-vision-qlora-fsdp2-flex.yaml +++ b/examples/llama-4/scout-vision-qlora-fsdp2-flex.yaml @@ -85,7 +85,7 @@ fsdp_config: fsdp_reshard_after_forward: true fsdp_activation_checkpointing: true special_tokens: - pad_token: <|finetune_right_pad_id|> + pad_token: <|finetune_right_pad|> eos_token: <|eot|> # save_first_step: true # uncomment this to validate checkpoint saving works with your config