tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Use '<|finetune_right_pad|>' as padding token for LLama4 (#2988) [skip ci]

Browse Source
This commit is contained in:
Vincenzo di Cicco
2025-07-30 12:38:13 +02:00
committed by GitHub
parent 90e5598930
commit 2eb7ff95af
7 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/llama-4/do-no-use-fa2/maverick-qlora-fsdp1.yaml
View File

@@ -84,7 +84,7 @@ fsdp_config:
fsdp_state_dict_type: FULL_STATE_DICT fsdp_state_dict_type: FULL_STATE_DICT
fsdp_sharding_strategy: FULL_SHARD fsdp_sharding_strategy: FULL_SHARD
special_tokens: special_tokens:
pad_token: <|finetune_right_pad_id|> pad_token: <|finetune_right_pad|>
eos_token: <|eot|> eos_token: <|eot|>
# save_first_step: true # uncomment this to validate checkpoint saving works with your config # save_first_step: true # uncomment this to validate checkpoint saving works with your config

2
examples/llama-4/do-no-use-fa2/scout-qlora-fsdp1.yaml
View File

@@ -88,7 +88,7 @@ fsdp_config:
fsdp_sharding_strategy: FULL_SHARD fsdp_sharding_strategy: FULL_SHARD
fsdp_activation_checkpointing: true fsdp_activation_checkpointing: true
special_tokens: special_tokens:
pad_token: <|finetune_right_pad_id|> pad_token: <|finetune_right_pad|>
eos_token: <|eot|> eos_token: <|eot|>
# save_first_step: true # uncomment this to validate checkpoint saving works with your config # save_first_step: true # uncomment this to validate checkpoint saving works with your config

2
examples/llama-4/do-no-use-fa2/scout-qlora-single-h100.yaml
View File

@@ -81,7 +81,7 @@ evals_per_epoch: 1
saves_per_epoch: 1 saves_per_epoch: 1
weight_decay: 0.0 weight_decay: 0.0
special_tokens: special_tokens:
pad_token: <|finetune_right_pad_id|> pad_token: <|finetune_right_pad|>
eos_token: <|eot|> eos_token: <|eot|>
# save_first_step: true # uncomment this to validate checkpoint saving works with your config # save_first_step: true # uncomment this to validate checkpoint saving works with your config

2
examples/llama-4/do-no-use-fa2/scout-vision-qlora-fsdp.yaml
View File

@@ -84,7 +84,7 @@ fsdp_config:
fsdp_sharding_strategy: FULL_SHARD fsdp_sharding_strategy: FULL_SHARD
fsdp_activation_checkpointing: true fsdp_activation_checkpointing: true
special_tokens: special_tokens:
pad_token: <|finetune_right_pad_id|> pad_token: <|finetune_right_pad|>
eos_token: <|eot|> eos_token: <|eot|>
# save_first_step: true # uncomment this to validate checkpoint saving works with your config # save_first_step: true # uncomment this to validate checkpoint saving works with your config

2
examples/llama-4/scout-qlora-flexattn-fsdp2.yaml
View File

@@ -82,7 +82,7 @@ fsdp_config:
fsdp_reshard_after_forward: true fsdp_reshard_after_forward: true
fsdp_activation_checkpointing: true fsdp_activation_checkpointing: true
special_tokens: special_tokens:
pad_token: <|finetune_right_pad_id|> pad_token: <|finetune_right_pad|>
eos_token: <|eot|> eos_token: <|eot|>
# save_first_step: true # uncomment this to validate checkpoint saving works with your config # save_first_step: true # uncomment this to validate checkpoint saving works with your config

2
examples/llama-4/scout-qlora-single-h100-flex.yaml
View File

@@ -80,7 +80,7 @@ saves_per_epoch: 1
weight_decay: 0.0 weight_decay: 0.0
special_tokens: special_tokens:
pad_token: <|finetune_right_pad_id|> pad_token: <|finetune_right_pad|>
eos_token: <|eot|> eos_token: <|eot|>
# save_first_step: true # uncomment this to validate checkpoint saving works with your config # save_first_step: true # uncomment this to validate checkpoint saving works with your config

2
examples/llama-4/scout-vision-qlora-fsdp2-flex.yaml
View File

@@ -85,7 +85,7 @@ fsdp_config:
fsdp_reshard_after_forward: true fsdp_reshard_after_forward: true
fsdp_activation_checkpointing: true fsdp_activation_checkpointing: true
special_tokens: special_tokens:
pad_token: <|finetune_right_pad_id|> pad_token: <|finetune_right_pad|>
eos_token: <|eot|> eos_token: <|eot|>
# save_first_step: true # uncomment this to validate checkpoint saving works with your config # save_first_step: true # uncomment this to validate checkpoint saving works with your config