tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Fix(tokenizer): Fix condition to add pad token (#477)

Browse Source 
* Fix(tokenizer): Fix condition to add pad token

* chore: fix lint
This commit is contained in:
NanoCode012
2023-08-25 14:30:50 +09:00
committed by GitHub
parent cb9797ef5a
commit 71bd06243c
1 changed files with 9 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/axolotl/utils/models.py
View File

@@ -55,10 +55,15 @@ def load_tokenizer(cfg):
**tokenizer_kwargs, **tokenizer_kwargs,
) )
if tokenizer.__class__.__name__ in [ if (
"LlamaTokenizer", tokenizer.__class__.__name__
"LlamaTokenizerFast", in [
] and not hasattr(tokenizer, "pad_token"): "LlamaTokenizer",
"LlamaTokenizerFast",
]
and hasattr(tokenizer, "pad_token")
and not tokenizer.pad_token
):
# set a pad_token, but use eos_token so we don't add a new token # set a pad_token, but use eos_token so we don't add a new token
tokenizer.pad_token = LLAMA_DEFAULT_EOS_TOKEN tokenizer.pad_token = LLAMA_DEFAULT_EOS_TOKEN