tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Fix(tokenizer): Make sure to add pad for CodeLlamaTokenizer (#489)

Browse Source
This commit is contained in:
NanoCode012
2023-08-28 09:39:10 +09:00
committed by GitHub
parent f144e98a32
commit 4c37bd0b54
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/axolotl/utils/models.py
View File

@@ -59,6 +59,7 @@ def load_tokenizer(cfg):
in [ in [
"LlamaTokenizer", "LlamaTokenizer",
"LlamaTokenizerFast", "LlamaTokenizerFast",
"CodeLlamaTokenizer",
] ]
and hasattr(tokenizer, "pad_token") and hasattr(tokenizer, "pad_token")
and not tokenizer.pad_token and not tokenizer.pad_token