tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

improve llama pad token handling (#475)

Browse Source 
* improve llama pad token handling

* tweak logic to not clobber
This commit is contained in:
Wing Lian
2023-08-24 13:20:35 -04:00
committed by GitHub
parent bde3c5a478
commit cb9797ef5a
4 changed files with 10 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/axolotl/utils/models.py
View File

@@ -22,7 +22,7 @@ from transformers import ( # noqa: F401
PreTrainedTokenizerBase,
)
from axolotl.prompt_tokenizers import LLAMA_DEFAULT_PAD_TOKEN
from axolotl.prompt_tokenizers import LLAMA_DEFAULT_EOS_TOKEN
from axolotl.utils.bench import log_gpu_memory_usage
LOG = logging.getLogger("axolotl")
@@ -58,8 +58,9 @@ def load_tokenizer(cfg):
if tokenizer.__class__.__name__ in [
"LlamaTokenizer",
"LlamaTokenizerFast",
]:
tokenizer.pad_token = LLAMA_DEFAULT_PAD_TOKEN
] and not hasattr(tokenizer, "pad_token"):
# set a pad_token, but use eos_token so we don't add a new token
tokenizer.pad_token = LLAMA_DEFAULT_EOS_TOKEN
LOG.debug(f"EOS: {tokenizer.eos_token_id} / {tokenizer.eos_token}")
LOG.debug(f"BOS: {tokenizer.bos_token_id} / {tokenizer.bos_token}")