tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Fix(tokenizer): Set rstrip,lstrip,norm to False (#678)

Browse Source
This commit is contained in:
NanoCode012
2023-10-06 03:50:49 +09:00
committed by GitHub
parent 43856c0a39
commit e0b7eeabfd
1 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/axolotl/utils/models.py
View File

@@ -11,6 +11,7 @@ from optimum.bettertransformer import BetterTransformer
from peft import PeftConfig, prepare_model_for_kbit_training from peft import PeftConfig, prepare_model_for_kbit_training
from peft.tuners.lora import QuantLinear from peft.tuners.lora import QuantLinear
from transformers import ( # noqa: F401 from transformers import ( # noqa: F401
AddedToken,
AutoConfig, AutoConfig,
AutoModelForCausalLM, AutoModelForCausalLM,
AutoTokenizer, AutoTokenizer,
@@ -82,9 +83,16 @@ def load_tokenizer(cfg):
if cfg.special_tokens: if cfg.special_tokens:
for k, val in cfg.special_tokens.items(): for k, val in cfg.special_tokens.items():
tokenizer.add_special_tokens({k: val}) tokenizer.add_special_tokens(
{k: AddedToken(val, rstrip=False, lstrip=False, normalized=False)}
)
if cfg.tokens: if cfg.tokens:
tokenizer.add_tokens(list(cfg.tokens)) tokenizer.add_tokens(
[
AddedToken(token, rstrip=False, lstrip=False, normalized=False)
for token in cfg.tokens
]
)
return tokenizer return tokenizer