tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

simplify load_tokenizer

Browse Source
This commit is contained in:
Aman Karmani
2023-08-13 01:33:38 +00:00
committed by Aman Gupta Karmani
parent 7b55fe6419
commit efb3b2c95e
3 changed files with 24 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
tests/test_tokenizers.py
View File

@@ -13,17 +13,22 @@ class TestTokenizers(unittest.TestCase):
"""
def test_default_use_fast(self):
cfg = DictDefault({})
tokenizer = load_tokenizer("huggyllama/llama-7b", None, cfg)
cfg = DictDefault(
{
"tokenizer_config": "huggyllama/llama-7b",
}
)
tokenizer = load_tokenizer(cfg)
assert "Fast" in tokenizer.__class__.__name__
def test_dont_use_fast(self):
cfg = DictDefault(
{
"tokenizer_config": "huggyllama/llama-7b",
"tokenizer_use_fast": False,
}
)
tokenizer = load_tokenizer("huggyllama/llama-7b", None, cfg)
tokenizer = load_tokenizer(cfg)
assert "Fast" not in tokenizer.__class__.__name__