tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

set tokenizer_config in fixture

Browse Source
This commit is contained in:
Wing Lian
2025-07-06 12:24:21 -04:00
parent c40da3b5eb
commit 07ab737a55
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/e2e/multigpu/test_llama.py
View File

@@ -33,6 +33,7 @@ def download_model():
def sft_base_cfg(): def sft_base_cfg():
cfg = DictDefault( cfg = DictDefault(
base_model="HuggingFaceTB/SmolLM2-135M", base_model="HuggingFaceTB/SmolLM2-135M",
tokenizer_config="HuggingFaceTB/SmolLM2-135M", # this has to be manually set since we haven't done validation
sequence_len=2048, sequence_len=2048,
special_tokens={ special_tokens={
"pad_token": "<|endoftext|>", "pad_token": "<|endoftext|>",