add missing fixture decorator for predownload dataset (#2117) [skip ci]

* add missing fixture decorator for predownload dataset

* also pre download the tokenizer files
This commit is contained in:
Wing Lian
2024-12-03 18:08:46 -05:00
committed by GitHub
parent d87df2c776
commit 418ad2b586
2 changed files with 13 additions and 0 deletions

View File

@@ -4,6 +4,7 @@ shared fixtures for prompt strategies tests
import pytest
from datasets import Dataset
from huggingface_hub import hf_hub_download
from transformers import AutoTokenizer
@@ -60,6 +61,17 @@ def fixture_basic_dataset():
@pytest.fixture(name="llama3_tokenizer")
def fixture_llama3_tokenizer():
hf_hub_download(
repo_id="NousResearch/Meta-Llama-3-8B-Instruct",
filename="special_tokens_map.json",
)
hf_hub_download(
repo_id="NousResearch/Meta-Llama-3-8B-Instruct",
filename="tokenizer_config.json",
)
hf_hub_download(
repo_id="NousResearch/Meta-Llama-3-8B-Instruct", filename="tokenizer.json"
)
tokenizer = AutoTokenizer.from_pretrained("NousResearch/Meta-Llama-3-8B-Instruct")
return tokenizer