tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

clean up

Browse Source
This commit is contained in:
Johan Hansson
2024-01-10 22:50:05 +01:00
parent 0d1d00a363
commit 37a934bdb3
1 changed files with 2 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tests/prompt_strategies/test_alpacha.py
View File

@@ -1,5 +1,5 @@
""" """
Test module for sharegpt integration w chatml Test module for alpacha integration w chatml
""" """
import pytest import pytest
from datasets import Dataset from datasets import Dataset
@@ -7,13 +7,12 @@ from tokenizers import AddedToken
from transformers import AutoTokenizer from transformers import AutoTokenizer
from axolotl.datasets import TokenizedPromptDataset from axolotl.datasets import TokenizedPromptDataset
from axolotl.prompt_strategies.sharegpt import SimpleShareGPTPromptTokenizingStrategy
from axolotl.prompt_tokenizers import AlpacaPromptTokenizingStrategy from axolotl.prompt_tokenizers import AlpacaPromptTokenizingStrategy
from axolotl.prompters import AlpacaPrompter from axolotl.prompters import AlpacaPrompter
@pytest.fixture(name="alpacha_dataset") @pytest.fixture(name="alpacha_dataset")
def fixture_sharegpt_dataset(): def fixture_alpacha_dataset():
return Dataset.from_list( return Dataset.from_list(
[ [
{"instruction": "Evaluate this sentence for spelling and grammar mistakes", "input": "He finnished his meal and left the resturant", "output": "He finished his meal and left the restaurant."} {"instruction": "Evaluate this sentence for spelling and grammar mistakes", "input": "He finnished his meal and left the resturant", "output": "He finished his meal and left the restaurant."}