tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

make sure the batch dataset patcher for multipack is always loaded when handling datasets

Browse Source
This commit is contained in:
Wing Lian
2025-01-10 18:42:28 -05:00
committed by Dan Saunders
parent 705e7dc270
commit 18a36b31ef
2 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/axolotl/cli/utils.py
View File

@@ -15,7 +15,6 @@ import requests
from pydantic import BaseModel from pydantic import BaseModel
from transformers import PreTrainedModel, PreTrainedTokenizer, PreTrainedTokenizerFast from transformers import PreTrainedModel, PreTrainedTokenizer, PreTrainedTokenizerFast
import axolotl.monkeypatch.data.batch_dataset_fetcher # pylint: disable=unused-import # noqa: F401
from axolotl.logging_config import configure_logging from axolotl.logging_config import configure_logging
from axolotl.utils.dict import DictDefault from axolotl.utils.dict import DictDefault
from axolotl.utils.models import load_model, load_tokenizer from axolotl.utils.models import load_model, load_tokenizer

1
src/axolotl/common/datasets.py
View File

@@ -8,6 +8,7 @@ from typing import Optional, Union
from datasets import Dataset from datasets import Dataset
import axolotl.monkeypatch.data.batch_dataset_fetcher # pylint: disable=unused-import # noqa: F401
from axolotl.cli.args import PreprocessCliArgs, TrainerCliArgs from axolotl.cli.args import PreprocessCliArgs, TrainerCliArgs
from axolotl.utils.data import prepare_dataset from axolotl.utils.data import prepare_dataset
from axolotl.utils.data.rl import load_prepare_dpo_datasets from axolotl.utils.data.rl import load_prepare_dpo_datasets