From 18a36b31efaa821179873af85fd950ed1ccf5aa9 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Fri, 10 Jan 2025 18:42:28 -0500 Subject: [PATCH] make sure the batch dataset patcher for multipack is always loaded when handling datasets --- src/axolotl/cli/utils.py | 1 - src/axolotl/common/datasets.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/axolotl/cli/utils.py b/src/axolotl/cli/utils.py index d19d73246..c0676b128 100644 --- a/src/axolotl/cli/utils.py +++ b/src/axolotl/cli/utils.py @@ -15,7 +15,6 @@ import requests from pydantic import BaseModel from transformers import PreTrainedModel, PreTrainedTokenizer, PreTrainedTokenizerFast -import axolotl.monkeypatch.data.batch_dataset_fetcher # pylint: disable=unused-import # noqa: F401 from axolotl.logging_config import configure_logging from axolotl.utils.dict import DictDefault from axolotl.utils.models import load_model, load_tokenizer diff --git a/src/axolotl/common/datasets.py b/src/axolotl/common/datasets.py index ebe82ac64..12be18ea5 100644 --- a/src/axolotl/common/datasets.py +++ b/src/axolotl/common/datasets.py @@ -8,6 +8,7 @@ from typing import Optional, Union from datasets import Dataset +import axolotl.monkeypatch.data.batch_dataset_fetcher # pylint: disable=unused-import # noqa: F401 from axolotl.cli.args import PreprocessCliArgs, TrainerCliArgs from axolotl.utils.data import prepare_dataset from axolotl.utils.data.rl import load_prepare_dpo_datasets