tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

prepare datasets only flag

Browse Source
This commit is contained in:
Wing Lian
2023-04-15 16:30:55 -04:00
parent d060c803ce
commit 23938015c8
1 changed files with 5 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/finetune.py
View File

@@ -306,6 +306,7 @@ def setup_trainer(cfg, train_dataset, eval_dataset, model, tokenizer):
def train( def train(
config: Path = Path("configs/"), config: Path = Path("configs/"),
prepare_ds_only: bool = False,
**kwargs, **kwargs,
): ):
if Path(config).is_dir(): if Path(config).is_dir():
@@ -396,6 +397,10 @@ def train(
else: else:
dataset.save_to_disk(DEFAULT_DATASET_PREPARED_PATH) dataset.save_to_disk(DEFAULT_DATASET_PREPARED_PATH)
if prepare_ds_only:
logger.info("Finished preparing dataset. Exiting...")
return
train_dataset = dataset["train"] train_dataset = dataset["train"]
eval_dataset = dataset["test"] eval_dataset = dataset["test"]