tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

only process eval dataset for packing if not None

Browse Source
This commit is contained in:
Wing Lian
2023-07-30 22:55:17 -04:00
parent e74eab6e73
commit 958d423e7c
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/axolotl/utils/trainer.py
View File

@@ -182,6 +182,7 @@ def setup_trainer(cfg, train_dataset, eval_dataset, model, tokenizer):
train_dataset = train_dataset.filter(drop_long).map( train_dataset = train_dataset.filter(drop_long).map(
add_position_ids, num_proc=os.cpu_count() add_position_ids, num_proc=os.cpu_count()
) )
if eval_dataset:
eval_dataset = eval_dataset.filter(drop_long).map( eval_dataset = eval_dataset.filter(drop_long).map(
add_position_ids, num_proc=os.cpu_count() add_position_ids, num_proc=os.cpu_count()
) )