From 958d423e7c9cbea8b1ee2b4cf4a7ecd21c46dfb6 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Sun, 30 Jul 2023 22:55:17 -0400 Subject: [PATCH] only process eval dataset for packing if not None --- src/axolotl/utils/trainer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/axolotl/utils/trainer.py b/src/axolotl/utils/trainer.py index 72724d60a..e8a14df1f 100644 --- a/src/axolotl/utils/trainer.py +++ b/src/axolotl/utils/trainer.py @@ -182,9 +182,10 @@ def setup_trainer(cfg, train_dataset, eval_dataset, model, tokenizer): train_dataset = train_dataset.filter(drop_long).map( add_position_ids, num_proc=os.cpu_count() ) - eval_dataset = eval_dataset.filter(drop_long).map( - add_position_ids, num_proc=os.cpu_count() - ) + if eval_dataset: + eval_dataset = eval_dataset.filter(drop_long).map( + add_position_ids, num_proc=os.cpu_count() + ) if cfg.sample_packing_eff_est: total_num_tokens = ( cfg.total_num_tokens