only process eval dataset for packing if not None

2023-07-30 22:55:17 -04:00
parent e74eab6e73
commit 958d423e7c
1 changed files with 4 additions and 3 deletions
--- a/src/axolotl/utils/trainer.py
+++ b/src/axolotl/utils/trainer.py
@@ -182,6 +182,7 @@ def setup_trainer(cfg, train_dataset, eval_dataset, model, tokenizer):
        train_dataset = train_dataset.filter(drop_long).map(
            add_position_ids, num_proc=os.cpu_count()
        )
+        if eval_dataset:
            eval_dataset = eval_dataset.filter(drop_long).map(
                add_position_ids, num_proc=os.cpu_count()
            )