tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

bugfix

Browse Source
This commit is contained in:
Dan Saunders
2025-03-17 22:57:55 +00:00
parent a09d1ccbf2
commit 411df76a97
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/axolotl/core/trainers/base.py
View File

@@ -613,7 +613,8 @@ class AxolotlTrainer(SchedulerMixin, OptimizerMixin, Trainer):
self.data_collator = ( # pylint: disable=attribute-defined-outside-init self.data_collator = ( # pylint: disable=attribute-defined-outside-init
self.eval_data_collator self.eval_data_collator
) )
eval_dataset = eval_dataset.remove_columns(["length"]) if "length" in eval_dataset.column_names:
eval_dataset = eval_dataset.remove_columns(["length"])
dataloader = super().get_eval_dataloader(eval_dataset) dataloader = super().get_eval_dataloader(eval_dataset)
self.data_collator = ( # pylint: disable=attribute-defined-outside-init self.data_collator = ( # pylint: disable=attribute-defined-outside-init
self.train_data_collator self.train_data_collator
@@ -633,7 +634,8 @@ class AxolotlTrainer(SchedulerMixin, OptimizerMixin, Trainer):
if hasattr(self, "eval_data_collator") and self.eval_data_collator if hasattr(self, "eval_data_collator") and self.eval_data_collator
else self.data_collator else self.data_collator
) )
eval_dataset = eval_dataset.remove_columns(["length"]) if "length" in eval_dataset.column_names:
eval_dataset = eval_dataset.remove_columns(["length"])
# Handle dataset preprocessing for SP # Handle dataset preprocessing for SP
if self.args.sequence_parallel_degree > 1: if self.args.sequence_parallel_degree > 1: