tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

don't add position_ids for evals (#591)

Browse Source
This commit is contained in:
Wing Lian
2023-09-16 16:11:57 -04:00
committed by GitHub
parent 21ec195c9f
commit ab534d75ba
1 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/axolotl/utils/trainer.py
View File

@@ -403,10 +403,11 @@ def process_datasets_for_packing(cfg, train_dataset, eval_dataset):
if cfg.sample_packing: if cfg.sample_packing:
train_dataset = train_dataset.map(add_position_ids, num_proc=os.cpu_count()) train_dataset = train_dataset.map(add_position_ids, num_proc=os.cpu_count())
if eval_dataset: if cfg.eval_sample_packing is not False:
eval_dataset = eval_dataset.map( if eval_dataset:
add_position_ids, num_proc=os.cpu_count() eval_dataset = eval_dataset.map(
) add_position_ids, num_proc=os.cpu_count()
)
return train_dataset, eval_dataset return train_dataset, eval_dataset