fix: high eval loss w/ sample packing (#3478) [skip ci]
* check if eval_sp * radable condition
This commit is contained in:
@@ -348,7 +348,9 @@ def _load_raw_datasets(
|
||||
dataset = handle_long_seq_in_dataset(dataset, cfg.eval_sequence_len, cfg)
|
||||
else:
|
||||
dataset = handle_long_seq_in_dataset(dataset, cfg.sequence_len, cfg)
|
||||
if cfg.sample_packing:
|
||||
if (split == "train" and cfg.sample_packing) or (
|
||||
split == "test" and cfg.eval_sample_packing
|
||||
):
|
||||
dataset, _ = process_datasets_for_packing(cfg, dataset, None)
|
||||
|
||||
# Deduplicate before saving so the saved dataset is already de-duplicated
|
||||
|
||||
Reference in New Issue
Block a user