recommend padding when using sample packing (#531)

This commit is contained in:
Wing Lian
2023-09-06 17:00:21 -04:00
committed by GitHub
parent 245c5c41e2
commit 343714972b
11 changed files with 28 additions and 0 deletions

View File

@@ -97,6 +97,11 @@ def validate_config(cfg):
)
)
if cfg.sample_packing and not cfg.pad_to_sequence_len:
LOG.warning(
"`pad_to_sequence_len: true` is recommended when using sample_packing"
)
if cfg.gradient_accumulation_steps and cfg.batch_size:
raise ValueError(
"please set only one of gradient_accumulation_steps or batch_size"