diff --git a/README.md b/README.md index 067ef0d05..8dbb535cd 100644 --- a/README.md +++ b/README.md @@ -505,6 +505,9 @@ torchdistx_path: # Set padding for data collator to 'longest' collator_pad_to_longest: +# Set to HF dataset for type: 'completion' for streaming instead of pre-tokenize +pretraining_dataset: + # Debug mode debug: