diff --git a/src/axolotl/cli/args.py b/src/axolotl/cli/args.py index 0618e07f1..d451b45a6 100644 --- a/src/axolotl/cli/args.py +++ b/src/axolotl/cli/args.py @@ -13,6 +13,12 @@ class PreprocessCliArgs: debug_num_examples: int = field(default=1) prompter: Optional[str] = field(default=None) download: Optional[bool] = field(default=True) + iterable: Optional[bool] = field( + default=None, + metadata={ + "help": "Use IterableDataset for streaming processing of large datasets" + }, + ) @dataclass