add iterable argument to preprocess-cli

2025-01-27 14:31:12 -05:00
parent 887513285d
commit 1cfb8feb2d
1 changed files with 6 additions and 0 deletions
--- a/src/axolotl/cli/args.py
+++ b/src/axolotl/cli/args.py
@@ -13,6 +13,12 @@ class PreprocessCliArgs:
    debug_num_examples: int = field(default=1)
    prompter: Optional[str] = field(default=None)
    download: Optional[bool] = field(default=True)
+    iterable: Optional[bool] = field(
+        default=None,
+        metadata={
+            "help": "Use IterableDataset for streaming processing of large datasets"
+        },
+    )


@dataclass