From 1cfb8feb2ded1ffaeee93fb5101f554610e0add6 Mon Sep 17 00:00:00 2001
From: Wing Lian <wing@axolotl.ai>
Date: Mon, 27 Jan 2025 14:31:12 -0500
Subject: [PATCH] add iterable argument to preprocess-cli

---
 src/axolotl/cli/args.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/axolotl/cli/args.py b/src/axolotl/cli/args.py
index 0618e07f1..d451b45a6 100644
--- a/src/axolotl/cli/args.py
+++ b/src/axolotl/cli/args.py
@@ -13,6 +13,12 @@ class PreprocessCliArgs:
     debug_num_examples: int = field(default=1)
     prompter: Optional[str] = field(default=None)
     download: Optional[bool] = field(default=True)
+    iterable: Optional[bool] = field(
+        default=None,
+        metadata={
+            "help": "Use IterableDataset for streaming processing of large datasets"
+        },
+    )
 
 
 @dataclass