split completion text to sequence_len (#616)

src/axolotl/datasets.py

@@ -38,10 +38,15 @@ class TokenizedPromptDataset(Dataset):
     def process(self, dataset):
         features = dataset.features.keys()
         num_proc = min(64, os.cpu_count())
+        map_kwargs = {}
+        if self.prompt_tokenizer.supports_batched:
+            map_kwargs["batched"] = True
+            map_kwargs["batch_size"] = 100
         return dataset.map(
             self.prompt_tokenizer.tokenize_prompt,
             num_proc=num_proc,
             remove_columns=features,
+            **map_kwargs,
         )