fix batch size calculation

2023-05-31 14:11:32 -04:00
parent f94dd626f0
commit 5a631b305b
2 changed files with 6 additions and 3 deletions
--- a/src/axolotl/utils/data.py
+++ b/src/axolotl/utils/data.py
@@ -233,6 +233,7 @@ def load_tokenized_prepared_datasets(
                datasets.append(ds_wrapper)
            else:
                logging.error(f"unhandled prompt tokenization strategy: {d.type}")
+                raise ValueError(f"unhandled prompt tokenization strategy: {d.type}")
        logging.info("tokenizing, merging, and shuffling master dataset")

        samples: List[int] = []