logging improvements (#1808) [skip ci]

* logging improvements * fix sort
2024-08-06 10:31:50 -04:00
parent 35d5e59d78
commit c56e0a79a5
3 changed files with 10 additions and 5 deletions
--- a/src/axolotl/utils/data/sft.py
+++ b/src/axolotl/utils/data/sft.py
@@ -187,7 +187,12 @@ def load_tokenized_prepared_datasets(
    else:
        if cfg.push_dataset_to_hub:
            LOG.info("Unable to find prepared dataset in Huggingface hub")
-        LOG.info(f"Unable to find prepared dataset in {prepared_ds_path}")
+        if cfg.is_preprocess:
+            LOG.info(
+                f"Skipping prepared dataset in {prepared_ds_path} for pre-processing..."
+            )
+        else:
+            LOG.info(f"Unable to find prepared dataset in {prepared_ds_path}")
        LOG.info("Loading raw datasets...")
        if not cfg.is_preprocess:
            LOG.warning(