fix: loading locally downloaded dataset (#2056) [skip ci]

2024-11-16 08:35:26 +07:00
parent d42f202046
commit fd70eec577
2 changed files with 79 additions and 36 deletions
--- a/src/axolotl/utils/data/sft.py
+++ b/src/axolotl/utils/data/sft.py
@@ -350,7 +350,15 @@ def load_tokenized_prepared_datasets(
                            split=None,
                        )
                    else:
-                        ds = load_from_disk(config_dataset.path)
+                        try:
+                            ds = load_from_disk(config_dataset.path)
+                        except FileNotFoundError:
+                            ds = load_dataset(
+                                config_dataset.path,
+                                name=config_dataset.name,
+                                streaming=False,
+                                split=None,
+                            )
                elif local_path.is_file():
                    ds_type = get_ds_type(config_dataset)