Feat(data): Allow loading local csv and text (#594)

* Feat(data): Allow loading local csv and text * chore: update readme for loading data
2023-09-18 00:32:27 +09:00
parent b15b19eb8d
commit 00dce35fb2
2 changed files with 8 additions and 4 deletions
--- a/src/axolotl/utils/data.py
+++ b/src/axolotl/utils/data.py
@@ -183,6 +183,10 @@ def load_tokenized_prepared_datasets(
                        ds_type = "parquet"
                    elif ".arrow" in d.path:
                        ds_type = "arrow"
+                    elif ".csv" in d.path:
+                        ds_type = "csv"
+                    elif ".txt" in d.path:
+                        ds_type = "text"
                    ds = load_dataset(
                        ds_type,
                        name=d.name,