tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Add ability to pass 'name' argument to load_dataset

Browse Source
This commit is contained in:
Charles Goddard
2023-07-14 16:46:39 -07:00
parent 168a7a09cc
commit 88089e8b32
1 changed files with 13 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/axolotl/utils/data.py
View File

@@ -94,6 +94,7 @@ def load_tokenized_prepared_datasets(
try:
load_dataset(
d.path,
name=d.name,
streaming=True,
use_auth_token=use_auth_token,
)
@@ -107,6 +108,7 @@ def load_tokenized_prepared_datasets(
if local_path.is_dir():
ds = load_dataset(
d.path,
name=d.name,
data_files=d.data_files,
streaming=False,
split=None,
@@ -114,6 +116,7 @@ def load_tokenized_prepared_datasets(
elif local_path.is_file():
ds = load_dataset(
"json",
name=d.name,
data_files=d.path,
streaming=False,
split=None,
@@ -123,26 +126,22 @@ def load_tokenized_prepared_datasets(
"unhandled dataset load: local path exists, but is neither a directory or a file"
)
elif ds_from_hub:
if d.data_files:
ds = load_dataset(
d.path,
streaming=False,
data_files=d.data_files,
use_auth_token=use_auth_token,
)
else:
ds = load_dataset(
d.path,
streaming=False,
use_auth_token=use_auth_token,
)
ds = load_dataset(
d.path,
name=d.name,
streaming=False,
data_files=d.data_files,
use_auth_token=use_auth_token,
)
else:
fp = hf_hub_download(
repo_id=d.path,
repo_type="dataset",
filename=d.data_files,
)
ds = load_dataset("json", data_files=fp, streaming=False, split=None)
ds = load_dataset(
"json", name=d.name, data_files=fp, streaming=False, split=None
)
if not ds:
raise ValueError("unhandled dataset load")
# support for using a subset of the data