tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

missed ...

Browse Source
This commit is contained in:
Wing Lian
2023-05-25 12:42:56 -04:00
parent d2a6f79fd1
commit 943961fd10
1 changed files with 1 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/axolotl/utils/data.py
View File

@@ -62,7 +62,6 @@ def load_tokenized_prepared_datasets(
dataset = None dataset = None
try: try:
if cfg.push_dataset_to_hub: if cfg.push_dataset_to_hub:
use_auth_token = True
dataset = load_dataset( dataset = load_dataset(
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=True f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=True
) )
@@ -100,7 +99,7 @@ def load_tokenized_prepared_datasets(
d.path, d.path,
streaming=False, streaming=False,
data_files=d.data_files, data_files=d.data_files,
use_auth_token=use_auth_token, use_auth_token=True,
) )
else: else:
ds = load_dataset(d.path, streaming=False, use_auth_token=True) ds = load_dataset(d.path, streaming=False, use_auth_token=True)