tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

change auth token setting back

Browse Source
This commit is contained in:
Wing Lian
2023-05-25 12:41:17 -04:00
parent 98b1bce57e
commit d2a6f79fd1
1 changed files with 3 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/axolotl/utils/data.py
View File

@@ -60,12 +60,11 @@ def load_tokenized_prepared_datasets(
else Path(default_dataset_prepared_path) / ds_hash
)
dataset = None
use_auth_token = False
try:
if cfg.push_dataset_to_hub:
use_auth_token = True
dataset = load_dataset(
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=use_auth_token
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=True
)
dataset = dataset["train"]
except:
@@ -85,7 +84,7 @@ def load_tokenized_prepared_datasets(
ds = None
ds_from_hub = False
try:
load_dataset(d.path, streaming=True, use_auth_token=use_auth_token)
load_dataset(d.path, streaming=True, use_auth_token=True)
ds_from_hub = True
except FileNotFoundError:
pass
@@ -104,7 +103,7 @@ def load_tokenized_prepared_datasets(
use_auth_token=use_auth_token,
)
else:
ds = load_dataset(d.path, streaming=False, use_auth_token=use_auth_token)
ds = load_dataset(d.path, streaming=False, use_auth_token=True)
else:
fp = hf_hub_download(
repo_id=d.path, repo_type="dataset", filename=d.data_files