tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

change auth token setting back

Browse Source
This commit is contained in:
Wing Lian
2023-05-25 12:41:17 -04:00
parent 98b1bce57e
commit d2a6f79fd1
1 changed files with 3 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/axolotl/utils/data.py
View File

@@ -60,12 +60,11 @@ def load_tokenized_prepared_datasets(
else Path(default_dataset_prepared_path) / ds_hash else Path(default_dataset_prepared_path) / ds_hash
) )
dataset = None dataset = None
use_auth_token = False
try: try:
if cfg.push_dataset_to_hub: if cfg.push_dataset_to_hub:
use_auth_token = True use_auth_token = True
dataset = load_dataset( dataset = load_dataset(
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=use_auth_token f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=True
) )
dataset = dataset["train"] dataset = dataset["train"]
except: except:
@@ -85,7 +84,7 @@ def load_tokenized_prepared_datasets(
ds = None ds = None
ds_from_hub = False ds_from_hub = False
try: try:
load_dataset(d.path, streaming=True, use_auth_token=use_auth_token) load_dataset(d.path, streaming=True, use_auth_token=True)
ds_from_hub = True ds_from_hub = True
except FileNotFoundError: except FileNotFoundError:
pass pass
@@ -104,7 +103,7 @@ def load_tokenized_prepared_datasets(
use_auth_token=use_auth_token, use_auth_token=use_auth_token,
) )
else: else:
ds = load_dataset(d.path, streaming=False, use_auth_token=use_auth_token) ds = load_dataset(d.path, streaming=False, use_auth_token=True)
else: else:
fp = hf_hub_download( fp = hf_hub_download(
repo_id=d.path, repo_type="dataset", filename=d.data_files repo_id=d.path, repo_type="dataset", filename=d.data_files