tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Fix: Future deprecation warning with use_auth_token (#680)

Browse Source
This commit is contained in:
NanoCode012
2023-10-06 03:56:18 +09:00
committed by GitHub
parent e0b7eeabfd
commit 69fac9a020
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/axolotl/utils/data.py
View File

@@ -114,7 +114,7 @@ def load_tokenized_prepared_datasets(
if cfg.push_dataset_to_hub: if cfg.push_dataset_to_hub:
dataset = load_dataset( dataset = load_dataset(
f"{cfg.push_dataset_to_hub}/{ds_hash}", f"{cfg.push_dataset_to_hub}/{ds_hash}",
use_auth_token=use_auth_token, token=use_auth_token,
) )
dataset = dataset["train"] dataset = dataset["train"]
except Exception: # pylint: disable=broad-except # nosec except Exception: # pylint: disable=broad-except # nosec
@@ -155,7 +155,7 @@ def load_tokenized_prepared_datasets(
d.path, d.path,
name=d.name, name=d.name,
streaming=True, streaming=True,
use_auth_token=use_auth_token, token=use_auth_token,
) )
ds_from_hub = True ds_from_hub = True
except FileNotFoundError: except FileNotFoundError:
@@ -202,7 +202,7 @@ def load_tokenized_prepared_datasets(
name=d.name, name=d.name,
streaming=False, streaming=False,
data_files=d.data_files, data_files=d.data_files,
use_auth_token=use_auth_token, token=use_auth_token,
) )
else: else:
if isinstance(d.data_files, str): if isinstance(d.data_files, str):
@@ -417,7 +417,7 @@ def load_prepare_datasets(
) )
dataset = load_dataset( dataset = load_dataset(
f"{cfg.push_dataset_to_hub}/{ds_hash}", f"{cfg.push_dataset_to_hub}/{ds_hash}",
use_auth_token=use_auth_token, token=use_auth_token,
) )
dataset = dataset["train"] dataset = dataset["train"]
except Exception: # pylint: disable=broad-except # nosec except Exception: # pylint: disable=broad-except # nosec