tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

new hf_use_auth_token setting so login to hf isn't required

Browse Source
This commit is contained in:
Wing Lian
2023-05-28 13:08:49 -04:00
parent a798ba1659
commit 1c33eb88a7
4 changed files with 39 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/axolotl/utils/data.py
View File

@@ -61,10 +61,11 @@ def load_tokenized_prepared_datasets(
else Path(default_dataset_prepared_path) / ds_hash
)
dataset = None
use_auth_token = cfg.hf_use_auth_token
try:
if cfg.push_dataset_to_hub:
dataset = load_dataset(
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=True
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=use_auth_token
)
dataset = dataset["train"]
except:
@@ -84,7 +85,7 @@ def load_tokenized_prepared_datasets(
ds: Union[Dataset, DatasetDict] = None
ds_from_hub = False
try:
load_dataset(d.path, streaming=True, use_auth_token=True)
load_dataset(d.path, streaming=True, use_auth_token=use_auth_token)
ds_from_hub = True
except FileNotFoundError:
pass
@@ -100,10 +101,10 @@ def load_tokenized_prepared_datasets(
d.path,
streaming=False,
data_files=d.data_files,
use_auth_token=True,
use_auth_token=use_auth_token,
)
else:
ds: Dataset = load_dataset(d.path, streaming=False, use_auth_token=True)
ds: Dataset = load_dataset(d.path, streaming=False, use_auth_token=use_auth_token)
else:
fp = hf_hub_download(
repo_id=d.path, repo_type="dataset", filename=d.data_files
@@ -274,13 +275,14 @@ def load_prepare_datasets(
)
dataset = None
use_auth_token = cfg.hf_use_auth_token
try:
if cfg.push_dataset_to_hub:
logging.info(
f"Checking for packed prepared dataset from hub... {cfg.push_dataset_to_hub}/{ds_hash}"
)
dataset = load_dataset(
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=True
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=use_auth_token
)
dataset = dataset["train"]
except:

3
src/axolotl/utils/validation.py
View File

@@ -37,6 +37,9 @@ def validate_config(cfg):
"`trust_remote_code` is set to true. Please make sure that you reviewed the remote code/model."
)
if cfg.push_dataset_to_hub and cfg.hf_use_auth_token is not True:
raise ValueError("Require cfg.hf_use_auth_token to be True for push_dataset_to_hub")
# TODO
# MPT 7b
# https://github.com/facebookresearch/bitsandbytes/issues/25