Black formatting

This commit is contained in:
NanoCode012
2023-05-29 15:30:28 +09:00
parent afb31e13a3
commit b832a0ac62
3 changed files with 8 additions and 4 deletions

View File

@@ -107,7 +107,9 @@ def load_tokenized_prepared_datasets(
use_auth_token=use_auth_token, use_auth_token=use_auth_token,
) )
else: else:
ds: Dataset = load_dataset(d.path, streaming=False, use_auth_token=use_auth_token) ds: Dataset = load_dataset(
d.path, streaming=False, use_auth_token=use_auth_token
)
else: else:
fp = hf_hub_download( fp = hf_hub_download(
repo_id=d.path, repo_type="dataset", filename=d.data_files repo_id=d.path, repo_type="dataset", filename=d.data_files
@@ -293,7 +295,8 @@ def load_prepare_datasets(
f"Checking for packed prepared dataset from hub... {cfg.push_dataset_to_hub}/{ds_hash}" f"Checking for packed prepared dataset from hub... {cfg.push_dataset_to_hub}/{ds_hash}"
) )
dataset = load_dataset( dataset = load_dataset(
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=use_auth_token f"{cfg.push_dataset_to_hub}/{ds_hash}",
use_auth_token=use_auth_token,
) )
dataset = dataset["train"] dataset = dataset["train"]
except Exception: # pylint: disable=broad-except except Exception: # pylint: disable=broad-except

View File

@@ -40,7 +40,9 @@ def validate_config(cfg):
) )
if cfg.push_dataset_to_hub and cfg.hf_use_auth_token is not True: if cfg.push_dataset_to_hub and cfg.hf_use_auth_token is not True:
raise ValueError("Require cfg.hf_use_auth_token to be True for push_dataset_to_hub") raise ValueError(
"Require cfg.hf_use_auth_token to be True for push_dataset_to_hub"
)
# TODO # TODO
# MPT 7b # MPT 7b

View File

@@ -117,4 +117,3 @@ class ValidationTest(unittest.TestCase):
} }
) )
validate_config(cfg) validate_config(cfg)