Black formatting
This commit is contained in:
@@ -107,7 +107,9 @@ def load_tokenized_prepared_datasets(
|
|||||||
use_auth_token=use_auth_token,
|
use_auth_token=use_auth_token,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
ds: Dataset = load_dataset(d.path, streaming=False, use_auth_token=use_auth_token)
|
ds: Dataset = load_dataset(
|
||||||
|
d.path, streaming=False, use_auth_token=use_auth_token
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
fp = hf_hub_download(
|
fp = hf_hub_download(
|
||||||
repo_id=d.path, repo_type="dataset", filename=d.data_files
|
repo_id=d.path, repo_type="dataset", filename=d.data_files
|
||||||
@@ -293,7 +295,8 @@ def load_prepare_datasets(
|
|||||||
f"Checking for packed prepared dataset from hub... {cfg.push_dataset_to_hub}/{ds_hash}"
|
f"Checking for packed prepared dataset from hub... {cfg.push_dataset_to_hub}/{ds_hash}"
|
||||||
)
|
)
|
||||||
dataset = load_dataset(
|
dataset = load_dataset(
|
||||||
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=use_auth_token
|
f"{cfg.push_dataset_to_hub}/{ds_hash}",
|
||||||
|
use_auth_token=use_auth_token,
|
||||||
)
|
)
|
||||||
dataset = dataset["train"]
|
dataset = dataset["train"]
|
||||||
except Exception: # pylint: disable=broad-except
|
except Exception: # pylint: disable=broad-except
|
||||||
|
|||||||
@@ -40,7 +40,9 @@ def validate_config(cfg):
|
|||||||
)
|
)
|
||||||
|
|
||||||
if cfg.push_dataset_to_hub and cfg.hf_use_auth_token is not True:
|
if cfg.push_dataset_to_hub and cfg.hf_use_auth_token is not True:
|
||||||
raise ValueError("Require cfg.hf_use_auth_token to be True for push_dataset_to_hub")
|
raise ValueError(
|
||||||
|
"Require cfg.hf_use_auth_token to be True for push_dataset_to_hub"
|
||||||
|
)
|
||||||
|
|
||||||
# TODO
|
# TODO
|
||||||
# MPT 7b
|
# MPT 7b
|
||||||
|
|||||||
@@ -117,4 +117,3 @@ class ValidationTest(unittest.TestCase):
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
validate_config(cfg)
|
validate_config(cfg)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user