tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fixes w/ example for super basic lora starter

Browse Source
This commit is contained in:
Wing Lian
2023-05-25 11:59:08 -04:00
parent 951facbb1f
commit a5d739b66b
3 changed files with 74 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/axolotl/prompters.py
View File

@@ -18,7 +18,7 @@ class AlpacaPrompter:
prompt_style = None
def __init__(self, prompt_style="instruct"):
self.prompt_style = prompt_style
self.prompt_style = prompt_style if prompt_style else PromptStyle.instruct.value
self.match_prompt_style()
def match_prompt_style(self):

10
src/axolotl/utils/data.py
View File

@@ -60,10 +60,12 @@ def load_tokenized_prepared_datasets(
else Path(default_dataset_prepared_path) / ds_hash
)
dataset = None
use_auth_token = False
try:
if cfg.push_dataset_to_hub:
use_auth_token = True
dataset = load_dataset(
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=True
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=use_auth_token
)
dataset = dataset["train"]
except:
@@ -83,7 +85,7 @@ def load_tokenized_prepared_datasets(
ds = None
ds_from_hub = False
try:
load_dataset(d.path, streaming=True, use_auth_token=True)
load_dataset(d.path, streaming=True, use_auth_token=use_auth_token)
ds_from_hub = True
except FileNotFoundError:
pass
@@ -99,10 +101,10 @@ def load_tokenized_prepared_datasets(
d.path,
streaming=False,
data_files=d.data_files,
use_auth_token=True,
use_auth_token=use_auth_token,
)
else:
ds = load_dataset(d.path, streaming=False, use_auth_token=True)
ds = load_dataset(d.path, streaming=False, use_auth_token=use_auth_token)
else:
fp = hf_hub_download(
repo_id=d.path, repo_type="dataset", filename=d.data_files