tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 4 Packages Projects Releases Wiki Activity

helpful info output

Browse Source
This commit is contained in:
Wing Lian
2023-04-15 00:03:43 -04:00
parent 902dd0ab47
commit 937f44f021
3 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
configs/llama_65B_alpaca.yml
View File

@@ -1,4 +1,4 @@
base_model: huggyllama/llama-7b base_model: huggyllama/llama-65b
model_type: LlamaForCausalLM model_type: LlamaForCausalLM
tokenizer_type: LlamaTokenizer tokenizer_type: LlamaTokenizer
load_in_8bit: true load_in_8bit: true

3
requirements.txt
View File

@@ -1,5 +1,5 @@
git+https://github.com/huggingface/transformers.git
git+https://github.com/huggingface/peft.git git+https://github.com/huggingface/peft.git
git+https://github.com/huggingface/transformers.git
attrdict attrdict
fire fire
PyYAML==6.0 PyYAML==6.0
@@ -12,4 +12,3 @@ wandb
flash-attn flash-attn
deepspeed deepspeed
einops einops

3
scripts/finetune.py
View File

@@ -258,7 +258,9 @@ def train(
datasets = [] datasets = []
if not isinstance(cfg.datasets, list) and isinstance(cfg.datasets, str): if not isinstance(cfg.datasets, list) and isinstance(cfg.datasets, str):
# assumption that we are loading a previously saved/cached dataset # assumption that we are loading a previously saved/cached dataset
print("Loading prepared dataset from disk...")
dataset = load_from_disk(cfg.datasets) dataset = load_from_disk(cfg.datasets)
print("Prepared dataset loaded from disk...")
else: else:
for d in cfg.datasets: for d in cfg.datasets:
ds: IterableDataset = load_dataset( ds: IterableDataset = load_dataset(
@@ -289,6 +291,7 @@ def train(
dataset = Dataset.from_list( dataset = Dataset.from_list(
[_ for _ in constant_len_dataset] [_ for _ in constant_len_dataset]
).train_test_split(test_size=cfg.val_set_size, shuffle=True, seed=42) ).train_test_split(test_size=cfg.val_set_size, shuffle=True, seed=42)
print("Saving prepared dataset to disk...")
dataset.save_to_disk("data/last_run") dataset.save_to_disk("data/last_run")
train_dataset = dataset["train"] train_dataset = dataset["train"]