tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Merge pull request #166 from NanoCode012/fix/seed

Browse Source 
Fix: Set to use cfg.seed or 42 for seed
This commit is contained in:
NanoCode012
2023-06-09 01:15:08 +09:00
committed by GitHub
parent c8242de725 2cfe9e9b16
commit 7eae90333e
2 changed files with 14 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/axolotl/utils/data.py
View File

@@ -78,6 +78,13 @@ def load_tokenized_prepared_datasets(
else: else:
logging.info(f"Unable to find prepared dataset in {prepared_ds_path}") logging.info(f"Unable to find prepared dataset in {prepared_ds_path}")
logging.info("Loading raw datasets...") logging.info("Loading raw datasets...")
if cfg.seed:
seed = cfg.seed
else:
logging.info("No seed provided, using default seed of 42")
seed = 42
datasets = [] datasets = []
# pylint: disable=invalid-name # pylint: disable=invalid-name
for d in cfg.datasets: for d in cfg.datasets:
@@ -127,11 +134,11 @@ def load_tokenized_prepared_datasets(
# support for using a subset of the data # support for using a subset of the data
if d.shards: if d.shards:
if "train" in ds: if "train" in ds:
ds = ds.shuffle(seed=42)["train"].shard( ds = ds.shuffle(seed=seed)["train"].shard(
num_shards=d.shards, index=0 num_shards=d.shards, index=0
) )
else: else:
ds = ds.shuffle(seed=42).shard(num_shards=d.shards, index=0) ds = ds.shuffle(seed=seed).shard(num_shards=d.shards, index=0)
d_type = d.type d_type = d.type
d_type_split = d_type.split(":") d_type_split = d_type.split(":")
d_base_type = d_type_split[0] d_base_type = d_type_split[0]
@@ -239,7 +246,7 @@ def load_tokenized_prepared_datasets(
samples: List[int] = [] samples: List[int] = []
for d in datasets: for d in datasets:
samples = samples + list(d) samples = samples + list(d)
dataset = Dataset.from_list(samples).shuffle(seed=42) dataset = Dataset.from_list(samples).shuffle(seed=seed)
if cfg.local_rank == 0: if cfg.local_rank == 0:
logging.info( logging.info(
f"Saving merged prepared dataset to disk... {prepared_ds_path}" f"Saving merged prepared dataset to disk... {prepared_ds_path}"

4
src/axolotl/utils/trainer.py
View File

@@ -74,6 +74,10 @@ def setup_trainer(cfg, train_dataset, eval_dataset, model, tokenizer):
training_arguments_kwargs["tf32"] = cfg.tf32 training_arguments_kwargs["tf32"] = cfg.tf32
training_arguments_kwargs["warmup_steps"] = warmup_steps training_arguments_kwargs["warmup_steps"] = warmup_steps
training_arguments_kwargs["logging_steps"] = logging_steps training_arguments_kwargs["logging_steps"] = logging_steps
if cfg.seed:
training_arguments_kwargs["seed"] = cfg.seed
if cfg.gradient_checkpointing: if cfg.gradient_checkpointing:
if cfg.gptq: if cfg.gptq:
from alpaca_lora_4bit.gradient_checkpointing import ( from alpaca_lora_4bit.gradient_checkpointing import (