tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Merge pull request #58 from OpenAccess-AI-Collective/shards-fix

Browse Source 
shard fix
This commit is contained in:
Wing Lian
2023-05-25 16:32:31 -04:00
committed by GitHub
parent b2fb61845e ac79360161
commit be3d3963cd
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/axolotl/utils/data.py
View File

@@ -112,7 +112,7 @@ def load_tokenized_prepared_datasets(
raise Exception("unhandled dataset load") raise Exception("unhandled dataset load")
# support for using a subset of the data # support for using a subset of the data
if d.shards: if d.shards:
ds = ds.shuffle(seed=42)["train"].shard(num_shards=cfg.shards, index=0) ds = ds.shuffle(seed=42)["train"].shard(num_shards=d.shards, index=0)
d_type = d.type d_type = d.type
d_type_split = d_type.split(":") d_type_split = d_type.split(":")
d_base_type = d_type_split[0] d_base_type = d_type_split[0]