Merge pull request #58 from OpenAccess-AI-Collective/shards-fix

shard fix
2023-05-25 16:32:31 -04:00
parent b2fb61845e ac79360161
commit be3d3963cd
1 changed files with 1 additions and 1 deletions
--- a/src/axolotl/utils/data.py
+++ b/src/axolotl/utils/data.py
@@ -112,7 +112,7 @@ def load_tokenized_prepared_datasets(
                raise Exception("unhandled dataset load")
            # support for using a subset of the data
            if d.shards:
-                ds = ds.shuffle(seed=42)["train"].shard(num_shards=cfg.shards, index=0)
+                ds = ds.shuffle(seed=42)["train"].shard(num_shards=d.shards, index=0)
            d_type = d.type
            d_type_split = d_type.split(":")
            d_base_type = d_type_split[0]