tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

handle possibly empty batch

Browse Source
This commit is contained in:
Dan Saunders
2025-06-26 10:59:27 -04:00
parent 46675496a3
commit 4ebd4aae3d
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/axolotl/monkeypatch/data/batch_dataset_fetcher.py
View File

@@ -9,6 +9,9 @@ from torch.utils.data._utils.worker import _worker_loop
class _MapDatasetFetcher(_BaseDatasetFetcher): class _MapDatasetFetcher(_BaseDatasetFetcher):
def fetch(self, possibly_batched_index): def fetch(self, possibly_batched_index):
if not possibly_batched_index:
return self.collate_fn([])
if isinstance(possibly_batched_index[0], list): if isinstance(possibly_batched_index[0], list):
data = [None for i in possibly_batched_index] data = [None for i in possibly_batched_index]
for i, possibly_batched_index_ in enumerate(possibly_batched_index): for i, possibly_batched_index_ in enumerate(possibly_batched_index):