tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

optimization if total_num_tokens is already known

Browse Source
This commit is contained in:
Wing Lian
2023-08-10 19:01:17 -04:00
parent 2565c2f259
commit ac4b700daa
1 changed files with 1 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/axolotl/utils/dataloader.py
View File

@@ -261,9 +261,7 @@ class MultipackDistributedDataloader:
batch_gen_thread.join() batch_gen_thread.join()
def _len_est(self): def _len_est(self):
indices = range(0, len(self.dataset)) lengths_sum = np.sum(self.lengths)
lengths = self.lengths[indices]
lengths_sum = np.cumsum(lengths)[-1]
lengths_sum_per_device = lengths_sum // self.device_count lengths_sum_per_device = lengths_sum // self.device_count
LOG.info( LOG.info(
f"packing_efficiency_estimate: {self.packing_efficiency_estimate} " f"packing_efficiency_estimate: {self.packing_efficiency_estimate} "