tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Fix: remove the numerous sequential log (#2461)

Browse Source 
* fix: remove sequential logs

* feat(doc): add for sample pack sequentially and curriculum sampling
This commit is contained in:
NanoCode012
2025-04-01 20:20:00 +07:00
committed by GitHub
parent 9b95e06cbb
commit f4ae8816bb
2 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/config.qmd
View File

@@ -320,9 +320,13 @@ total_num_tokens:
sample_packing_group_size: 100000 sample_packing_group_size: 100000
# The number of samples which can be packed into one sequence. Increase if using a large sequence_len with many short samples. # The number of samples which can be packed into one sequence. Increase if using a large sequence_len with many short samples.
sample_packing_bin_size: 200 sample_packing_bin_size: 200
sample_pack_sequentially: # Optional[bool]. Whether to pack samples sequentially.
# whether to concatenate samples during pretraining # whether to concatenate samples during pretraining
pretraining_sample_concatenation: pretraining_sample_concatenation:
curriculum_sampling: # Optional[bool]. Whether to use sequential sampling for curriculum learning
# Use batch flattening for speedups when not using sample_packing # Use batch flattening for speedups when not using sample_packing
batch_flattening: batch_flattening:

6
src/axolotl/utils/samplers/multipack.py
View File

@@ -12,7 +12,9 @@ from torch.utils.data import BatchSampler, Sampler, SequentialSampler
from axolotl.utils.distributed import reduce_and_broadcast from axolotl.utils.distributed import reduce_and_broadcast
LOG = logging.getLogger("axolotl.utils.samplers.multipack") LOG = logging.getLogger(__name__)
LOG.setLevel(logging.INFO)
@numba.njit @numba.njit
@@ -202,7 +204,6 @@ class MultipackBatchSampler(BatchSampler):
lengths_cumsum = np.cumsum(lengths) lengths_cumsum = np.cumsum(lengths)
if self.sequential: if self.sequential:
LOG.debug("using sequential sample packing algorithm")
batches, total_used, total_slots = allocate_sequentially( batches, total_used, total_slots = allocate_sequentially(
lengths=lengths, lengths=lengths,
rank=0, rank=0,
@@ -210,7 +211,6 @@ class MultipackBatchSampler(BatchSampler):
n=1, n=1,
) )
else: else:
LOG.debug("using non-sequential sample packing algorithm")
batches, total_used, total_slots = allocate( batches, total_used, total_slots = allocate(
lengths=lengths, lengths=lengths,
lengths_cumsum=lengths_cumsum, lengths_cumsum=lengths_cumsum,