tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

remove un-necessary zero-first guard as it's already only called in a parent fn (#1810) [skip ci]

Browse Source
This commit is contained in:
Wing Lian
2024-08-06 09:29:23 -04:00
committed by GitHub
parent ecdda006de
commit fbbeb4fee0
1 changed files with 73 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/axolotl/utils/trainer.py
View File

@@ -16,7 +16,7 @@ from torch.utils.data import DataLoader, RandomSampler
from transformers.utils import is_torch_bf16_gpu_available from transformers.utils import is_torch_bf16_gpu_available
from axolotl.core.trainer_builder import HFCausalTrainerBuilder, HFRLTrainerBuilder from axolotl.core.trainer_builder import HFCausalTrainerBuilder, HFRLTrainerBuilder
from axolotl.utils.distributed import is_main_process, reduce_and_broadcast, zero_first from axolotl.utils.distributed import reduce_and_broadcast
from axolotl.utils.samplers import MultipackBatchSampler, get_dataset_lengths from axolotl.utils.samplers import MultipackBatchSampler, get_dataset_lengths
LOG = get_logger("axolotl") LOG = get_logger("axolotl")
@@ -183,7 +183,7 @@ def process_datasets_for_packing(cfg, train_dataset, eval_dataset):
sequence_len=cfg.sequence_len, sequence_len=cfg.sequence_len,
min_sequence_len=cfg.min_sample_len or 2, min_sequence_len=cfg.min_sample_len or 2,
) )
with zero_first(is_main_process()):
if cfg.is_preprocess: if cfg.is_preprocess:
min_input_len = np.min(get_dataset_lengths(train_dataset)) min_input_len = np.min(get_dataset_lengths(train_dataset))
LOG.debug(f"min_input_len: {min_input_len}", main_process_only=True) LOG.debug(f"min_input_len: {min_input_len}", main_process_only=True)