tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

feat: print out dataset length even if not preprocess (#2034) [skip ci]

Browse Source
This commit is contained in:
NanoCode012
2024-11-13 22:06:00 +07:00
committed by GitHub
parent 810ebc2c0e
commit 7b78a31593
1 changed files with 4 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/axolotl/utils/trainer.py
View File

@@ -185,7 +185,6 @@ def process_datasets_for_packing(cfg, train_dataset, eval_dataset):
min_sequence_len=cfg.min_sample_len or 2, min_sequence_len=cfg.min_sample_len or 2,
) )
if cfg.is_preprocess:
min_input_len = np.min(get_dataset_lengths(train_dataset)) min_input_len = np.min(get_dataset_lengths(train_dataset))
LOG.debug(f"min_input_len: {min_input_len}", main_process_only=True) LOG.debug(f"min_input_len: {min_input_len}", main_process_only=True)
max_input_len = np.max(get_dataset_lengths(train_dataset)) max_input_len = np.max(get_dataset_lengths(train_dataset))