tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: print out dataset length even if not preprocess (#2034) [skip ci]

Browse Source
This commit is contained in:
NanoCode012
2024-11-13 22:06:00 +07:00
committed by GitHub
parent 810ebc2c0e
commit 7b78a31593
1 changed files with 4 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/axolotl/utils/trainer.py
View File

@@ -185,11 +185,10 @@ def process_datasets_for_packing(cfg, train_dataset, eval_dataset):
min_sequence_len=cfg.min_sample_len or 2, min_sequence_len=cfg.min_sample_len or 2,
) )
if cfg.is_preprocess: min_input_len = np.min(get_dataset_lengths(train_dataset))
min_input_len = np.min(get_dataset_lengths(train_dataset)) LOG.debug(f"min_input_len: {min_input_len}", main_process_only=True)
LOG.debug(f"min_input_len: {min_input_len}", main_process_only=True) max_input_len = np.max(get_dataset_lengths(train_dataset))
max_input_len = np.max(get_dataset_lengths(train_dataset)) LOG.debug(f"max_input_len: {max_input_len}", main_process_only=True)
LOG.debug(f"max_input_len: {max_input_len}", main_process_only=True)
if cfg.model_config_type == "mamba": if cfg.model_config_type == "mamba":
LOG.info("dropping attention_mask column") LOG.info("dropping attention_mask column")