tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Disable datasets caching when preparing dataset for packing

Browse Source
This commit is contained in:
Casper
2024-01-15 23:48:24 +01:00
parent c1b741d9fb
commit eea6e8303a
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/axolotl/utils/trainer.py
View File

@@ -108,7 +108,7 @@ def disable_datasets_caching():
def process_datasets_for_packing(cfg, train_dataset, eval_dataset, tokenizer): def process_datasets_for_packing(cfg, train_dataset, eval_dataset, tokenizer):
drop_long = partial(drop_long_seq, sequence_len=cfg.sequence_len) drop_long = partial(drop_long_seq, sequence_len=cfg.sequence_len)
with zero_first(is_main_process()): with zero_first(is_main_process()), disable_datasets_caching():
if cfg.group_by_length: if cfg.group_by_length:
train_dataset = train_dataset.map( train_dataset = train_dataset.map(
add_length, num_proc=cfg.dataset_processes add_length, num_proc=cfg.dataset_processes