tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

don't use tokenizer parallelism when using packing (#2862) [skip ci]

Browse Source
This commit is contained in:
Wing Lian
2025-07-06 21:55:09 -04:00
committed by GitHub
parent bf38e507fb
commit b37ddf9778
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/axolotl/utils/trainer.py
View File

@@ -609,6 +609,9 @@ def prepare_opinionated_env(cfg):
if cfg.qlora_sharded_model_loading: if cfg.qlora_sharded_model_loading:
# model loading is forked after the tokenizer # model loading is forked after the tokenizer
os.environ["TOKENIZERS_PARALLELISM"] = "false" os.environ["TOKENIZERS_PARALLELISM"] = "false"
if cfg.sample_packing:
# multipack parallel packing sampler defaults to using fork
os.environ["TOKENIZERS_PARALLELISM"] = "false"
def setup_trainer( def setup_trainer(