tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

simplify load_tokenizer

Browse Source
This commit is contained in:
Aman Karmani
2023-08-13 01:33:38 +00:00
committed by Aman Gupta Karmani
parent 7b55fe6419
commit efb3b2c95e
3 changed files with 24 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/finetune.py
View File

@@ -177,9 +177,8 @@ def train(
setup_wandb_env_vars(cfg)
# load the tokenizer first
tokenizer_config = cfg.tokenizer_config or cfg.base_model_config
LOG.info(f"loading tokenizer... {tokenizer_config}")
tokenizer = load_tokenizer(tokenizer_config, cfg.tokenizer_type, cfg)
LOG.info(f"loading tokenizer... {cfg.tokenizer_config or cfg.base_model_config}")
tokenizer = load_tokenizer(cfg)
if (
check_not_in(["shard", "merge_lora"], kwargs) and not cfg.inference