fix up tokenizer config, isort fix

2023-05-30 23:00:02 -04:00
parent 2520ecd6df
commit 39a208c2bc
2 changed files with 13 additions and 7 deletions
--- a/scripts/finetune.py
+++ b/scripts/finetune.py
@@ -171,8 +171,9 @@ def train(
    validate_config(cfg)

    # load the tokenizer first
-    logging.info("loading tokenizer...")
-    tokenizer = load_tokenizer(cfg.base_model_config, cfg.tokenizer_type, cfg)
+    tokenizer_config = cfg.tokenizer_config or cfg.base_model_config
+    logging.info(f"loading tokenizer... {tokenizer_config}")
+    tokenizer = load_tokenizer(tokenizer_config, cfg.tokenizer_type, cfg)

    if check_not_in(
        ["inference", "shard", "merge_lora"], kwargs