Merge pull request #120 from OpenAccess-AI-Collective/model-from-path

split up llama model loading so config can be loaded from base config and models can be loaded from a path

scripts/finetune.py

@@ -173,8 +173,9 @@ def train(
         cfg.bf16 = False
 
     # load the tokenizer first
-    logging.info("loading tokenizer...")
-    tokenizer = load_tokenizer(cfg.base_model_config, cfg.tokenizer_type, cfg)
+    tokenizer_config = cfg.tokenizer_config or cfg.base_model_config
+    logging.info(f"loading tokenizer... {tokenizer_config}")
+    tokenizer = load_tokenizer(tokenizer_config, cfg.tokenizer_type, cfg)
 
     if check_not_in(
         ["inference", "shard", "merge_lora"], kwargs