From e3966543199c23df068f37cce18f73defa43cdb7 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Thu, 25 May 2023 12:15:12 -0400 Subject: [PATCH] fix tokenizer loading, got openllama 3b working --- .../{lora-alpaca-7b => lora-openllama-3b}/config.yml | 10 +++++----- src/axolotl/utils/models.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) rename examples/{lora-alpaca-7b => lora-openllama-3b}/config.yml (86%) diff --git a/examples/lora-alpaca-7b/config.yml b/examples/lora-openllama-3b/config.yml similarity index 86% rename from examples/lora-alpaca-7b/config.yml rename to examples/lora-openllama-3b/config.yml index 0499b265f..393942d96 100644 --- a/examples/lora-alpaca-7b/config.yml +++ b/examples/lora-openllama-3b/config.yml @@ -1,5 +1,5 @@ -base_model: huggyllama/llama-7b -base_model_config: huggyllama/llama-7b +base_model: openlm-research/open_llama_3b_600bt_preview +base_model_config: openlm-research/open_llama_3b_600bt_preview model_type: LlamaForCausalLM tokenizer_type: LlamaTokenizer load_in_8bit: true @@ -32,9 +32,9 @@ wandb_watch: wandb_run_id: wandb_log_model: output_dir: ./lora-out -batch_size: 4 -micro_batch_size: 1 -num_epochs: 4 +batch_size: 16 +micro_batch_size: 4 +num_epochs: 3 optimizer: adamw_bnb_8bit torchdistx_path: lr_scheduler: cosine diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py index 5b243bec4..de04e9333 100644 --- a/src/axolotl/utils/models.py +++ b/src/axolotl/utils/models.py @@ -211,12 +211,12 @@ def load_model( try: if is_llama_derived_model and "LlamaTokenizer" in globals(): tokenizer = LlamaTokenizer.from_pretrained( - model, + base_model_config, trust_remote_code=True if cfg.trust_remote_code is True else False, ) else: tokenizer = getattr(transformers, tokenizer_type).from_pretrained( - model, + base_model_config, trust_remote_code=True if cfg.trust_remote_code is True else False, ) except: