From e3c494ca7b8f5332233337298e0c382e39b36a9d Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Tue, 30 May 2023 23:55:45 -0400 Subject: [PATCH] remove unused import and update readme --- README.md | 3 +++ src/axolotl/utils/models.py | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e1391e39b..f8f226963 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,9 @@ base_model_ignore_patterns: # if the base_model repo on hf hub doesn't include configuration .json files, # you can set that here, or leave this empty to default to base_model base_model_config: ./llama-7b-hf +# Optional tokenizer configuration override in case you want to use a different tokenizer +# than the one defined in the base model +tokenizer_config: # If you want to specify the type of model to load, AutoModelForCausalLM is a good choice too model_type: AutoModelForCausalLM # Corresponding tokenizer for the model AutoTokenizer is a good choice diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py index dc303bca6..cf351a78d 100644 --- a/src/axolotl/utils/models.py +++ b/src/axolotl/utils/models.py @@ -30,7 +30,6 @@ from axolotl.prompt_tokenizers import LLAMA_DEFAULT_PAD_TOKEN if TYPE_CHECKING: from peft import PeftConfig # noqa: F401 - from transformers import PreTrainedTokenizer # noqa: F401 from axolotl.utils.dict import DictDefault # noqa: F401