Fix(tokenizer): Make sure to add pad for CodeLlamaTokenizer (#489)

2023-08-28 09:39:10 +09:00
parent f144e98a32
commit 4c37bd0b54
1 changed files with 1 additions and 0 deletions
--- a/src/axolotl/utils/models.py
+++ b/src/axolotl/utils/models.py
@@ -59,6 +59,7 @@ def load_tokenizer(cfg):
        in [
            "LlamaTokenizer",
            "LlamaTokenizerFast",
+            "CodeLlamaTokenizer",
        ]
        and hasattr(tokenizer, "pad_token")
        and not tokenizer.pad_token