From 360788296aa6c526fab62d9cebe42dc15bd988db Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Fri, 15 Sep 2023 15:47:09 -0400 Subject: [PATCH] don't resize embeddings if it's already large enough (#577) * don't resize embeddings if it's already large enough * make sure to tie weights, even if we aren't resizing --- src/axolotl/utils/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py index ca56e79d8..ab5bbc267 100644 --- a/src/axolotl/utils/models.py +++ b/src/axolotl/utils/models.py @@ -302,7 +302,10 @@ def load_model( if cfg.resize_token_embeddings_to_32x else len(tokenizer) ) - model.resize_token_embeddings(embeddings_len) + if model.get_input_embeddings().num_embeddings < embeddings_len: + model.resize_token_embeddings(embeddings_len) + else: + model.tie_weights() if ( hasattr(model.config, "max_position_embeddings")