From 7f091064375837bc69d1be62b8131aafab9c1601 Mon Sep 17 00:00:00 2001 From: Wing Lian Date: Fri, 9 Jun 2023 20:42:33 -0400 Subject: [PATCH] fix for max sequence len across different model types --- src/axolotl/utils/models.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py index 433c96dee..8ceaa0d53 100644 --- a/src/axolotl/utils/models.py +++ b/src/axolotl/utils/models.py @@ -255,8 +255,15 @@ def load_model( ) # Shouldn't be a problem most of the time. will obviously error if the model doesn't support this # when training starts - if config.max_seq_len and cfg.sequence_len > config.max_seq_len: + if hasattr(config, "max_seq_len") and cfg.sequence_len > config.max_seq_len: config.max_seq_len = cfg.sequence_len + logging.warning(f"increasing context length to {cfg.sequence_len}") + elif ( + hasattr(config, "max_sequence_length") + and cfg.sequence_len > config.max_sequence_length + ): + config.max_sequence_length = cfg.sequence_len + logging.warning(f"increasing context length to {cfg.sequence_len}") model = AutoModelForCausalLM.from_pretrained( base_model, config=config,