diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py
index 433c96dee..8ceaa0d53 100644
--- a/src/axolotl/utils/models.py
+++ b/src/axolotl/utils/models.py
@@ -255,8 +255,15 @@ def load_model(
             )
             # Shouldn't be a problem most of the time. will obviously error if the model doesn't support this
             # when training starts
-            if config.max_seq_len and cfg.sequence_len > config.max_seq_len:
+            if hasattr(config, "max_seq_len") and cfg.sequence_len > config.max_seq_len:
                 config.max_seq_len = cfg.sequence_len
+                logging.warning(f"increasing context length to {cfg.sequence_len}")
+            elif (
+                hasattr(config, "max_sequence_length")
+                and cfg.sequence_len > config.max_sequence_length
+            ):
+                config.max_sequence_length = cfg.sequence_len
+                logging.warning(f"increasing context length to {cfg.sequence_len}")
             model = AutoModelForCausalLM.from_pretrained(
                 base_model,
                 config=config,