From 9a6e9d8d151a5e1f007c6c84a2f07c077fc55ece Mon Sep 17 00:00:00 2001
From: Dan Saunders <danjsaund@gmail.com>
Date: Tue, 19 Aug 2025 10:25:37 -0400
Subject: [PATCH] no sequence length support

---
 examples/llama-3/lora-1b.yml        | 2 +-
 src/axolotl/loaders/model.py        | 5 ++++-
 src/axolotl/prompt_tokenizers.py    | 2 +-
 src/axolotl/utils/schemas/config.py | 2 +-
 src/axolotl/utils/trainer.py        | 6 +++++-
 5 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/examples/llama-3/lora-1b.yml b/examples/llama-3/lora-1b.yml
index 2ae2f0056..1cc6eb172 100644
--- a/examples/llama-3/lora-1b.yml
+++ b/examples/llama-3/lora-1b.yml
@@ -12,7 +12,7 @@ output_dir: ./outputs/lora-out
 adapter: lora
 lora_model_dir:
 
-sequence_len: 2048
+sequence_len:
 sample_packing: true
 eval_sample_packing: true
 
diff --git a/src/axolotl/loaders/model.py b/src/axolotl/loaders/model.py
index 53ae428a2..08b5aa2ea 100644
--- a/src/axolotl/loaders/model.py
+++ b/src/axolotl/loaders/model.py
@@ -268,7 +268,10 @@ class ModelLoader:
             hasattr(self.model, "config")
             and hasattr(self.model.config, "max_position_embeddings")
             and self.model.config.max_position_embeddings
-            and self.cfg.sequence_len > self.model.config.max_position_embeddings
+            and (
+                self.cfg.sequence_len is not None
+                and self.cfg.sequence_len > self.model.config.max_position_embeddings
+            )
         ):
             LOG.warning(
                 "increasing model.config.max_position_embeddings from "
diff --git a/src/axolotl/prompt_tokenizers.py b/src/axolotl/prompt_tokenizers.py
index 9ca645de3..4113e19d9 100644
--- a/src/axolotl/prompt_tokenizers.py
+++ b/src/axolotl/prompt_tokenizers.py
@@ -91,7 +91,7 @@ class PromptTokenizingStrategy(abc.ABC):
 
         if (
             result["input_ids"][-1] != self.tokenizer.eos_token_id
-            and len(result["input_ids"]) < self.max_length
+            and (self.max_length is None or len(result["input_ids"]) < self.max_length)
             and add_eos_token
         ):
             result["input_ids"].append(self.tokenizer.eos_token_id)
diff --git a/src/axolotl/utils/schemas/config.py b/src/axolotl/utils/schemas/config.py
index a607b3dca..1898dc593 100644
--- a/src/axolotl/utils/schemas/config.py
+++ b/src/axolotl/utils/schemas/config.py
@@ -408,7 +408,7 @@ class AxolotlInputConfig(
 
     unfrozen_parameters: list[str] | None = None
 
-    sequence_len: int = Field(
+    sequence_len: int | None = Field(
         default=512,
         json_schema_extra={
             "description": "The maximum length of an input to train with, this should typically be less than 2048 as most models have a token/context limit of 2048"
diff --git a/src/axolotl/utils/trainer.py b/src/axolotl/utils/trainer.py
index e424cb55a..b223ee0c5 100644
--- a/src/axolotl/utils/trainer.py
+++ b/src/axolotl/utils/trainer.py
@@ -229,7 +229,10 @@ def drop_long_seq(sample, sequence_len=2048, min_sequence_len=2):
     results = []
     for seq in input_ids:
         length = len(seq)
-        results.append(min_sequence_len <= length <= sequence_len)
+        if sequence_len is not None:
+            results.append(min_sequence_len <= length <= sequence_len)
+        else:
+            results.append(min_sequence_len <= length)
     return results
 
 
@@ -406,6 +409,7 @@ def calculate_total_num_steps(cfg, train_dataset, update=True):
             cfg.total_num_tokens = total_num_tokens
 
     skip_estimates = cfg.model_config_type == "mamba"
+    skip_estimates = True
 
     if (
         not skip_estimates