tag v0.12.2

examples/llama-3/lora-1b.yml

@@ -12,7 +12,7 @@ output_dir: ./outputs/lora-out
 adapter: lora
 lora_model_dir:
 
-sequence_len:
+sequence_len: 2048
 sample_packing: true
 eval_sample_packing: true
 

src/axolotl/__init__.py

@@ -4,4 +4,4 @@ import pkgutil
 
 __path__ = pkgutil.extend_path(__path__, __name__)  # Make this a namespace package
 
-__version__ = "0.13.0.dev"
+__version__ = "0.12.2"

src/axolotl/loaders/model.py

@@ -268,10 +268,7 @@ class ModelLoader:
             hasattr(self.model, "config")
             and hasattr(self.model.config, "max_position_embeddings")
             and self.model.config.max_position_embeddings
-            and (
-                self.cfg.sequence_len is not None
-                and self.cfg.sequence_len > self.model.config.max_position_embeddings
-            )
+            and self.cfg.sequence_len > self.model.config.max_position_embeddings
         ):
             LOG.warning(
                 "increasing model.config.max_position_embeddings from "

src/axolotl/prompt_tokenizers.py

@@ -91,7 +91,7 @@ class PromptTokenizingStrategy(abc.ABC):
 
         if (
             result["input_ids"][-1] != self.tokenizer.eos_token_id
-            and (self.max_length is None or len(result["input_ids"]) < self.max_length)
+            and len(result["input_ids"]) < self.max_length
             and add_eos_token
         ):
             result["input_ids"].append(self.tokenizer.eos_token_id)

src/axolotl/utils/schemas/config.py

@@ -408,7 +408,7 @@ class AxolotlInputConfig(
 
     unfrozen_parameters: list[str] | None = None
 
-    sequence_len: int | None = Field(
+    sequence_len: int = Field(
         default=512,
         json_schema_extra={
             "description": "The maximum length of an input to train with, this should typically be less than 2048 as most models have a token/context limit of 2048"

src/axolotl/utils/trainer.py

@@ -229,10 +229,7 @@ def drop_long_seq(sample, sequence_len=2048, min_sequence_len=2):
     results = []
     for seq in input_ids:
         length = len(seq)
-        if sequence_len is not None:
-            results.append(min_sequence_len <= length <= sequence_len)
-        else:
-            results.append(min_sequence_len <= length)
+        results.append(min_sequence_len <= length <= sequence_len)
     return results
 
 
@@ -408,7 +405,7 @@ def calculate_total_num_steps(cfg, train_dataset, update=True):
         if update:
             cfg.total_num_tokens = total_num_tokens
 
-    skip_estimates = cfg.sequence_len is None or cfg.model_config_type == "mamba"
+    skip_estimates = cfg.model_config_type == "mamba"
 
     if (
         not skip_estimates