feat: add draft wizard cli

_quarto.yml

@@ -139,8 +139,7 @@ quartodoc:
         - utils.optimizers.adopt
         - utils.data.pretraining
         - utils.data.sft
-        - utils.gradient_checkpointing.offload_cpu
-        - utils.gradient_checkpointing.offload_disk
+        - utils.gradient_checkpointing.unsloth
     - title: Schemas
       desc: Pydantic data models for Axolotl config
       contents:

docs/config.qmd

@@ -332,8 +332,6 @@ dataset_shard_idx:
 # The maximum length of an input to train with, this should typically be less than 2048
 # as most models have a token/context limit of 2048
 sequence_len: 2048
-# How to handle sequences that overflow the sequence_len: 'drop' (default, removes sample) or 'truncate' (cuts off excess tokens).
-sequence_len_overflow_handling: drop
 # Pad inputs so each step uses constant sized buffers
 # This will reduce memory fragmentation and may prevent OOMs, by re-using memory more efficiently
 pad_to_sequence_len:
@@ -541,7 +539,7 @@ train_on_inputs: false
 # Note that training loss may have an oscillating pattern with this enabled.
 group_by_length: false
 
-# Whether to use gradient checkpointing. Available options are: true, false, "offload", "offload_disk".
+# Whether to use gradient checkpointing. Available options are: true, false, "offload".
 # https://huggingface.co/docs/transformers/v4.18.0/en/performance#gradient-checkpointing
 gradient_checkpointing: false
 # additional kwargs to pass to the trainer for gradient checkpointing

src/axolotl/cli/main.py

@@ -342,6 +342,13 @@ def delinearize_llama4(model: str, output: str) -> None:
     do_delinearize_llama4(model, output)
 
 
+@cli.command()
+def wizard():
+    from axolotl.cli.wizard import do_wizard
+
+    do_wizard()
+
+
 cli.add_command(lm_eval)
 
 

src/axolotl/cli/wizard.py

@@ -0,0 +1,429 @@
+"""Wizard for creating yaml configs."""
+
+import click
+import torch
+import yaml
+from packaging import version
+from transformers.training_args import OptimizerNames
+
+from axolotl.cli.art import print_axolotl_text_art
+from axolotl.utils.dict import DictDefault
+from axolotl.utils.models import load_model_config
+from axolotl.utils.schemas.enums import CustomSupportedOptimizers
+
+
+def do_wizard():
+    print_axolotl_text_art()
+
+    # Ask where to save the config
+    cfg = DictDefault({})
+    config_path = click.prompt(
+        "Where do you want to save the config?", type=str, default="config.yaml"
+    )
+
+    # Ask base model
+    base_model = click.prompt("What base model do you want to use?", type=str)
+    cfg["base_model"] = base_model.strip()
+
+    # Ask whether want to enable Vision model
+    # TODO: check if model has vision layers instead of asking user
+    train_vision_model = click.confirm(
+        "If this model has vision layers, do you want to train them?", default=False
+    )
+
+    if train_vision_model:
+        cfg["processor_type"] = "AutoProcessor"
+        cfg["skip_prepare_dataset"] = True
+        cfg["remove_unused_columns"] = False
+        cfg["sample_packing"] = False
+
+    # Ask whether they want to set any advanced model features (custom tokenizer, custom config, etc)
+    advanced_model_features = click.confirm(
+        "Do you want to set any advanced model features? (custom tokenizer, custom config, remote code etc)",
+        default=False,
+    )
+
+    if advanced_model_features:
+        # Ask whether they want to use a custom config
+        base_model_config = click.prompt(
+            "What model config do you want to use? (leave blank for default)",
+            type=str,
+            default="",
+        )
+
+        if base_model_config:
+            cfg["base_model_config"] = base_model_config
+
+        # Ask whether they want to use a specific revision of the model
+        revision_of_model = click.prompt(
+            "What revision of the model do you want to use? (leave blank for default)",
+            type=str,
+            default="",
+        )
+
+        if revision_of_model:
+            cfg["revision_of_model"] = revision_of_model
+
+        # Ask whether they want to use a custom tokenizer
+        tokenizer_config = click.prompt(
+            "What tokenizer do you want to use? (leave blank for default)",
+            type=str,
+            default="",
+        )
+
+        if tokenizer_config:
+            cfg["tokenizer_config"] = tokenizer_config
+
+        # Ask whether they want to use remote code
+        trust_remote_code = click.confirm(
+            "Do you want to use remote code?", default=False
+        )
+
+        if trust_remote_code:
+            cfg["trust_remote_code"] = trust_remote_code
+
+        # Whether to resize token embeddings
+        resize_token_embeddings_to_32x = click.confirm(
+            "Do you want to resize token embeddings to 32x?", default=False
+        )
+
+        if resize_token_embeddings_to_32x:
+            cfg["resize_token_embeddings_to_32x"] = resize_token_embeddings_to_32x
+
+        # Whether to shrink embeddings to len(tokenizer)
+        shrink_embeddings = click.confirm(
+            "Do you want to shrink embeddings to len(tokenizer)?", default=False
+        )
+
+        if shrink_embeddings:
+            cfg["shrink_embeddings"] = shrink_embeddings
+
+        # Whether to skip upcast embeddings
+        embeddings_skip_upcast = click.confirm(
+            "Do you want to skip upcast embeddings?", default=False
+        )
+
+        if embeddings_skip_upcast:
+            cfg["embeddings_skip_upcast"] = embeddings_skip_upcast
+
+        # Whether to random init weights
+        random_init_weights = click.confirm(
+            "Do you want to random init weights?", default=False
+        )
+
+        if random_init_weights:
+            cfg["random_init_weights"] = random_init_weights
+
+    # Get model type
+    config = load_model_config(cfg)
+    model_type = config.model_type
+
+    # Ask sequence length
+    sequence_length = click.prompt("What sequence length do you want to use?", type=int)
+    cfg["sequence_length"] = sequence_length
+
+    # Whether to turn on sample packing
+    if cfg["sample_packing"] is None:
+        cfg["sample_packing"] = click.confirm(
+            "Do you want to turn on sample packing? This will speed up training by packing multiple samples into a single batch.",
+            default=True,
+        )
+
+        if cfg["sample_packing"]:
+            cfg["pad_to_sequence_len"] = True
+
+        # Whether to turn off eval sample packing
+        no_eval_sample_packing = click.confirm(
+            "Do you want to turn off eval sample packing? This will slow down evaluation but is recommended if you are using a small validation set.",
+            default=False,
+        )
+
+        if no_eval_sample_packing:
+            cfg["eval_sample_packing"] = False
+
+    # Hardware check
+    try:
+        is_ampere_or_newer = torch.cuda.get_device_capability()[0] >= 8
+    except RuntimeError:
+        is_ampere_or_newer = False
+    except AssertionError:  # this is raised if no cuda is available
+        is_ampere_or_newer = False
+
+    # Get num gpus
+    try:
+        num_gpus = torch.cuda.device_count()
+    except RuntimeError:
+        num_gpus = 0
+
+    # Get torch version
+    torch_version = str(torch.__version__).split("+", maxsplit=1)[0]
+
+    is_torch_2_6_or_newer = version.parse(torch_version) >= version.parse("2.6.0")
+
+    # Whether to turn on attention
+    opt = ["xformers", "sdp"]
+
+    if is_ampere_or_newer:
+        opt.append("flash")
+
+    if is_torch_2_6_or_newer:
+        opt.append("flex")
+
+    if cfg["sample_packing"]:
+        if "flash" in opt:
+            default_opt = "flash"
+        elif "flex" in opt:
+            default_opt = "flex"
+        else:
+            default_opt = opt[0]
+
+        attention = click.prompt(
+            "Which attention backend do you want to use? Sample packing requires an attention backend to be set.",
+            type=click.Choice(opt),
+            default=default_opt,
+        )
+    else:
+        # non-sample packing supports no attention and S2
+        opt.extend(["none", "s2"])
+
+        attention = click.prompt(
+            "Which attention backend do you want to use?",
+            type=click.Choice(opt),
+            default="none",
+        )
+
+        if attention == "none":
+            attention = None
+
+    # TODO: if xformers, check if FA is installed
+    # TODO: flex doc mentioned requiring seq len to be divisible by 128. Unclear if limitation still exists
+
+    # TODO: requires #2489
+    cfg["attention"] = attention
+
+    # Whether to turn on gradient checkpointing
+    # TODO: need to wait for offload_disk PR to be merged
+    gradient_checkpointing = click.prompt(
+        "Which gradient checkpointing strategy do you want to use?",
+        type=click.Choice(["none", "true", "offload", "offload_disk"]),
+        default="true",
+    )
+
+    if gradient_checkpointing == "none":
+        gradient_checkpointing = False
+    elif gradient_checkpointing == "true":
+        gradient_checkpointing = True
+
+    # Ask whether to set use_reentrant
+    # TODO: get correct defaults based on SFT/RL mode and single/multigpu
+    # use_reentrant = click.confirm(
+    #     "Do you want to set use_reentrant?",
+    #     default=True,
+    # )
+
+    # if use_reentrant:
+    #     cfg["use_reentrant"] = use_reentrant
+
+    # Optimizer
+    cfg["optimizer"] = click.prompt(
+        "Which optimizer do you want to use?",
+        type=click.Choice((OptimizerNames | CustomSupportedOptimizers)),
+        default=OptimizerNames.ADAMW_TORCH_FUSED,
+    )
+
+    cfg["lr_scheduler"] = click.prompt(
+        "Which learning rate scheduler do you want to use?",
+        type=click.Choice(
+            [
+                "cosine",
+                "one_cycle",
+                "rex",
+                "log_sweep",
+                "linear",
+                "cosine_with_restarts",
+                "polynomial",
+                "constant",
+                "constant_with_warmup",
+                "inverse_sqrt",
+                "reduce_lr_on_plateau",
+                "cosine_with_min_lr",
+                "warmup_stable_decay",
+            ]
+        ),
+        default="cosine",
+    )
+
+    # Plugins
+
+    cfg["plugins"] = []
+
+    # Whether to turn on cut cross entropy
+    if is_ampere_or_newer:
+        # Note: This may error if users don't have CCE installed
+        from axolotl.integrations.cut_cross_entropy.monkeypatch.patch import (
+            CUT_CROSS_ENTROPY_MODEL_MAPPING,
+        )
+
+        if model_type in CUT_CROSS_ENTROPY_MODEL_MAPPING:
+            cut_cross_entropy = click.confirm(
+                "Do you want to turn on cut cross entropy? This will save VRAM if the model has a large vocab size.",
+                default=True,
+            )
+
+            if cut_cross_entropy:
+                cfg["plugins"].append(
+                    "axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin"
+                )
+
+                cfg["cut_cross_entropy"] = True
+
+    use_liger_kernel = click.confirm(
+        "Do you want to use the liger kernel? This will speed up training and save VRAM.",
+        default=True,
+    )
+
+    if use_liger_kernel:
+        cfg["plugins"].append("axolotl.integrations.liger.LigerPlugin")
+
+        cfg["liger_rope"] = click.confirm(
+            "Do you want to enable liger rope?",
+            default=True,
+        )
+
+        cfg["liger_rms_norm"] = click.confirm(
+            "Do you want to enable liger rms norm?",
+            default=True,
+        )
+
+        cfg["liger_glu_activation"] = click.confirm(
+            "Do you want to enable liger glu activation?",
+            default=True,
+        )
+
+        cfg["liger_layer_norm"] = click.confirm(
+            "Do you want to enable liger layer norm?",
+            default=True,
+        )
+
+        if cfg["cut_cross_entropy"] is not True:
+            cfg["liger_fused_linear_cross_entropy"] = click.confirm(
+                "Do you want to enable liger fused linear cross entropy?",
+                default=True,
+            )
+
+    # TODO: lora kernels (but they auto enable via validator already)
+
+    # TODO: is there incompat between torch compile and liger?
+    cfg["torch_compile"] = click.confirm(
+        "Do you want to enable torch compile?",
+        default=True,
+    )
+
+    # Multi-gpu
+    if num_gpus > 1:
+        # Ask whether to use DDP/Deepspeed/FSDP
+        multi_gpu_mode = click.prompt(
+            "Which multi-gpu mode do you want to use?",
+            type=click.Choice(["ddp", "deepspeed", "fsdp"]),
+            default="ddp",
+        )
+
+        if multi_gpu_mode == "deepspeed":
+            # Ask which deepspeed config to use
+            cfg["deepspeed"] = click.prompt(
+                "Which deepspeed config do you want to use? The higher the number, the more VRAM you will save, but the slower it will run.",
+                type=click.Choice(
+                    [
+                        "zero1.json",
+                        "zero1_torch_compile.json",
+                        "zero2.json",
+                        "zero3.json",
+                        "zero3_bf16.json",
+                        "zero3_bf16_cpuoffload_all.json",
+                        "zero3_bf16_cpuoffload_params.json",
+                    ]
+                ),
+                default="zero1.json",
+            )
+        elif multi_gpu_mode == "fsdp":
+            fsdp_version = click.prompt(
+                "Which fsdp version do you want to use?",
+                type=click.Choice([1, 2]),
+                default=2,
+            )
+
+            # TODO: Handle FSDP config
+
+            if fsdp_version == 1:
+                cfg["fsdp"] = ["full_shard", "auto_wrap"]
+
+                # Ask which state dict type to use
+                fsdp_state_dict_type = click.prompt(
+                    "Which fsdp state dict type do you want to use?",
+                    type=click.Choice(["FULL_STATE_DICT", "SHARDED_STATE_DICT"]),
+                    default="FULL_STATE_DICT",
+                )
+
+                fsdp_offload_params = click.confirm(
+                    "Do you want to offload parameters?",
+                    default=True,
+                )
+
+                # TODO: can we load the model class and auto pull a default for this?
+                fsdp_transformer_layer_cls_to_wrap = click.prompt(
+                    "Which transformer layer class to wrap? It is usually the Decoder layer class.",
+                    type=str,
+                )
+
+                # TODO: add other options
+
+                cfg["fsdp_config"] = {
+                    "state_dict_type": fsdp_state_dict_type,
+                    "offload_params": fsdp_offload_params,
+                    "transformer_layer_cls_to_wrap": fsdp_transformer_layer_cls_to_wrap,
+                }
+
+            elif fsdp_version == 2:
+                raise NotImplementedError()
+
+    # Training mode (sft or rl)
+    training_mode = click.prompt(
+        "Which training mode do you want to use?",
+        type=click.Choice(["sft", "rl"]),
+        default="sft",
+    )
+
+    if training_mode == "rl":
+        cfg["rl"] = click.prompt(
+            "Which rl mode do you want to use?",
+            type=click.Choice(["dpo", "ipo", "orpo", "kto", "grpo", "simpo"]),
+        )
+
+        # TODO: handle RL options
+
+    # Whether to use adapter
+
+    # Get batch/grad accu
+
+    # Get learning rate
+
+    # Get weight decay
+
+    # Get max grad norm
+
+    # Get num train epochs
+
+    # Get warmup ratio
+
+    # Get save ratio
+
+    # Get eval ratio
+
+    # Get dataset config
+
+    # Load metric tracker
+
+    # Save config to yaml
+    # TODO: improve output yaml formatting. Need to add comments to help separate sections
+    with open(config_path, "w", encoding="utf-8") as f:
+        yaml.dump(cfg.to_dict(), f, sort_keys=False)

src/axolotl/utils/data/pretraining.py

@@ -11,7 +11,6 @@ from torch.utils.data import RandomSampler
 from transformers import PreTrainedTokenizerBase
 
 from axolotl.utils.collators import PretrainingBatchSamplerDataCollatorForSeq2Seq
-from axolotl.utils.data.utils import DEFAULT_SEQUENCE_LEN_OVERFLOW_HANDLING
 from axolotl.utils.samplers import MultipackBatchSampler, get_dataset_lengths
 from axolotl.utils.trainer import process_pretraining_datasets_for_packing
 
@@ -251,22 +250,6 @@ def encode_packed_pretraining(
     # pylint: disable=duplicate-code
     # tokenize all the examples
     # rows get split with stride (overlap)
-    """
-    Encodes and packs input examples into fixed-length batches for pretraining with optional multipack attention.
-    
-    Wraps and processes input examples into a dataset, applies sequence packing with configurable overflow handling, and batches the data using a multipack sampler. Each batch is collated and features are aggregated into lists keyed by feature name.
-    
-    Args:
-        collate_fn: Function to collate individual feature dictionaries into batch tensors.
-        ds_wrapper: Callable that wraps a Hugging Face Dataset for further processing.
-        examples: Dictionary of input examples to encode and pack.
-        max_seq_length: Maximum sequence length for each packed sequence.
-        batch_size: Number of sequences to pack per batch.
-        multipack_attn: If True, enables multipack attention and drops attention masks.
-    
-    Returns:
-        Dictionary where each key is a feature name and each value is a list of packed feature tensors.
-    """
     train_dataset = ds_wrapper(Dataset.from_dict(examples))[0]
 
     train_dataset = process_pretraining_datasets_for_packing(
@@ -276,10 +259,6 @@ def encode_packed_pretraining(
         # FIXME using attention mask unpad/pad with trainer and packed pretraining is broken atm
         # workaround by using the position id logic for now in trainer
         drop_attention_mask=multipack_attn,
-        # pass through handling mode from config via ds_wrapper function
-        handling=getattr(ds_wrapper, "cfg", {}).get(
-            "sequence_len_overflow_handling", DEFAULT_SEQUENCE_LEN_OVERFLOW_HANDLING
-        ),
     )
 
     sampler = MultipackBatchSampler(

src/axolotl/utils/data/rl.py

@@ -79,33 +79,8 @@ def map_dataset(cfg, data_set, ds_transform_fn, tokenizer, **map_kwargs):
 
 
 def drop_long_rl_seq(
-    sample,
-    rl,
-    tokenizer,
-    sequence_len,
-    handling="drop",  # Use the default handling mode
+    sample, rl, tokenizer, sequence_len  # pylint: disable=invalid-name
 ):
-    """
-    Handles samples exceeding a maximum sequence length for various RL dataset types by either truncating or dropping them.
-    
-    Depending on the RL type and the `handling` mode, this function either truncates response fields to fit within the specified sequence length or determines whether the sample should be dropped. For DPO, IPO, ORPO, and SIMPO types, both "chosen" and "rejected" responses are considered; for KTO, the "completion" is considered. For GRPO, samples are always retained. If truncation is not possible (e.g., the prompt alone exceeds the limit), the sample is returned unchanged for mapping, or dropped during filtering.
-    
-    Args:
-        sample: A dictionary representing a single dataset sample.
-        rl: The RLType indicating the dataset type.
-        tokenizer: The tokenizer used to compute token lengths and perform truncation.
-        sequence_len: The maximum allowed sequence length.
-        handling: Specifies how to handle overlong sequences ("drop" or "truncate").
-    
-    Returns:
-        For "truncate": The modified sample with responses truncated as needed, or the original sample if truncation is not possible.
-        For "drop": True if the sample fits within the sequence length, otherwise False.
-    
-    Raises:
-        ValueError: If required keys are missing for the specified RL type, or if the RL type is unknown.
-    """
-    result = None
-
     if rl in (RLType.DPO, RLType.IPO, RLType.ORPO, RLType.SIMPO):
         if not (
             sample.get("prompt") and sample.get("chosen") and sample.get("rejected")
@@ -122,65 +97,11 @@ def drop_long_rl_seq(
         len_chosen = len(tokenizer(chosen, add_special_tokens=False)["input_ids"])
         len_rejected = len(tokenizer(rejected, add_special_tokens=False)["input_ids"])
 
-        # Truncate first, then drop if still invalid (although truncate should handle it)
-        if handling == "truncate":
-            # If both sequences fit, return sample unchanged
-            if (len_prompt + len_chosen) <= sequence_len and (
-                len_prompt + len_rejected
-            ) <= sequence_len:
-                result = sample
-            else:
-                # Calculate maximum response length that can fit with the prompt
-                max_response_len = sequence_len - len_prompt
+        return (len_prompt + len_chosen) <= sequence_len and (
+            len_prompt + len_rejected
+        ) <= sequence_len
 
-                if max_response_len <= 0:
-                    # Prompt is already too long, behavior depends on handling
-                    # If truncate is chosen, we technically can't truncate, but drop seems harsh.
-                    # Returning the sample might be unexpected. Let's stick to the filter logic
-                    # which would drop this in the `filter` step later if needed.
-                    # For now, return sample to map, or False to filter.
-                    # Let's simplify: truncate *should* result in a valid sample if possible.
-                    # If prompt >= seq_len, truncate won't work. Filter will catch this later.
-                    # So, if max_response_len <= 0, we pass it through for map, drop for filter.
-                    # However, the filter/map logic is applied *after* this function.
-                    # This function needs to return the *modified* sample for map, or bool for filter.
-
-                    # Re-think: If handling==truncate, return the modified sample if possible.
-                    # If prompt >= seq_len, modification is impossible. What should map return?
-                    # Maybe return the original sample? But map expects *modified* sample.
-                    # Let's stick to the original logic: if prompt is too long, return False for filter
-                    # and original sample for map.
-
-                    result = (
-                        sample  # For map, let downstream handle it if still invalid?
-                    )
-                    # Or maybe return None/empty dict? Let's return sample for now.
-                    # If handling was drop, filter would remove this.
-
-                else:
-                    # Truncate the chosen and rejected responses if needed
-                    if len_chosen > max_response_len:
-                        chosen_tokens = tokenizer(chosen, add_special_tokens=False)[
-                            "input_ids"
-                        ][:max_response_len]
-                        sample["chosen"] = tokenizer.decode(
-                            chosen_tokens, skip_special_tokens=True
-                        )
-
-                    if len_rejected > max_response_len:
-                        rejected_tokens = tokenizer(rejected, add_special_tokens=False)[
-                            "input_ids"
-                        ][:max_response_len]
-                        sample["rejected"] = tokenizer.decode(
-                            rejected_tokens, skip_special_tokens=True
-                        )
-                    result = sample
-        else:  # handling == "drop"
-            result = (len_prompt + len_chosen) <= sequence_len and (
-                len_prompt + len_rejected
-            ) <= sequence_len
-
-    elif rl == RLType.KTO:
+    if rl is RLType.KTO:
         if not (sample.get("prompt") and sample.get("completion")):
             raise ValueError("Prompt and completion keys are required for KTO datasets")
 
@@ -192,54 +113,15 @@ def drop_long_rl_seq(
             tokenizer(completion, add_special_tokens=False)["input_ids"]
         )
 
-        # Truncate first
-        if handling == "truncate":
-            # If sequence fits, return sample unchanged
-            if (len_prompt + len_completion) <= sequence_len:
-                result = sample
-            else:
-                # Calculate maximum completion length
-                max_completion_len = sequence_len - len_prompt
+        return (len_prompt + len_completion) <= sequence_len
 
-                if max_completion_len <= 0:
-                    # Prompt too long, return sample for map
-                    result = sample
-                else:
-                    # Truncate the completion if needed
-                    if len_completion > max_completion_len:
-                        completion_tokens = tokenizer(
-                            completion, add_special_tokens=False
-                        )["input_ids"][:max_completion_len]
-                        sample["completion"] = tokenizer.decode(
-                            completion_tokens, skip_special_tokens=True
-                        )
-                    result = sample
-        else:  # handling == "drop"
-            result = (len_prompt + len_completion) <= sequence_len
+    if rl is RLType.GRPO:
+        return True
 
-    elif rl == RLType.GRPO:
-        # GRPO doesn't involve sequence length checks in the same way?
-        # The original code returned True for drop. What should it return for truncate?
-        # Let's assume for now it always passes.
-        result = sample if handling == "truncate" else True
-    else:
-        raise ValueError("Unknown RL type")
-
-    return result
+    raise ValueError("Unknown RL type")
 
 
 def load_prepare_preference_datasets(cfg):
-    """
-    Loads, preprocesses, and prepares preference datasets for RL training and evaluation.
-    
-    This function orchestrates the loading, transformation, sequence length handling, optional deduplication, and caching of datasets for Direct Preference Optimization (DPO) and related RL types. It supports configurable handling of overlong sequences (dropping or truncating), applies dataset-specific transformations, and manages train/validation/test splits as needed.
-    
-    Args:
-        cfg: Configuration object specifying dataset sources, RL type, tokenizer, sequence length, and processing options.
-    
-    Returns:
-        A tuple containing the prepared training and evaluation datasets.
-    """
     def load_split(dataset_cfgs, _cfg):
         split_datasets: List[Any] = []
         use_auth_token = _cfg.hf_use_auth_token
@@ -283,43 +165,25 @@ def load_prepare_preference_datasets(cfg):
                 split_datasets[i] = data_set
 
             if not cfg.skip_prepare_dataset:
-                # Determine handling mode
-                handling = cfg.get("sequence_len_overflow_handling", "drop")
-
                 drop_long = partial(
                     drop_long_rl_seq,
                     rl=_cfg.rl,
                     tokenizer=tokenizer,
                     sequence_len=cfg.sequence_len,
-                    handling=handling,  # Pass the handling mode
                 )
 
                 prior_len = len(split_datasets[i])
-
-                # Use map for truncate mode and filter for drop mode
-                if handling == "truncate":
-                    split_datasets[i] = split_datasets[i].map(
-                        drop_long,  # Function now returns modified sample or original
-                        num_proc=cfg.dataset_processes,
-                        load_from_cache_file=not cfg.is_preprocess,
-                        desc="Truncating Long Sequences",
+                split_datasets[i] = split_datasets[i].filter(
+                    drop_long,
+                    num_proc=cfg.dataset_processes,
+                    load_from_cache_file=not cfg.is_preprocess,
+                    desc="Dropping Long Sequences",
+                )
+                dropped = prior_len - len(split_datasets[i])
+                if dropped:
+                    LOG.warning(
+                        f"Dropped {dropped} long samples from dataset index {i}"
                     )
-                    # Note: Length might not change if truncation always occurs
-                    LOG.info(
-                        f"Processed dataset index {i} with truncation handling for sequence length {cfg.sequence_len}"
-                    )
-                else:  # handling == "drop"
-                    split_datasets[i] = split_datasets[i].filter(
-                        drop_long,  # Function now returns boolean
-                        num_proc=cfg.dataset_processes,
-                        load_from_cache_file=not cfg.is_preprocess,
-                        desc="Dropping Long Sequences",
-                    )
-                    dropped = prior_len - len(split_datasets[i])
-                    if dropped:
-                        LOG.warning(
-                            f"Dropped {dropped} long samples from dataset index {i}"
-                        )
 
         combined_datasets = concatenate_datasets(split_datasets)
         combined_datasets = combined_datasets.shuffle(seed=cfg.seed or 42)

src/axolotl/utils/data/utils.py

@@ -13,12 +13,10 @@ from datasets import Dataset, IterableDataset
 
 from axolotl.utils.dict import DictDefault
 from axolotl.utils.samplers.utils import get_dataset_lengths
-from axolotl.utils.trainer import truncate_or_drop_long_seq
+from axolotl.utils.trainer import drop_long_seq
 
 LOG = logging.getLogger(__name__)
 
-DEFAULT_SEQUENCE_LEN_OVERFLOW_HANDLING = "drop"
-
 
 class RetryStrategy(Enum):
     """
@@ -161,33 +159,16 @@ def deduplicate_and_log_datasets(
 
 
 def drop_long_seq_in_dataset(dataset: Dataset, cfg: DictDefault):
-    """
-    Processes a dataset to handle sequences exceeding a configured maximum length by either truncating or dropping them.
-    
-    If the dataset lacks an "input_ids" column, the function returns the dataset unchanged. The handling mode is determined by the configuration parameter "sequence_len_overflow_handling", defaulting to "drop". In "truncate" mode, sequences longer than the maximum length are truncated; in "drop" mode, such sequences are removed from the dataset. The function logs information about sequence lengths and the number of samples affected when applicable.
-    
-    Args:
-        dataset: The Huggingface Dataset to process.
-        cfg: Configuration object specifying sequence length parameters and handling mode.
-    
-    Returns:
-        The processed dataset with long sequences either truncated or dropped according to the configuration.
-    """
     if "input_ids" not in dataset.column_names:
         LOG.warning(
             "Dataset does not contain 'input_ids' column. Skip drop long seq. This is expected for RewardModeling."
         )
         return dataset
 
-    # Get the handling method from config, default to "drop" for backward compatibility
-    handling = cfg.get("sequence_len_overflow_handling", "drop")
-
-    # Use the new function with the specified handling mode
-    seq_handler = functools.partial(
-        truncate_or_drop_long_seq,
+    drop_long = functools.partial(
+        drop_long_seq,
         sequence_len=cfg.sequence_len,
         min_sequence_len=cfg.min_sample_len,
-        handling=handling,
     )
 
     try:
@@ -212,31 +193,17 @@ def drop_long_seq_in_dataset(dataset: Dataset, cfg: DictDefault):
 
     drop_long_kwargs = {}
     if filter_map_kwargs:
-        if handling == "truncate":
-            drop_long_kwargs["desc"] = "Truncating Long Sequences"
-        else:  # handling == "drop"
-            drop_long_kwargs["desc"] = "Dropping Long Sequences"
+        drop_long_kwargs["desc"] = "Dropping Long Sequences"
 
-    if handling == "truncate":
-        # Use map for truncate mode
-        dataset = dataset.map(
-            seq_handler,
-            batched=True,
-            **filter_map_kwargs,
-            **drop_long_kwargs,
-        )
-        LOG.info(f"Truncated long samples in dataset to {cfg.sequence_len} tokens")
-    else:  # handling == "drop"
-        # Use filter for drop mode
-        dataset = dataset.filter(
-            seq_handler,
-            batched=True,
-            **filter_map_kwargs,
-            **drop_long_kwargs,
-        )
-        if prior_len:
-            dropped = prior_len - len(dataset)
-            if dropped:
-                LOG.warning(f"Dropped {dropped} long samples from dataset")
+    dataset = dataset.filter(
+        drop_long,
+        batched=True,
+        **filter_map_kwargs,
+        **drop_long_kwargs,
+    )
+    if prior_len:
+        dropped = prior_len - len(dataset)
+        if dropped:
+            LOG.warning(f"Dropped {dropped} long samples from dataset")
 
     return dataset

src/axolotl/utils/gradient_checkpointing/__init__.py

@@ -5,11 +5,8 @@ from functools import partial
 
 from packaging import version
 
-from axolotl.utils.gradient_checkpointing.offload_cpu import (
-    CPU_Offloaded_Gradient_Checkpointer,
-)
-from axolotl.utils.gradient_checkpointing.offload_disk import (
-    Disco,
+from axolotl.utils.gradient_checkpointing.unsloth import (
+    Unsloth_Offloaded_Gradient_Checkpointer,
 )
 
 transformers_version = version.parse(importlib.metadata.version("transformers"))
@@ -29,31 +26,12 @@ def hf_grad_checkpoint_offload_wrapper(
     decoder_layer, *args, use_reentrant=None
 ):  # pylint: disable=unused-argument
     if uses_gc_layers(decoder_layer):
-        return CPU_Offloaded_Gradient_Checkpointer.apply(
+        return Unsloth_Offloaded_Gradient_Checkpointer.apply(
             decoder_layer,
             *args,
         )
 
-    return CPU_Offloaded_Gradient_Checkpointer.apply(
-        (
-            decoder_layer.func.__self__
-            if isinstance(decoder_layer, partial)
-            else decoder_layer.__self__
-        ),
-        *args,
-    )
-
-
-def hf_grad_checkpoint_disk_offload_wrapper(
-    decoder_layer, *args, use_reentrant=None
-):  # pylint: disable=unused-argument
-    if uses_gc_layers(decoder_layer):
-        return Disco.apply(
-            decoder_layer,
-            *args,
-        )
-
-    return Disco.apply(
+    return Unsloth_Offloaded_Gradient_Checkpointer.apply(
         (
             decoder_layer.func.__self__
             if isinstance(decoder_layer, partial)

src/axolotl/utils/gradient_checkpointing/offload_disk.py

@@ -1,531 +0,0 @@
-"""
-DISCO - DIsk-based Storage and Checkpointing with Optimized prefetching
-"""
-
-# Copyright 2025 Axolotl AI. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import atexit
-import concurrent.futures
-import logging
-import os
-import queue
-import shutil
-import tempfile
-import threading
-import time
-import uuid
-from collections import deque
-from concurrent.futures import Future
-from typing import Dict
-
-import torch
-
-torch_cuda_amp_custom_fwd = torch.amp.custom_fwd(device_type="cuda")
-torch_cuda_amp_custom_bwd = torch.amp.custom_bwd(device_type="cuda")
-
-# Setup logger
-logger = logging.getLogger(__name__)
-
-
-class DiskOffloadManager:
-    """
-    Manages offloaded tensors and handles prefetching in a separate thread.
-    Includes synchronization to prevent race conditions.
-    """
-
-    def __init__(
-        self,
-        prefetch_size: int = 3,
-        prefetch_to_gpu: bool = True,
-        save_workers: int = 4,
-    ):
-        """
-        Args:
-            prefetch_size: Maximum number of tensors to prefetch in the background.
-            prefetch_to_gpu: Whether to prefetch tensors directly to GPU memory.
-            save_workers: Maximum number of concurrent save operations.
-        """
-        self.temp_dir = tempfile.mkdtemp(prefix="disco_")
-
-        # Track tensor paths and their status
-        self.tensor_paths: deque = deque()  # Ordered history of tensor paths (LIFO)
-        self.file_locks: Dict[str, threading.Lock] = (
-            {}
-        )  # Maps file_path -> threading.Lock()
-        # Maps file_path -> status ("saving", "ready", "prefetching", "loaded", "deleted")
-        self.file_status: Dict[str, str] = {}
-
-        self.max_prefetch = prefetch_size
-        self.prefetch_to_gpu = prefetch_to_gpu
-
-        # Thread synchronization
-        self.manager_lock = threading.RLock()  # Used for thread-safe operations
-
-        # Prefetch queue and cache
-        self.prefetch_queue: queue.Queue = queue.Queue()
-        self.prefetch_cache: Dict[str, torch.Tensor] = {}  # Maps file_path -> tensor
-
-        # Save queue and thread pool
-        self.save_queue: queue.Queue = queue.Queue()
-        self.save_pool = concurrent.futures.ThreadPoolExecutor(max_workers=save_workers)
-        self.save_futures: Dict[str, Future] = {}
-        self.save_semaphore = threading.Semaphore(
-            save_workers * 2
-        )  # Limit concurrent save operations
-
-        # Start prefetch worker thread
-        self.stop_event = threading.Event()
-        # start multiple threads for prefetching
-        self.prefetch_worker_count = 2
-        self.prefetch_workers = []
-        for _ in range(self.prefetch_worker_count):
-            worker = threading.Thread(target=self._prefetch_worker, daemon=True)
-            worker.start()
-            self.prefetch_workers.append(worker)
-
-        # Start save worker thread
-        self.save_worker = threading.Thread(target=self._save_worker, daemon=True)
-        self.save_worker.start()
-        self.idx = 0
-
-        atexit.register(self.cleanup)
-
-    def _save_worker(self):
-        """Background thread that processes the save queue"""
-        while not self.stop_event.is_set():
-            try:
-                save_item = self.save_queue.get(timeout=0.5)
-                if save_item is None:
-                    continue
-
-                tensor, file_path = save_item
-
-                # Submit the save task to the thread pool
-                future = self.save_pool.submit(
-                    self._save_tensor_to_disk, tensor, file_path
-                )
-                with self.manager_lock:
-                    self.save_futures[file_path] = future
-
-                self.save_queue.task_done()
-
-            except queue.Empty:
-                time.sleep(0.01)  # Small sleep to prevent CPU spinning
-                continue
-
-    def _save_tensor_to_disk(self, tensor: torch.Tensor, file_path: str):
-        """Actually save the tensor to disk"""
-        try:
-            # Save tensor to disk
-            cpu_tensor = tensor.detach().cpu()
-            torch.save(cpu_tensor, file_path)
-            del cpu_tensor
-
-            with self.manager_lock:
-                # Mark file as ready
-                self.file_status[file_path] = "ready"
-
-            # Release semaphore
-            self.save_semaphore.release()
-
-            return True
-        except FileNotFoundError as e:
-            logger.error(f"Error saving tensor to {file_path}: {e}")
-            with self.manager_lock:
-                self.file_status[file_path] = "error"
-
-            # Release semaphore
-            self.save_semaphore.release()
-
-            return False
-
-    def _prefetch_worker(self):
-        """Background thread that loads tensors from disk ahead of time"""
-        while not self.stop_event.is_set():
-            try:
-                file_path = self.prefetch_queue.get(timeout=0.5)
-                if file_path is None:
-                    continue
-
-                # Check if file is available and not already in cache
-                with self.manager_lock:
-                    if (
-                        file_path not in self.file_status
-                        or self.file_status[file_path] == "deleted"
-                    ):
-                        self.prefetch_queue.task_done()
-                    if file_path in self.prefetch_cache:
-                        self.prefetch_queue.task_done()
-                        continue
-
-                    # If file is still being saved, wait for it
-                    if (
-                        self.file_status[file_path] == "saving"
-                        and file_path in self.save_futures
-                    ):
-                        # Re-queue this prefetch request with a little delay
-                        self.prefetch_queue.task_done()
-                        time.sleep(0.1)
-                        self.prefetch_queue.put(file_path)
-                        continue
-
-                    # Mark file as being prefetched
-                    self.file_status[file_path] = "prefetching"
-
-                # Load tensor from disk and store in cache
-                try:
-                    if os.path.exists(file_path):
-                        if self.prefetch_to_gpu:
-                            tensor = torch.load(
-                                file_path,
-                                map_location=torch.device("cuda"),
-                                weights_only=True,
-                            )
-                        else:
-                            tensor = torch.load(file_path, weights_only=True)
-
-                        with self.manager_lock:
-                            self.prefetch_cache[file_path] = tensor
-                            self.file_status[file_path] = "ready"
-                    else:
-                        with self.manager_lock:
-                            if self.file_status.get(file_path) != "deleted":
-                                logger.warning(
-                                    f"Prefetch error: File not found {file_path}"
-                                )
-                                self.file_status[file_path] = "missing"
-
-                except FileNotFoundError as e:
-                    with self.manager_lock:
-                        if self.file_status.get(file_path) != "deleted":
-                            logger.warning(f"Prefetch error for {file_path}: {e}")
-                            self.file_status[file_path] = "error"
-
-                self.prefetch_queue.task_done()
-
-            except queue.Empty:
-                time.sleep(0.01)  # Small sleep to prevent CPU spinning
-                continue
-
-    def save_tensor(self, tensor: torch.Tensor):
-        """Save tensor to disk asynchronously and return file path with thread-safe operations"""
-        # Generate unique file path
-        self.idx += 1
-        file_path: str = os.path.join(
-            self.temp_dir, f"{self.idx:06d}-{uuid.uuid4()}.pt"
-        )
-
-        with self.manager_lock:
-            # Mark file as being saved
-            self.file_locks[file_path] = threading.Lock()
-            self.file_status[file_path] = "saving"
-            # Add to history
-            self.tensor_paths.append(file_path)
-
-        # Acquire semaphore to limit concurrent save operations
-        self.save_semaphore.acquire()  # pylint: disable=consider-using-with
-        # Queue tensor for saving in background
-        self.save_queue.put((tensor.detach(), file_path))
-
-        return file_path
-
-    def wait_for_save(self, file_path, timeout=None) -> None:
-        """Wait for a tensor to be saved to disk"""
-        start_time = time.time()
-        while timeout is None or time.time() - start_time < timeout:
-            with self.manager_lock:
-                if self.file_status.get(file_path) == "ready":
-                    return
-                if self.file_status.get(file_path) in ["error", "missing", "deleted"]:
-                    return
-
-                if file_path in self.save_futures:
-                    future = self.save_futures[file_path]
-                    if future.done():
-                        return
-
-            # Small sleep to prevent CPU spinning
-            time.sleep(0.01)
-
-        # Timeout
-        logger.warning(f"Timeout waiting for tensor to be saved: {file_path}")
-        return
-
-    def load_tensor(self, file_path, target_device="cuda"):
-        """Load tensor from disk or prefetch cache with proper synchronization"""
-        # Wait for tensor to be saved if it's still in progress
-        self.wait_for_save(file_path)
-
-        tensor = None
-
-        # Try to get from cache first
-        with self.manager_lock:
-            # Check if tensor is already in cache
-            if file_path in self.prefetch_cache:
-                tensor = self.prefetch_cache[file_path]
-                del self.prefetch_cache[file_path]
-                self.file_status[file_path] = "loaded"
-
-        if tensor is not None:
-            # Ensure tensor is on correct device
-            if target_device != "cpu" and tensor.device.type == "cpu":
-                tensor = tensor.to(target_device, non_blocking=True)
-            return tensor
-
-        # If not in cache, load directly from disk
-        try:
-            if not os.path.exists(file_path):
-                logger.error(f"File not found for loading: {file_path}")
-                raise FileNotFoundError(f"File not found: {file_path}")
-
-            tensor = torch.load(file_path, weights_only=True)
-
-            with self.manager_lock:
-                self.file_status[file_path] = "loaded"
-
-            if target_device != "cpu":
-                tensor = tensor.to(target_device, non_blocking=True)
-
-            return tensor
-
-        except Exception as e:
-            logger.error(f"Error loading tensor from {file_path}: {e}")
-            raise
-
-    def _safe_delete_file(self, file_path):
-        """Safely delete a file with proper synchronization"""
-        with self.manager_lock:
-            # Make sure any save operation is completed
-            if file_path in self.save_futures:
-                future = self.save_futures[file_path]
-                try:
-                    if not future.done():
-                        future.cancel()
-                    del self.save_futures[file_path]
-                except FileNotFoundError as e:
-                    logger.warning(
-                        f"Error canceling save operation for {file_path}: {e}"
-                    )
-
-            # Only delete if file exists and is not being prefetched
-            status = self.file_status.get(file_path)
-            if status in ["ready", "loaded", "error", "missing"]:
-                try:
-                    if os.path.exists(file_path):
-                        os.remove(file_path)
-                    self.file_status[file_path] = "deleted"
-                    return True
-                except FileNotFoundError as e:
-                    logger.warning(f"Error deleting file {file_path}: {e}")
-            return False
-
-    def trigger_prefetch(self, n=None):
-        """Trigger prefetching of the next N tensors with proper synchronization"""
-        if n is None:
-            n = self.max_prefetch
-
-        prefetch_paths = []
-        with self.manager_lock:
-            # Find files that are ready to be prefetched (not already in cache or being prefetched)
-            for path in reversed(self.tensor_paths):
-                if (
-                    path not in self.prefetch_cache
-                    and self.file_status.get(path) == "ready"
-                ):
-                    prefetch_paths.append(path)
-                    if len(prefetch_paths) >= n:
-                        break
-
-        # Queue files for prefetching
-        for path in prefetch_paths:
-            self.prefetch_queue.put(path)
-
-    def cleanup_tensor(self, file_path: str):
-        """Clean up a specific tensor file after it's been used"""
-        with self.manager_lock:
-            if file_path in self.tensor_paths:
-                self.tensor_paths.remove(file_path)
-
-            # Remove from prefetch cache if present
-            if file_path in self.prefetch_cache:
-                del self.prefetch_cache[file_path]
-
-            # Remove from save futures if present
-            if file_path in self.save_futures:
-                future = self.save_futures[file_path]
-                if not future.done():
-                    future.cancel()
-                del self.save_futures[file_path]
-
-        # Try to delete the file
-        self._safe_delete_file(file_path)
-
-    def cleanup(self):
-        """Clean up all temp files and stop prefetch thread with proper synchronization"""
-        self.stop_event.set()
-
-        # Cancel all pending save operations
-        with self.manager_lock:
-            for _, future in self.save_futures.items():
-                if not future.done():
-                    future.cancel()
-            self.save_futures.clear()
-
-        # Drain the save queue
-        while not self.save_queue.empty():
-            try:
-                self.save_queue.get_nowait()
-                self.save_queue.task_done()
-            except queue.Empty:
-                break
-
-        # Shutdown the save pool
-        self.save_pool.shutdown(wait=False)
-
-        # Join the save worker thread
-        if self.save_worker.is_alive():
-            self.save_worker.join(timeout=2.0)
-
-        # Join the prefetch worker threads
-        for thread in self.prefetch_workers:
-            if thread.is_alive():
-                thread.join(timeout=2.0)
-
-        # Clear cache and remove all temporary files
-        with self.manager_lock:
-            self.prefetch_cache.clear()
-            paths_to_delete = list(self.tensor_paths)
-            self.tensor_paths.clear()
-
-        # Delete all temporary files
-        for path in paths_to_delete:
-            self._safe_delete_file(path)
-
-        # Remove temp directory
-        try:
-            if os.path.exists(self.temp_dir):
-                shutil.rmtree(self.temp_dir, ignore_errors=True)
-        except FileNotFoundError as e:
-            logger.warning(f"Error removing temporary directory {self.temp_dir}: {e}")
-
-
-class Disco(torch.autograd.Function):
-    """
-    Disco: DIsk-based Storage and Checkpointing with Optimized prefetching
-    Advanced disk-based gradient checkpointer with prefetching.
-    """
-
-    # Shared manager instance across all checkpointing operations
-    _manager = None
-
-    @staticmethod
-    def get_instance(prefetch_size=1, prefetch_to_gpu=True, save_workers=4):
-        """Get or create the offload manager"""
-        if Disco._manager is None:
-            Disco._manager = DiskOffloadManager(
-                prefetch_size=prefetch_size,
-                prefetch_to_gpu=prefetch_to_gpu,
-                save_workers=save_workers,
-            )
-        return Disco._manager
-
-    @staticmethod
-    @torch_cuda_amp_custom_fwd
-    def forward(
-        ctx,
-        forward_function,
-        hidden_states,
-        *args,
-        prefetch_size=1,
-        prefetch_to_gpu=True,
-        save_workers=4,
-    ):
-        """Forward pass that offloads activations to disk asynchronously"""
-        # Get or create the manager
-        manager = Disco.get_instance(
-            prefetch_size=prefetch_size,
-            prefetch_to_gpu=prefetch_to_gpu,
-            save_workers=save_workers,
-        )
-
-        # Save tensor to disk asynchronously
-        file_path = manager.save_tensor(hidden_states)
-
-        # Run forward pass immediately without waiting for save to complete
-        with torch.no_grad():
-            output = forward_function(hidden_states, *args)
-
-        # Store what we need for backward
-        ctx.save_for_backward(torch.tensor([0]))  # Dummy tensor
-        ctx.file_path = file_path
-        ctx.forward_function = forward_function
-        ctx.args = args
-
-        return output
-
-    @staticmethod
-    @torch_cuda_amp_custom_bwd
-    def backward(ctx, *grad_outputs):
-        """Backward pass that loads activations from disk with prefetching"""
-        # Get the manager
-        manager = Disco._manager
-
-        # Trigger prefetching for future tensors
-        # This happens at the start of backward, so should have time to complete
-        manager.trigger_prefetch()
-
-        # Load hidden states from disk or prefetch cache
-        file_path = ctx.file_path
-        try:
-            # Ensure the file is saved before we try to load it
-            manager.wait_for_save(file_path)
-
-            hidden_states = manager.load_tensor(file_path)
-            hidden_states.requires_grad = True
-
-            # Compute gradients
-            with torch.enable_grad():
-                output = ctx.forward_function(hidden_states, *ctx.args)
-
-                # Handle tuple outputs properly
-                if isinstance(output, tuple):
-                    if len(grad_outputs) == len(output):
-                        torch.autograd.backward(output, grad_outputs)
-                    else:
-                        torch.autograd.backward(output, grad_outputs[0])
-                else:
-                    torch.autograd.backward(output, grad_outputs[0])
-
-            # Clean up the file after we're done with it
-            manager.cleanup_tensor(file_path)
-
-            return (
-                (
-                    None,  # forward_function
-                    hidden_states.grad,  # hidden_states grad
-                )
-                + (None,) * len(ctx.args)  # for each arg
-                + (
-                    None,  # prefetch_size
-                    None,  # prefetch_to_gpu
-                    None,  # save_workers
-                )
-            )
-
-        except Exception as e:
-            logger.error(f"Error in backward pass: {e}")
-            # Clean up the file even on error
-            manager.cleanup_tensor(file_path)
-            raise

src/axolotl/utils/gradient_checkpointing/unsloth.py

@@ -1,4 +1,4 @@
-"""CPU offloaded checkpointing"""
+"""Unsloth checkpointing"""
 
 # Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
 #
@@ -26,7 +26,7 @@ else:
     torch_cuda_amp_custom_bwd = torch.amp.custom_bwd(device_type="cuda")
 
 
-class CPU_Offloaded_Gradient_Checkpointer(  # pylint: disable=invalid-name
+class Unsloth_Offloaded_Gradient_Checkpointer(  # pylint: disable=invalid-name
     torch.autograd.Function
 ):
     """

src/axolotl/utils/models.py

@@ -70,10 +70,7 @@ from axolotl.utils.distributed import (
     is_local_main_process,
     is_main_process,
 )
-from axolotl.utils.gradient_checkpointing import (
-    hf_grad_checkpoint_disk_offload_wrapper,
-    hf_grad_checkpoint_offload_wrapper,
-)
+from axolotl.utils.gradient_checkpointing import hf_grad_checkpoint_offload_wrapper
 from axolotl.utils.lora_embeddings import get_linear_embedding_layers
 from axolotl.utils.model_shard_quant import load_sharded_model, load_sharded_model_quant
 from axolotl.utils.schemas.enums import RLType
@@ -623,10 +620,6 @@ class ModelLoader:
 
         if self.cfg.gradient_checkpointing in ["unsloth", "offload"]:
             transformers.modeling_utils.checkpoint = hf_grad_checkpoint_offload_wrapper
-        if self.cfg.gradient_checkpointing == "offload_disk":
-            transformers.modeling_utils.checkpoint = (
-                hf_grad_checkpoint_disk_offload_wrapper
-            )
 
         if self.cfg.flash_attention:
             self.patch_attention()

src/axolotl/utils/schemas/config.py

@@ -83,6 +83,7 @@ class AxolotlInputConfig(
     # optionally shrink the embeddings when the tokenizer vocab size is smaller
     shrink_embeddings: bool | None = None
     embeddings_skip_upcast: bool | None = None
+    random_init_weights: bool | None = None
 
     rl: RLType | None = None
     trl: TRLConfig | None = Field(
@@ -178,7 +179,7 @@ class AxolotlInputConfig(
 
     # torch_dtype: torch.dtype | None
 
-    gradient_checkpointing: Literal["offload", "offload_disk"] | bool | None = Field(
+    gradient_checkpointing: Literal["unsloth", "offload"] | bool | None = Field(
         default=False
     )
     gradient_checkpointing_kwargs: dict[str, Any] | None = None
@@ -186,12 +187,6 @@ class AxolotlInputConfig(
     unfrozen_parameters: list[str] | None = None
 
     sequence_len: int = Field(default=512)
-    sequence_len_overflow_handling: Literal["drop", "truncate"] = Field(
-        default="drop",
-        json_schema_extra={
-            "description": "How to handle sequences that overflow the sequence_len: 'drop' (remove the sample) or 'truncate' (cut off excess tokens)."
-        },
-    )
     min_sample_len: int | None = None
     max_prompt_len: int = Field(
         default=512,

src/axolotl/utils/trainer.py

@@ -207,18 +207,10 @@ def add_length(sample):
 
 def drop_long_seq(sample, sequence_len=2048, min_sequence_len=2):
     """
-    Determines whether samples should be kept based on sequence length constraints.
-    
-    For a single example or a batch, returns True (or a list of booleans) if each sequence's length is within the specified range; otherwise, returns False (or a list with False for out-of-range sequences).
-    
-    Args:
-        sample: A dictionary containing "input_ids" as a list of ints or a list of lists of ints.
-        sequence_len: Maximum allowed sequence length (inclusive).
-        min_sequence_len: Minimum allowed sequence length (inclusive).
-    
-    Returns:
-        True if the single example is within the length range, False otherwise.
-        For batched input, returns a list of booleans indicating which sequences are within the range.
+    Drop samples whose sequence length is either too long (> sequence_len)
+    or too short (< min_sequence_len).
+
+    Works for both single-example (list[int]) or batched (list[list[int]]).
     """
     min_sequence_len = min_sequence_len or 2
 
@@ -243,121 +235,7 @@ def drop_long_seq(sample, sequence_len=2048, min_sequence_len=2):
     return results
 
 
-def truncate_or_drop_long_seq(
-    sample, sequence_len=2048, min_sequence_len=2, handling="drop"
-):
-    """
-    Drops or truncates samples based on sequence length constraints.
-    
-    If handling is "drop", returns a boolean or list of booleans indicating whether each sample's sequence length is within the specified range. If handling is "truncate", returns the sample with sequences longer than sequence_len truncated and sequences shorter than min_sequence_len omitted. Supports both single-example and batched inputs.
-    
-    Args:
-        sample: A dictionary containing at least an "input_ids" field, representing either a single sequence or a batch of sequences.
-        sequence_len: Maximum allowed sequence length.
-        min_sequence_len: Minimum allowed sequence length.
-        handling: "drop" to filter out samples outside the range, "truncate" to truncate long sequences.
-    
-    Returns:
-        In "drop" mode, a boolean or list of booleans indicating which samples to keep. In "truncate" mode, the modified sample with sequences truncated as needed.
-    """
-    min_sequence_len = min_sequence_len or 2
-    result = None
-
-    if handling == "drop":
-        return drop_long_seq(sample, sequence_len, min_sequence_len)
-
-    input_ids = sample["input_ids"]
-
-    # Edge case: if input_ids is empty
-    if not input_ids:
-        result = False if handling == "drop" else sample
-    # Single example (input_ids is a list of int)
-    elif isinstance(input_ids[0], int):
-        length = len(input_ids)
-
-        # Handle samples that are too short - always drop them
-        if length < min_sequence_len:
-            result = False if handling == "drop" else sample
-        # If truncation is enabled and the sample is too long, truncate it
-        elif length > sequence_len and handling == "truncate":
-            sample["input_ids"] = input_ids[:sequence_len]
-
-            # Also truncate attention_mask if present
-            if "attention_mask" in sample:
-                sample["attention_mask"] = sample["attention_mask"][:sequence_len]
-
-            # Also truncate labels if present
-            if "labels" in sample:
-                sample["labels"] = sample["labels"][:sequence_len]
-
-            # Also truncate position_ids if present
-            if "position_ids" in sample:
-                sample["position_ids"] = sample["position_ids"][:sequence_len]
-
-            # Update length if present
-            if "length" in sample:
-                sample["length"] = sequence_len
-
-            result = sample
-        # For drop mode or if the sample doesn't exceed max length
-        else:
-            result = (
-                min_sequence_len <= length <= sequence_len
-                if handling == "drop"
-                else sample
-            )
-    # Batched (input_ids is a list of lists)
-    else:
-        if handling == "drop":
-            results = []
-            for seq in input_ids:
-                length = len(seq)
-                results.append(min_sequence_len <= length <= sequence_len)
-            result = results
-        else:  # truncate
-            # Check each sequence in the batch
-            for i, seq in enumerate(input_ids):
-                length = len(seq)
-
-                # Skip sequences that are too short
-                if length < min_sequence_len:
-                    continue
-
-                # Truncate sequences that are too long
-                if length > sequence_len:
-                    input_ids[i] = seq[:sequence_len]
-
-                    # Also truncate attention_mask if present
-                    if "attention_mask" in sample:
-                        sample["attention_mask"][i] = sample["attention_mask"][i][
-                            :sequence_len
-                        ]
-
-                    # Also truncate labels if present
-                    if "labels" in sample:
-                        sample["labels"][i] = sample["labels"][i][:sequence_len]
-
-                    # Also truncate position_ids if present
-                    if "position_ids" in sample:
-                        sample["position_ids"][i] = sample["position_ids"][i][
-                            :sequence_len
-                        ]
-
-                    # Update length if present
-                    if "length" in sample:
-                        sample["length"][i] = sequence_len
-
-            result = sample
-
-    return result
-
-
 def process_datasets_for_packing(cfg, train_dataset, eval_dataset):
-    """
-    Prepares training and evaluation datasets for sample packing and model-specific requirements.
-    
-    Removes unnecessary columns based on model type, filters out samples with no trainable tokens, and optionally adds length or position ID columns for sample packing or PoSE techniques. Returns the processed training and evaluation datasets.
-    """
     drop_attn_mask = cfg.model_config_type in ["mamba", "gemma3"]
     if drop_attn_mask:
         LOG.info("dropping attention_mask column")
@@ -492,48 +370,15 @@ def process_datasets_for_packing(cfg, train_dataset, eval_dataset):
 
 
 def process_pretraining_datasets_for_packing(
-    train_dataset,
-    sequence_len,
-    skip_position_ids=True,
-    drop_attention_mask=False,
-    handling="drop",
+    train_dataset, sequence_len, skip_position_ids=True, drop_attention_mask=False
 ):
-    # Define the function to use for handling sequences based on the mode
-    """
-    Processes a pretraining dataset by truncating or dropping sequences based on length.
-    
-    Depending on the handling mode, sequences longer than `sequence_len` are either truncated or dropped, and sequences shorter than `min_sequence_len` are dropped. Optionally adds position IDs and removes the attention mask column.
-    
-    Args:
-        train_dataset: The dataset to process.
-        sequence_len: Maximum allowed sequence length.
-        skip_position_ids: If False, adds position IDs to each sample.
-        drop_attention_mask: If True, removes the attention mask column.
-        handling: "drop" to remove long sequences, "truncate" to truncate them.
-    
-    Returns:
-        The processed dataset with sequences handled according to the specified mode.
-    """
-    seq_handler_fn = partial(
-        truncate_or_drop_long_seq,
-        sequence_len=sequence_len,
-        handling=handling,  # Pass handling mode
+    drop_long = partial(drop_long_seq, sequence_len=sequence_len)
+
+    train_dataset = train_dataset.filter(
+        drop_long,
+        desc="Dropping Long Sequences",
+        load_from_cache_file=False,
     )
-
-    # Use map for truncate mode and filter for drop mode
-    if handling == "truncate":
-        train_dataset = train_dataset.map(
-            seq_handler_fn,
-            desc="Truncating Long Sequences",
-            load_from_cache_file=False,
-        )
-    else:  # handling == "drop"
-        train_dataset = train_dataset.filter(
-            seq_handler_fn,  # Use the same function, it returns boolean for drop mode
-            desc="Dropping Long Sequences",
-            load_from_cache_file=False,
-        )
-
     if not skip_position_ids:
         train_dataset = train_dataset.map(
             add_position_ids,

tests/e2e/patched/test_activation_checkpointing.py

@@ -26,15 +26,10 @@ class TestActivationCheckpointing:
     E2E tests for activation checkpointing
     """
 
-    @pytest.mark.parametrize(
-        "gradient_checkpointing",
-        ["offload", "offload_disk"],
-    )
     def test_activation_checkpointing_offload(
         self,
         temp_dir,
         fix_checkpoint_after_test,  # pylint: disable=unused-argument,redefined-outer-name
-        gradient_checkpointing,
     ):
         # pylint: disable=duplicate-code
         cfg = DictDefault(
@@ -69,7 +64,7 @@ class TestActivationCheckpointing:
                 "sample_packing": True,
                 "bf16": True,
                 "save_safetensors": True,
-                "gradient_checkpointing": gradient_checkpointing,
+                "gradient_checkpointing": "offload",
             }
         )
 

tests/test_data.py

@@ -3,12 +3,10 @@ test module for the axolotl.utils.data module
 """
 
 import unittest
-from unittest.mock import MagicMock
 
 from transformers import LlamaTokenizer
 
 from axolotl.utils.data import encode_pretraining, md5
-from axolotl.utils.data.rl import drop_long_rl_seq
 
 from tests.hf_offline_utils import enable_hf_offline
 
@@ -60,328 +58,11 @@ class TestEncodePretraining(unittest.TestCase):
         self.assertEqual(result["input_ids"][0][14], self.tokenizer.pad_token_id)
 
     def test_md5(self):
-        """
-        Tests that the md5 function returns the correct hash for a given string and encoding.
-        """
         self.assertEqual(md5("hello world"), "5eb63bbbe01eeed093cb22bb8f5acdc3")
         self.assertEqual(
             md5("hello world", "utf-8"), "5eb63bbbe01eeed093cb22bb8f5acdc3"
         )
 
 
-class TestDropLongRLSeq(unittest.TestCase):
-    """
-    Tests for the drop_long_rl_seq function.
-    """
-
-    def setUp(self):
-        # Mock tokenizer that returns length based on input string length
-        """
-        Sets up a mock tokenizer and sequence length for RL sequence length tests.
-        
-        The mock tokenizer simulates tokenization by returning input IDs equal to the input string's length and decodes tokens as repeated "x" characters. The sequence length limit is set to 20.
-        """
-        self.tokenizer = MagicMock()
-
-        def side_effect_func(
-            text, add_special_tokens=False
-        ):  # pylint: disable=unused-argument
-            """
-            Simulates tokenization by returning input IDs as a sequence of integers equal to the input text length.
-            
-            Args:
-                text: The input string to tokenize.
-                add_special_tokens: Ignored parameter included for interface compatibility.
-            
-            Returns:
-                A dictionary with 'input_ids' as a list of integers from 0 to len(text) - 1.
-            """
-            return {"input_ids": list(range(len(text)))}
-
-        self.tokenizer.side_effect = side_effect_func
-        self.tokenizer.decode = lambda tokens, skip_special_tokens: "".join(
-            ["x"] * len(tokens)
-        )  # pylint: disable=unused-argument
-
-        self.sequence_len = 20
-
-    def test_dpo_drop_mode_valid(self):
-        """
-        Tests that drop_long_rl_seq returns True in drop mode for a DPO sample within the sequence length limit.
-        """
-        sample = {
-            "prompt": "p" * 5,
-            "chosen": "c" * 7,
-            "rejected": "r" * 6,
-        }  # 5+7=12 <= 20, 5+6=11 <= 20
-        result = drop_long_rl_seq(
-            sample, "dpo", self.tokenizer, self.sequence_len, handling="drop"
-        )
-        self.assertTrue(result)
-
-    def test_dpo_drop_mode_invalid_chosen(self):
-        """
-        Tests that in DPO drop mode, a sample is rejected when the prompt and chosen lengths exceed the sequence limit.
-        """
-        sample = {
-            "prompt": "p" * 5,
-            "chosen": "c" * 16,
-            "rejected": "r" * 6,
-        }  # 5+16=21 > 20
-        result = drop_long_rl_seq(
-            sample, "dpo", self.tokenizer, self.sequence_len, handling="drop"
-        )
-        self.assertFalse(result)
-
-    def test_dpo_drop_mode_invalid_rejected(self):
-        """
-        Tests that in DPO drop mode, a sample is rejected when the prompt plus rejected response exceeds the sequence length limit.
-        """
-        sample = {
-            "prompt": "p" * 5,
-            "chosen": "c" * 7,
-            "rejected": "r" * 16,
-        }  # 5+16=21 > 20
-        result = drop_long_rl_seq(
-            sample, "dpo", self.tokenizer, self.sequence_len, handling="drop"
-        )
-        self.assertFalse(result)
-
-    def test_dpo_truncate_mode_no_truncation_needed(self):
-        """
-        Verifies that in DPO truncate mode, samples within the sequence length limit are returned unchanged.
-        """
-        sample = {
-            "prompt": "p" * 5,
-            "chosen": "c" * 7,
-            "rejected": "r" * 6,
-        }  # 5+7=12 <= 20, 5+6=11 <= 20
-        original_sample = sample.copy()
-        result = drop_long_rl_seq(
-            sample, "dpo", self.tokenizer, self.sequence_len, handling="truncate"
-        )
-        self.assertEqual(
-            result, original_sample
-        )  # Should return the original sample unchanged
-
-    def test_dpo_truncate_mode_prompt_too_long(self):
-        """
-        Tests that in DPO truncate mode, if the prompt exceeds the sequence length limit,
-        the original sample is returned unchanged.
-        """
-        sample = {"prompt": "p" * 25, "chosen": "c" * 7, "rejected": "r" * 6}
-        original_sample = sample.copy()
-        result = drop_long_rl_seq(
-            sample, "dpo", self.tokenizer, self.sequence_len, handling="truncate"
-        )
-        # Even though truncation isn't possible, the function should return the original sample
-        # for the map operation, assuming downstream filtering will catch it.
-        self.assertEqual(result, original_sample)
-
-    def test_dpo_truncate_mode_chosen_truncated(self):
-        """
-        Tests that in DPO truncate mode, only the 'chosen' field is truncated when it exceeds the allowed sequence length, while 'prompt' and 'rejected' remain unchanged.
-        """
-        prompt_len = 5
-        max_resp_len = self.sequence_len - prompt_len  # 20 - 5 = 15
-        sample = {
-            "prompt": "p" * prompt_len,
-            "chosen": "c" * 18,
-            "rejected": "r" * 10,
-        }  # 5+18=23 > 20, 5+10=15 <= 20
-        result = drop_long_rl_seq(
-            sample, "dpo", self.tokenizer, self.sequence_len, handling="truncate"
-        )
-        self.assertEqual(len(result["prompt"]), prompt_len)
-        self.assertEqual(len(result["chosen"]), max_resp_len)  # Truncated to 15
-        self.assertEqual(
-            result["chosen"], "x" * max_resp_len
-        )  # Check decoded truncated value
-        self.assertEqual(len(result["rejected"]), 10)  # Unchanged
-
-    def test_dpo_truncate_mode_rejected_truncated(self):
-        """
-        Tests that in DPO truncate mode, only the 'rejected' field is truncated when it exceeds the sequence length limit, while 'prompt' and 'chosen' remain unchanged.
-        """
-        prompt_len = 5
-        max_resp_len = self.sequence_len - prompt_len  # 15
-        sample = {
-            "prompt": "p" * prompt_len,
-            "chosen": "c" * 10,
-            "rejected": "r" * 18,
-        }  # 5+10=15 <= 20, 5+18=23 > 20
-        result = drop_long_rl_seq(
-            sample, "dpo", self.tokenizer, self.sequence_len, handling="truncate"
-        )
-        self.assertEqual(len(result["prompt"]), prompt_len)
-        self.assertEqual(len(result["chosen"]), 10)  # Unchanged
-        self.assertEqual(len(result["rejected"]), max_resp_len)  # Truncated to 15
-        self.assertEqual(
-            result["rejected"], "x" * max_resp_len
-        )  # Check decoded truncated value
-
-    def test_dpo_truncate_mode_both_truncated(self):
-        """
-        Tests that in DPO truncate mode, both 'chosen' and 'rejected' fields are truncated when their combined lengths with the prompt exceed the sequence limit.
-        
-        Verifies that both fields are truncated to fit within the allowed response length and replaced with decoded placeholder content.
-        """
-        prompt_len = 8
-        max_resp_len = self.sequence_len - prompt_len  # 20 - 8 = 12
-        sample = {
-            "prompt": "p" * prompt_len,
-            "chosen": "c" * 15,
-            "rejected": "r" * 14,
-        }  # 8+15=23 > 20, 8+14=22 > 20
-        result = drop_long_rl_seq(
-            sample, "dpo", self.tokenizer, self.sequence_len, handling="truncate"
-        )
-        self.assertEqual(len(result["prompt"]), prompt_len)
-        self.assertEqual(len(result["chosen"]), max_resp_len)  # Truncated to 12
-        self.assertEqual(result["chosen"], "x" * max_resp_len)
-        self.assertEqual(len(result["rejected"]), max_resp_len)  # Truncated to 12
-        self.assertEqual(result["rejected"], "x" * max_resp_len)
-
-    def test_dpo_truncate_mode_no_truncation_needed_but_long(self):
-        """
-        Tests DPO truncate mode where only the overlong response is truncated.
-        
-        Verifies that when the prompt plus one response exceeds the sequence length, only the response exceeding the maximum allowed length is truncated, while the other remains unchanged.
-        """
-        # This tests the case where len(chosen) <= max_resp_len and len(rejected) <= max_resp_len
-        # but the initial check failed because e.g. prompt + chosen > sequence_len
-        # The current logic *will* truncate if len(chosen) > max_resp_len.
-        # Let's test a case where one is slightly too long causing the initial fail,
-        # but the other fits *within* the max_response_len, so only one gets truncated.
-        prompt_len = 10
-        max_resp_len = self.sequence_len - prompt_len  # 10
-        sample = {
-            "prompt": "p" * prompt_len,
-            "chosen": "c" * 11,
-            "rejected": "r" * 9,
-        }  # 10+11=21 > 20, 10+9=19 <= 20
-        result = drop_long_rl_seq(
-            sample, "dpo", self.tokenizer, self.sequence_len, handling="truncate"
-        )
-        self.assertEqual(len(result["prompt"]), prompt_len)
-        self.assertEqual(len(result["chosen"]), max_resp_len)  # Truncated to 10
-        self.assertEqual(result["chosen"], "x" * max_resp_len)
-        self.assertEqual(len(result["rejected"]), 9)  # Unchanged, as 9 <= 10
-
-    # Add similar tests for KTO if needed, checking prompt + completion length
-
-    def test_kto_drop_mode_valid(self):
-        """
-        Tests that drop_long_rl_seq returns True for a KTO sample within the sequence length limit.
-        """
-        sample = {"prompt": "p" * 5, "completion": "c" * 14}  # 5+14=19 <= 20
-        result = drop_long_rl_seq(
-            sample, "kto", self.tokenizer, self.sequence_len, handling="drop"
-        )
-        self.assertTrue(result)
-
-    def test_kto_drop_mode_invalid(self):
-        """
-        Tests that drop_long_rl_seq returns False when a KTO sample exceeds the sequence length limit in drop mode.
-        """
-        sample = {"prompt": "p" * 5, "completion": "c" * 16}  # 5+16=21 > 20
-        result = drop_long_rl_seq(
-            sample, "kto", self.tokenizer, self.sequence_len, handling="drop"
-        )
-        self.assertFalse(result)
-
-    def test_kto_truncate_mode_no_truncation_needed(self):
-        """
-        Tests that KTO truncate mode returns the original sample unchanged when the combined prompt and completion length does not exceed the sequence limit.
-        """
-        sample = {"prompt": "p" * 5, "completion": "c" * 14}  # 5+14=19 <= 20
-        original_sample = sample.copy()
-        result = drop_long_rl_seq(
-            sample, "kto", self.tokenizer, self.sequence_len, handling="truncate"
-        )
-        self.assertEqual(result, original_sample)
-
-    def test_kto_truncate_mode_prompt_too_long(self):
-        """
-        Tests that in KTO truncate mode, if the prompt exceeds the sequence length limit, the original sample is returned unchanged.
-        """
-        sample = {"prompt": "p" * 25, "completion": "c" * 7}
-        original_sample = sample.copy()
-        result = drop_long_rl_seq(
-            sample, "kto", self.tokenizer, self.sequence_len, handling="truncate"
-        )
-        self.assertEqual(result, original_sample)  # Returns original sample
-
-    def test_kto_truncate_mode_completion_truncated(self):
-        """
-        Tests that in KTO truncate mode, the completion is truncated when the combined prompt and completion exceed the sequence length limit.
-        
-        Verifies that the prompt remains unchanged and the completion is truncated to fit within the allowed length, with the truncated completion replaced by decoded "x" characters.
-        """
-        prompt_len = 8
-        max_comp_len = self.sequence_len - prompt_len  # 20 - 8 = 12
-        sample = {"prompt": "p" * prompt_len, "completion": "c" * 15}  # 8+15=23 > 20
-        result = drop_long_rl_seq(
-            sample, "kto", self.tokenizer, self.sequence_len, handling="truncate"
-        )
-        self.assertEqual(len(result["prompt"]), prompt_len)
-        self.assertEqual(len(result["completion"]), max_comp_len)  # Truncated to 12
-        self.assertEqual(result["completion"], "x" * max_comp_len)
-
-    def test_missing_keys_dpo(self):
-        """
-        Tests that a ValueError is raised when required keys are missing for DPO samples.
-        
-        Verifies that the function raises an error if the sample does not contain 'chosen' and 'rejected' keys.
-        """
-        sample = {"prompt": "p"}
-        with self.assertRaisesRegex(
-            ValueError, "Prompt, chosen and rejected keys are required"
-        ):
-            drop_long_rl_seq(sample, "dpo", self.tokenizer, self.sequence_len)
-
-    def test_missing_keys_kto(self):
-        """
-        Tests that a ValueError is raised when required keys are missing for RL type "kto".
-        
-        Verifies that calling drop_long_rl_seq with a sample missing the "completion" key raises
-        a ValueError with the expected error message.
-        """
-        sample = {"prompt": "p"}
-        with self.assertRaisesRegex(
-            ValueError, "Prompt and completion keys are required"
-        ):
-            drop_long_rl_seq(sample, "kto", self.tokenizer, self.sequence_len)
-
-    def test_unknown_rl_type(self):
-        """
-        Tests that a ValueError is raised when an unknown RL type is provided to drop_long_rl_seq.
-        """
-        sample = {}
-        with self.assertRaisesRegex(ValueError, "Unknown RL type"):
-            drop_long_rl_seq(sample, "xyz", self.tokenizer, self.sequence_len)
-
-    # GRPO test - current implementation always passes
-    def test_grpo_drop(self):
-        """
-        Tests that drop_long_rl_seq in GRPO drop mode always returns True, regardless of input.
-        """
-        sample = {}
-        result = drop_long_rl_seq(
-            sample, "grpo", self.tokenizer, self.sequence_len, handling="drop"
-        )
-        self.assertTrue(result)
-
-    def test_grpo_truncate(self):
-        """
-        Tests that in truncate mode for RL type "grpo", the original sample is returned unchanged.
-        """
-        sample = {"a": 1}
-        result = drop_long_rl_seq(
-            sample, "grpo", self.tokenizer, self.sequence_len, handling="truncate"
-        )
-        self.assertEqual(result, sample)
-
-
 if __name__ == "__main__":
     unittest.main()

tests/test_trainer_utils.py

@@ -1,175 +0,0 @@
-"""Module containing tests for trainer utility functions."""
-
-import unittest
-from functools import partial
-
-from axolotl.utils.trainer import truncate_or_drop_long_seq
-
-
-# Test cases for truncate_or_drop_long_seq
-class TestTruncateOrDropLongSeq(unittest.TestCase):
-    """
-    Test suite for truncate_or_drop_long_seq function.
-    """
-
-    def setUp(self):
-        # Example sequence length settings
-        """
-        Sets up default sequence length parameters for the test cases.
-        """
-        self.sequence_len = 10
-        self.min_sequence_len = 3
-
-    def test_drop_mode_single(self):
-        """
-        Verifies that 'drop' mode correctly filters single sequence examples based on length.
-        
-        Tests that sequences shorter than the minimum, longer than the maximum, or empty are dropped,
-        while sequences within the valid length range are kept.
-        """
-        handler = partial(
-            truncate_or_drop_long_seq,
-            sequence_len=self.sequence_len,
-            min_sequence_len=self.min_sequence_len,
-            handling="drop",
-        )
-
-        # Too short
-        sample_short = {"input_ids": [1, 2]}
-        self.assertFalse(handler(sample_short))
-
-        # Too long
-        sample_long = {"input_ids": list(range(self.sequence_len + 1))}
-        self.assertFalse(handler(sample_long))
-
-        # Just right
-        sample_ok = {"input_ids": list(range(self.min_sequence_len))}
-        self.assertTrue(handler(sample_ok))
-
-        # Empty
-        sample_empty = {"input_ids": []}
-        self.assertFalse(handler(sample_empty))
-
-    def test_truncate_mode_single(self):
-        """
-        Tests that 'truncate_or_drop_long_seq' correctly truncates or preserves single examples in "truncate" mode.
-        
-        Verifies that sequences longer than the maximum length are truncated, while sequences that are too short, empty, or within the valid range remain unchanged.
-        """
-        handler = partial(
-            truncate_or_drop_long_seq,
-            sequence_len=self.sequence_len,
-            min_sequence_len=self.min_sequence_len,
-            handling="truncate",
-        )
-
-        # Too short (should still be dropped implicitly by filter/map logic upstream,
-        # but the function itself might return the sample or False based on impl.)
-        # Current impl returns the original sample for map if too short, assuming upstream filters.
-        # Let's refine this test - the function *itself* returns the sample if too short when truncating.
-        sample_short = {"input_ids": [1, 2], "labels": [1, 2]}
-        result_short = handler(sample_short)
-        self.assertEqual(result_short["input_ids"], [1, 2])  # Unchanged
-
-        # Too long
-        original_long = list(range(self.sequence_len + 5))
-        sample_long = {"input_ids": list(original_long), "labels": list(original_long)}
-        result_long = handler(sample_long)
-        self.assertEqual(len(result_long["input_ids"]), self.sequence_len)
-        self.assertEqual(result_long["input_ids"], list(range(self.sequence_len)))
-        self.assertEqual(len(result_long["labels"]), self.sequence_len)
-        self.assertEqual(result_long["labels"], list(range(self.sequence_len)))
-
-        # Just right
-        sample_ok = {
-            "input_ids": list(range(self.min_sequence_len)),
-            "labels": list(range(self.min_sequence_len)),
-        }
-        result_ok = handler(sample_ok)
-        self.assertEqual(len(result_ok["input_ids"]), self.min_sequence_len)
-        self.assertEqual(result_ok, sample_ok)  # Should be unchanged
-
-        # Empty
-        sample_empty = {"input_ids": [], "labels": []}
-        result_empty = handler(sample_empty)
-        self.assertEqual(result_empty, sample_empty)  # Unchanged
-
-    def test_drop_mode_batched(self):
-        """
-        Tests that the "drop" handling mode correctly filters batched input sequences based on length constraints.
-        
-        Verifies that sequences shorter than the minimum length, longer than the maximum length, or empty are dropped (returns False), while sequences within the valid range are kept (returns True).
-        """
-        handler = partial(
-            truncate_or_drop_long_seq,
-            sequence_len=self.sequence_len,
-            min_sequence_len=self.min_sequence_len,
-            handling="drop",
-        )
-        sample = {
-            "input_ids": [
-                [1, 2],  # Too short
-                list(range(self.sequence_len + 1)),  # Too long
-                list(range(self.sequence_len)),  # OK (len = 10)
-                list(range(self.min_sequence_len)),  # OK (len = 3)
-                [],  # Empty
-            ]
-        }
-        expected = [False, False, True, True, False]
-        self.assertEqual(handler(sample), expected)
-
-    def test_truncate_mode_batched(self):
-        """
-        Tests that batched examples are correctly truncated in "truncate" mode.
-        
-        Verifies that sequences in both "input_ids" and "labels" longer than the maximum
-        allowed length are truncated, while sequences that are too short or empty remain
-        unchanged.
-        """
-        handler = partial(
-            truncate_or_drop_long_seq,
-            sequence_len=self.sequence_len,
-            min_sequence_len=self.min_sequence_len,
-            handling="truncate",
-        )
-        sample = {
-            "input_ids": [
-                [1, 2],  # Too short
-                list(range(self.sequence_len + 5)),  # Too long
-                list(range(self.sequence_len)),  # OK
-                list(range(self.min_sequence_len)),  # OK
-                [],  # Empty
-            ],
-            "labels": [  # Add labels to test truncation
-                [1, 2],
-                list(range(self.sequence_len + 5)),
-                list(range(self.sequence_len)),
-                list(range(self.min_sequence_len)),
-                [],
-            ],
-        }
-
-        result = handler(sample)
-
-        # Expected results after truncation (too short and empty remain unchanged by this function)
-        expected_input_ids = [
-            [1, 2],  # Unchanged (too short)
-            list(range(self.sequence_len)),  # Truncated
-            list(range(self.sequence_len)),  # Unchanged (OK)
-            list(range(self.min_sequence_len)),  # Unchanged (OK)
-            [],  # Unchanged (Empty)
-        ]
-        expected_labels = [
-            [1, 2],  # Unchanged (too short)
-            list(range(self.sequence_len)),  # Truncated
-            list(range(self.sequence_len)),  # Unchanged (OK)
-            list(range(self.min_sequence_len)),  # Unchanged (OK)
-            [],  # Unchanged (Empty)
-        ]
-
-        self.assertEqual(result["input_ids"], expected_input_ids)
-        self.assertEqual(result["labels"], expected_labels)
-
-
-if __name__ == "__main__":
-    unittest.main()