From 5724ca4e579f25806a675a7909edd952cf2e1530 Mon Sep 17 00:00:00 2001 From: Quarto GHA Workflow Runner Date: Thu, 2 Apr 2026 12:08:47 +0000 Subject: [PATCH] Built site for gh-pages --- .nojekyll | 2 +- FAQS.html | 30 + docs/agents/grpo.html | 1322 ++++++++++ docs/agents/preference_tuning.html | 1449 +++++++++++ docs/agents/pretraining.html | 1319 ++++++++++ docs/agents/reward_modelling.html | 1256 +++++++++ docs/agents/sft.html | 1443 +++++++++++ docs/amd_hpc.html | 30 + docs/api/cli.args.html | 30 + docs/api/cli.art.html | 30 + docs/api/cli.checks.html | 30 + docs/api/cli.cloud.base.html | 30 + docs/api/cli.cloud.modal_.html | 30 + docs/api/cli.config.html | 30 + docs/api/cli.delinearize_llama4.html | 30 + docs/api/cli.evaluate.html | 30 + docs/api/cli.inference.html | 30 + docs/api/cli.main.html | 30 + docs/api/cli.merge_lora.html | 30 + docs/api/cli.merge_sharded_fsdp_weights.html | 30 + docs/api/cli.preprocess.html | 30 + docs/api/cli.quantize.html | 30 + docs/api/cli.train.html | 30 + docs/api/cli.utils.args.html | 30 + docs/api/cli.utils.fetch.html | 30 + docs/api/cli.utils.html | 30 + docs/api/cli.utils.load.html | 30 + docs/api/cli.utils.sweeps.html | 30 + docs/api/cli.utils.train.html | 30 + docs/api/cli.vllm_serve.html | 30 + docs/api/common.architectures.html | 30 + docs/api/common.const.html | 30 + docs/api/common.datasets.html | 30 + docs/api/convert.html | 30 + docs/api/core.builders.base.html | 30 + docs/api/core.builders.causal.html | 30 + docs/api/core.builders.rl.html | 30 + docs/api/core.chat.format.chatml.html | 30 + docs/api/core.chat.format.llama3x.html | 30 + docs/api/core.chat.format.shared.html | 30 + docs/api/core.chat.messages.html | 30 + docs/api/core.datasets.chat.html | 30 + ...core.datasets.transforms.chat_builder.html | 30 + docs/api/core.trainers.base.html | 30 + docs/api/core.trainers.dpo.trainer.html | 30 + docs/api/core.trainers.grpo.sampler.html | 30 + docs/api/core.trainers.grpo.trainer.html | 30 + docs/api/core.trainers.mamba.html | 30 + docs/api/core.trainers.mixins.optimizer.html | 30 + ...core.trainers.mixins.rng_state_loader.html | 30 + docs/api/core.trainers.mixins.scheduler.html | 30 + docs/api/core.trainers.trl.html | 30 + docs/api/core.trainers.utils.html | 30 + docs/api/core.training_args.html | 30 + docs/api/datasets.html | 30 + docs/api/evaluate.html | 30 + docs/api/index.html | 30 + docs/api/integrations.base.html | 30 + .../integrations.cut_cross_entropy.args.html | 30 + docs/api/integrations.grokfast.optimizer.html | 30 + docs/api/integrations.kd.trainer.html | 30 + docs/api/integrations.liger.args.html | 30 + docs/api/integrations.lm_eval.args.html | 30 + docs/api/integrations.spectrum.args.html | 30 + docs/api/kernels.geglu.html | 30 + docs/api/kernels.lora.html | 30 + docs/api/kernels.quantize.html | 30 + docs/api/kernels.swiglu.html | 30 + docs/api/kernels.utils.html | 30 + docs/api/loaders.adapter.html | 30 + docs/api/loaders.constants.html | 30 + docs/api/loaders.model.html | 30 + docs/api/loaders.patch_manager.html | 30 + docs/api/loaders.processor.html | 30 + docs/api/loaders.tokenizer.html | 30 + docs/api/logging_config.html | 30 + docs/api/models.mamba.modeling_mamba.html | 30 + .../monkeypatch.btlm_attn_hijack_flash.html | 30 + ...onkeypatch.data.batch_dataset_fetcher.html | 30 + ...ch.gradient_checkpointing.offload_cpu.html | 30 + ...h.gradient_checkpointing.offload_disk.html | 30 + .../monkeypatch.llama_attn_hijack_flash.html | 30 + ...onkeypatch.llama_attn_hijack_xformers.html | 30 + docs/api/monkeypatch.lora_kernels.html | 30 + ...monkeypatch.mistral_attn_hijack_flash.html | 30 + docs/api/monkeypatch.mixtral.html | 30 + docs/api/monkeypatch.multipack.html | 30 + docs/api/monkeypatch.relora.html | 30 + ...onkeypatch.stablelm_attn_hijack_flash.html | 30 + docs/api/monkeypatch.trainer_fsdp_optim.html | 30 + .../monkeypatch.transformers_fa_utils.html | 30 + docs/api/monkeypatch.unsloth_.html | 30 + docs/api/monkeypatch.utils.html | 30 + docs/api/prompt_strategies.alpaca_chat.html | 30 + .../prompt_strategies.alpaca_instruct.html | 30 + .../prompt_strategies.alpaca_w_system.html | 30 + docs/api/prompt_strategies.base.html | 30 + ...rompt_strategies.bradley_terry.llama3.html | 30 + docs/api/prompt_strategies.chat_template.html | 30 + docs/api/prompt_strategies.completion.html | 30 + .../prompt_strategies.dpo.chat_template.html | 30 + docs/api/prompt_strategies.dpo.chatml.html | 30 + docs/api/prompt_strategies.dpo.llama3.html | 30 + .../prompt_strategies.dpo.passthrough.html | 30 + .../prompt_strategies.dpo.user_defined.html | 30 + docs/api/prompt_strategies.dpo.zephyr.html | 30 + docs/api/prompt_strategies.input_output.html | 30 + docs/api/prompt_strategies.kto.chatml.html | 30 + docs/api/prompt_strategies.kto.llama3.html | 30 + .../prompt_strategies.kto.user_defined.html | 30 + docs/api/prompt_strategies.llama2_chat.html | 30 + docs/api/prompt_strategies.messages.chat.html | 30 + docs/api/prompt_strategies.metharme.html | 30 + docs/api/prompt_strategies.orcamini.html | 30 + .../prompt_strategies.orpo.chat_template.html | 30 + docs/api/prompt_strategies.pygmalion.html | 30 + ...prompt_strategies.stepwise_supervised.html | 30 + docs/api/prompt_strategies.user_defined.html | 30 + docs/api/prompt_tokenizers.html | 30 + docs/api/train.html | 30 + docs/api/utils.bench.html | 30 + docs/api/utils.callbacks.comet_.html | 30 + docs/api/utils.callbacks.lisa.html | 30 + docs/api/utils.callbacks.mlflow_.html | 30 + docs/api/utils.callbacks.perplexity.html | 30 + docs/api/utils.callbacks.profiler.html | 30 + docs/api/utils.callbacks.qat.html | 30 + docs/api/utils.chat_templates.html | 30 + docs/api/utils.collators.batching.html | 30 + docs/api/utils.collators.core.html | 30 + docs/api/utils.collators.mamba.html | 30 + docs/api/utils.collators.mm_chat.html | 30 + .../utils.ctx_managers.sequence_parallel.html | 30 + docs/api/utils.data.sft.html | 30 + docs/api/utils.data.streaming.html | 30 + docs/api/utils.dict.html | 30 + docs/api/utils.distributed.html | 30 + docs/api/utils.freeze.html | 30 + docs/api/utils.lora.html | 30 + docs/api/utils.model_shard_quant.html | 30 + docs/api/utils.optimizers.adopt.html | 30 + docs/api/utils.quantization.html | 30 + docs/api/utils.samplers.multipack.html | 30 + docs/api/utils.schedulers.html | 30 + docs/api/utils.schemas.config.html | 30 + docs/api/utils.schemas.datasets.html | 30 + docs/api/utils.schemas.enums.html | 30 + docs/api/utils.schemas.integrations.html | 30 + docs/api/utils.schemas.model.html | 30 + docs/api/utils.schemas.multimodal.html | 30 + docs/api/utils.schemas.peft.html | 30 + docs/api/utils.schemas.training.html | 30 + docs/api/utils.schemas.trl.html | 30 + docs/api/utils.schemas.utils.html | 30 + docs/api/utils.tokenization.html | 30 + docs/api/utils.trainer.html | 30 + docs/attention.html | 30 + docs/batch_vs_grad.html | 30 + docs/checkpoint_saving.html | 30 + docs/choosing_method.html | 1725 +++++++++++++ docs/cli.html | 30 + docs/config-reference.html | 30 + docs/custom_integrations.html | 30 + docs/dataset-formats/conversation.html | 30 + docs/dataset-formats/index.html | 389 +-- docs/dataset-formats/inst_tune.html | 30 + docs/dataset-formats/pretraining.html | 95 +- docs/dataset-formats/stepwise_supervised.html | 30 + docs/dataset-formats/template_free.html | 30 + docs/dataset-formats/tokenized.html | 30 + docs/dataset_loading.html | 30 + docs/dataset_preprocessing.html | 30 + docs/debugging.html | 47 +- docs/docker.html | 30 + docs/ebft.html | 2269 ++++++++++++++++ docs/expert_quantization.html | 30 + docs/faq.html | 30 + docs/fsdp_qlora.html | 30 + docs/getting-started.html | 60 +- docs/gradient_checkpointing.html | 30 + docs/grpo.html | 2296 +++++++++++++++++ docs/inference.html | 30 + docs/input_output.html | 30 + docs/installation.html | 30 + docs/lora_optims.html | 30 + docs/lr_groups.html | 30 + docs/mac.html | 30 + docs/mixed_precision.html | 30 + docs/models/LiquidAI.html | 30 + docs/models/apertus.html | 30 + docs/models/arcee.html | 30 + docs/models/devstral.html | 30 + docs/models/gemma3n.html | 30 + docs/models/gpt-oss.html | 30 + docs/models/granite4.html | 30 + docs/models/hunyuan.html | 30 + docs/models/index.html | 30 + docs/models/internvl3_5.html | 30 + docs/models/jamba.html | 30 + docs/models/kimi-linear.html | 30 + docs/models/llama-2.html | 30 + docs/models/llama-4.html | 30 + docs/models/magistral.html | 30 + docs/models/magistral/think.html | 30 + docs/models/magistral/vision.html | 30 + docs/models/mimo.html | 30 + docs/models/ministral.html | 30 + docs/models/ministral3.html | 30 + docs/models/ministral3/think.html | 30 + docs/models/ministral3/vision.html | 30 + docs/models/mistral-small.html | 30 + docs/models/mistral.html | 30 + docs/models/olmo3.html | 30 + docs/models/orpheus.html | 30 + docs/models/phi.html | 30 + docs/models/plano.html | 30 + docs/models/qwen3-next.html | 30 + docs/models/qwen3.html | 30 + docs/models/seed-oss.html | 30 + docs/models/smolvlm2.html | 30 + docs/models/trinity.html | 30 + docs/models/voxtral.html | 30 + docs/multi-gpu.html | 30 + docs/multi-node.html | 30 + docs/multimodal.html | 30 + docs/multipack.html | 30 + docs/nccl.html | 30 + docs/nd_parallelism.html | 30 + docs/optimizations.html | 30 + docs/optimizers.html | 30 + docs/qat.html | 30 + docs/quantize.html | 30 + docs/ray-integration.html | 30 + docs/reward_modelling.html | 30 + docs/rlhf.html | 52 +- docs/sequence_parallelism.html | 30 + docs/streaming.html | 30 + docs/telemetry.html | 30 + docs/torchao.html | 30 + docs/training_stability.html | 1838 +++++++++++++ docs/unsloth.html | 30 + docs/vllm_serving.html | 1825 +++++++++++++ .../colab-axolotl-example.html | 30 + index.html | 30 + search.json | 1301 ++++++++-- sitemap.xml | 948 +++---- src/axolotl/integrations/LICENSE.html | 30 + .../cut_cross_entropy/ACKNOWLEDGEMENTS.html | 30 + 248 files changed, 25536 insertions(+), 1000 deletions(-) create mode 100644 docs/agents/grpo.html create mode 100644 docs/agents/preference_tuning.html create mode 100644 docs/agents/pretraining.html create mode 100644 docs/agents/reward_modelling.html create mode 100644 docs/agents/sft.html create mode 100644 docs/choosing_method.html create mode 100644 docs/ebft.html create mode 100644 docs/grpo.html create mode 100644 docs/training_stability.html create mode 100644 docs/vllm_serving.html diff --git a/.nojekyll b/.nojekyll index 04adbec1a..c2a64ff12 100644 --- a/.nojekyll +++ b/.nojekyll @@ -1 +1 @@ -f05ef313 \ No newline at end of file +17703de0 \ No newline at end of file diff --git a/FAQS.html b/FAQS.html index d81059f09..82918ae5b 100644 --- a/FAQS.html +++ b/FAQS.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart + +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/agents/grpo.html b/docs/agents/grpo.html new file mode 100644 index 000000000..d282738b7 --- /dev/null +++ b/docs/agents/grpo.html @@ -0,0 +1,1322 @@ + + + + + + + + + +grpo – Axolotl + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    + + +
    + +
    + + +
    + + + +
    + + + + +
    +

    GRPO — Agent Reference

    +

    Online RL with verifiable reward functions. For full config reference, async features, and scaling, see grpo.qmd. For vLLM setup, see vllm_serving.qmd.

    +
    +

    Architecture

    +
    Terminal 1 (GPU 0)                    Terminal 2 (GPU 1)
+┌──────────────────────┐              ┌──────────────────────────────────┐
+│  vLLM Server         │   HTTP       │  Trainer                         │
+│  Serves base model   │◄────────────►│  1. Send prompts to vLLM         │
+│  + LoRA adapter      │  /generate   │  2. Score completions (rewards)  │
+│                      │  /set_lora   │  3. Compute advantages           │
+│  Punica kernels for  │              │  4. PPO-clip gradient update     │
+│  LoRA inference      │              │  5. Sync LoRA weights to vLLM    │
+└──────────────────────┘              └──────────────────────────────────┘
    +
    +
    +

    Components Required

    +
      +
    1. A YAML config with rl: grpo
      2. +
    2. A reward module (Python file with reward functions)
      3. +
    3. A running vLLM server (axolotl vllm-serve config.yaml)
      4. +
    +
    +
    +

    Reward Function Signature

    +
    def my_reward(completions, **kwargs) -> list[float]:
+    # completions[i][0]["content"] = text of i-th completion
+    # **kwargs contains dataset columns not removed by transform
+    return [score_for_each_completion]
    +

    Multiple rewards: reward_funcs: [r1, r2] with reward_weights: [1.0, 0.5].

    +
    +
    +

    Key Async Features

    + +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    FeatureConfigPurpose
    Async prefetchasync_prefetch: trueOverlap generation with training
    LoRA syncvllm_lora_sync: trueFast adapter sync via filesystem
    Streaming scoringstreaming_partial_batch: trueScore one group at a time
    Zero-adv skipskip_zero_advantage_batches: trueSkip batches with no learning signal
    Replay bufferreplay_buffer_size: 100Cache high-signal groups
    IS correctionvllm_importance_sampling_correction: trueFix off-policy distribution shift
    +
    +
    +

    Health Checks

    +
      +
    • rewards/*/mean > 0.15 within 20 steps (else: test reward function standalone)
      • +
    • reward_std > 0 on most steps (else: no learning signal)
      • +
    • entropy 0.05-0.5 (< 0.01 = mode collapse)
      • +
    • grad_norm 0.001-1.0 (> 10 = unstable, 0.0 = zero-advantage skip)
      • +
    +

    See training_stability.qmd for detailed diagnostics.

    +
    +
    +

    File Map

    +
    src/axolotl/
+  cli/train.py                     # Entry point
+  cli/vllm_serve.py                # Entry point for vLLM server
+  core/trainers/grpo/
+    trainer.py                     # AxolotlGRPOTrainer
+    sampler.py                     # Sampling utilities
+  core/builders/rl.py              # HFRLTrainerBuilder — routes rl type → trainer
+  scripts/vllm_serve_lora.py       # vLLM serve script with LoRA sync support
+  utils/schemas/trl.py             # TRL config schema (all trl: options)
+
+docs/grpo.qmd                     # Full user docs: async, rewards, scaling, config reference
+docs/vllm_serving.qmd             # vLLM server modes, LoRA sync, weight sync
    + + +
    +
    + +
    + +
    + + + + + \ No newline at end of file diff --git a/docs/agents/preference_tuning.html b/docs/agents/preference_tuning.html new file mode 100644 index 000000000..f58e36f71 --- /dev/null +++ b/docs/agents/preference_tuning.html @@ -0,0 +1,1449 @@ + + + + + + + + + +preference_tuning – Axolotl + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    + + +
    + +
    + + +
    + + + +
    + + + + +
    +

    Preference Learning (RLHF) — Agent Reference

    +

    Reference for DPO, IPO, KTO, ORPO, and SimPO. For config templates and dataset format examples, see rlhf.qmd. For GRPO, see grpo.qmd. For EBFT, see ebft.qmd.

    +
    +

    Method Overview

    + ++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    MethodData RequirementKey IdeaBest For
    DPOPaired (chosen + rejected)Implicit reward via preference pairsGeneral alignment, most common
    IPOPaired (chosen + rejected)DPO with different loss (avoids overfitting)When DPO overfits
    KTOUnpaired (completion + binary label)Kahneman-Tversky loss, no pairs neededWhen you only have thumbs-up/down
    ORPOPaired (chosen + rejected)Combined SFT + preference, no ref modelSingle-stage alignment, saves VRAM
    SimPOPaired (chosen + rejected)Length-normalized, no ref modelSimple setup, length-robust
    +

    Default: start with DPO. All methods require sample_packing: false.

    +
    +
    +

    Architecture

    +
    ┌──────────────┐   ┌───────────────┐   ┌───────────────┐
+│ Policy Model │   │ Reference     │   │ Preference    │
+│ (trainable)  │   │ Model (frozen)│   │ Dataset       │
+└──────┬───────┘   └──────┬────────┘   └──────┬────────┘
+       └──────────┬───────┘                    │
+                  v                            │
+       Forward pass on chosen + rejected <─────┘
+                  │
+       Preference Loss (DPO/IPO/KTO/...)
+                  │
+       Backprop + Update
+
+Exception: ORPO and SimPO do NOT use a reference model (~50% less VRAM).
    +

    No vLLM server needed (unlike GRPO). Offline RL with pre-collected preference data.

    +
    +
    +

    Method Selection

    +
      +
    1. Paired preference data (chosen + rejected)? +
        +
      • Default → rl: dpo
        • +
      • Overfitting → rl: ipo
        • +
      • VRAM-limited → rl: orpo (no ref model)
        • +
      • Length-sensitive → rl: simpo (no ref model)
        • +
      2. +
    2. Only binary labels (good/bad)? → rl: kto
      3. +
    3. Single-stage training (no separate SFT)? → rl: orpo
      4. +
    + ++++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    DPOIPOKTOORPOSimPO
    Reference modelYesYesYesNoNo
    VRAM overhead~2x model~2x model~2x model~1x model~1x model
    TRL trainer classDPOTrainerDPOTrainerKTOTrainerORPOTrainerCPOTrainer
    +
    +
    +

    Prompt Strategy Resolution

    +

    The type field resolves to a Python function:

    +
    type: "chatml.intel"
+  → axolotl.prompt_strategies.dpo.chatml.intel(cfg, **kwargs)
+  → returns transform_fn(sample) → {"prompt", "chosen", "rejected"}
+
+type: "chat_template.default"
+  → axolotl.prompt_strategies.dpo.chat_template.default(cfg, dataset_idx, **kwargs)
+
+type: {"field_prompt": "prompt", ...}   (dict)
+  → axolotl.prompt_strategies.dpo.user_defined.default(...)
    +

    Module base: axolotl.prompt_strategies.{rl_method} — replace dpo with kto or orpo.

    +
    +
    +

    Healthy Training Indicators

    + +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    MetricHealthy RangeProblem
    train/lossDecreasing, 0.3-0.7Flat or increasing = broken data or too high LR
    rewards/chosenIncreasingFlat = model not learning preferences
    rewards/rejectedDecreasingIncreasing = model prefers wrong responses
    rewards/marginsPositive and increasingNegative = prefers rejected over chosen
    rewards/accuracies> 0.5, toward 0.7+< 0.5 = worse than random
    logps/rejectedDecreasingIncreasing = reward hacking
    grad_norm0.01 - 10.0> 100 = exploding gradients
    +

    Method-specific: DPO/IPO watch rewards/margins; KTO loss is noisier; ORPO monitor SFT + odds ratio components; SimPO check length-normalized reward separation.

    +
    +
    +

    Known Issues

    + ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    IssueFix
    Sample packing crashSet sample_packing: false (required for all preference methods)
    KTO KeyError: 'label'Ensure dataset has boolean label column
    ORPO/KTO KeyError during tokenizationAdd remove_unused_columns: false
    ORPO template not appliedORPO requires explicit chat_template setting
    OOM with ref model (DPO/IPO/KTO)Use LoRA/QLoRA, or switch to ORPO/SimPO (no ref model)
    IPO + label_smoothingDo not set dpo_label_smoothing when rl: ipo
    +

    Full troubleshooting: training_stability.qmd

    +
    +
    +

    File Map

    +
    src/axolotl/
+  core/trainers/dpo/              # DPO trainer, args, strategy
+  core/builders/rl.py             # HFRLTrainerBuilder — routes rl type → trainer class
+  core/training_args.py           # AxolotlKTOConfig, AxolotlORPOConfig, AxolotlCPOConfig
+  prompt_strategies/
+    dpo/                          # DPO/IPO/SimPO dataset strategies
+      chat_template.py            # chat_template.default, chat_template.argilla_chat
+      chatml.py                   # chatml.default/intel/icr/argilla_chat/prompt_pairs/ultra
+      llama3.py                   # llama3 variants (same subtypes as chatml)
+      user_defined.py             # Custom field mapping
+      passthrough.py              # No transform
+    kto/                          # KTO dataset strategies (chatml, llama3, user_defined)
+    orpo/                         # ORPO dataset strategies (chat_template.argilla)
+  utils/schemas/enums.py          # RLType enum (dpo, ipo, kto, orpo, simpo, grpo, gdpo, ebft)
+  utils/schemas/config.py         # All rl/dpo/kto/orpo/simpo config fields
+
+docs/rlhf.qmd                    # Full user docs: all dataset formats, config templates
+docs/choosing_method.qmd          # SFT vs DPO vs GRPO decision guide
+examples/qwen2/dpo.yaml           # DPO example
+examples/llama-3/qlora-1b-kto.yaml  # KTO example
    + + +
    +
    + +
    + +
    + + + + + \ No newline at end of file diff --git a/docs/agents/pretraining.html b/docs/agents/pretraining.html new file mode 100644 index 000000000..600ac92b3 --- /dev/null +++ b/docs/agents/pretraining.html @@ -0,0 +1,1319 @@ + + + + + + + + + +pretraining – Axolotl + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    + + +
    + +
    + + +
    + + + +
    + + + + +
    +

    Pretraining / Continual Pretraining — Agent Reference

    +

    Train on raw text with no input masking. Two approaches depending on dataset size.

    +
    +

    When to Use

    +
      +
    • Continual pretraining on domain-specific corpora
      • +
    • Adapting a base model to a new language or domain before fine-tuning
      • +
    • Pretraining-style data where the entire text is the training signal
      • +
    +
    +
    +

    Choosing an Approach

    + +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Non-streaming (type: completion)Streaming (pretraining_dataset)
    Dataset sizeFits in memoryToo large to fit in memory
    TokenizationPre-tokenized before trainingOn-demand during training
    Config keydatasets:pretraining_dataset:
    Long text handlingSplits texts exceeding sequence_lenConcatenates into fixed-length sequences
    BenefitCan preprocess on CPU, transfer to GPUStart training immediately, no preprocessing
    +
    +
    +

    Non-Streaming: type: completion

    +

    For smaller datasets that fit in memory. Pre-tokenizes the entire dataset.

    +
    datasets:
+  - path: my_corpus
+    type: completion
+    # field: text              # Column name (default: "text")
    +
    +
    +

    Streaming: pretraining_dataset

    +

    For large corpora. Streams data on-demand without loading everything into memory.

    +
    pretraining_dataset:
+  - path: HuggingFaceFW/fineweb-edu
+    type: pretrain
+    text_column: text
+    split: train
+
+max_steps: 1000                          # Required — axolotl can't infer dataset size
+streaming_multipack_buffer_size: 10000   # Buffer for sample packing
+pretrain_multipack_attn: true            # Prevent cross-attention between packed samples
    +

    max_steps is required for streaming — one step = sequence_len * micro_batch_size * gradient_accumulation_steps * num_gpus tokens.

    +

    Full streaming docs: streaming.qmd

    +
    +
    +

    Dataset Format

    +
    {"text": "The complete document text goes here."}
    +
    +
    +

    Key Settings

    +
      +
    • sample_packing: true + pad_to_sequence_len: true — pack documents into fixed-length sequences
      • +
    • flash_attention: true — required for sample packing
      • +
    • No adapter — typically full fine-tune for pretraining
      • +
    • train_on_inputs: true — default for completion (all tokens trained on)
      • +
    +
    +
    +

    File Map

    +
    src/axolotl/
+  prompt_strategies/completion.py    # Non-streaming: completion prompt strategy (no masking)
+  utils/data/sft.py                  # Non-streaming: dataset loading and processing
+  utils/data/streaming.py            # Streaming: encode_streaming(), wrap_streaming_dataset()
+  utils/schemas/config.py            # Config fields: pretraining_dataset, pretrain_multipack_attn, etc.
+
+examples/streaming/pretrain.yaml     # Full streaming pretraining example config
    + + +
    +
    + +
    + +
    + + + + + \ No newline at end of file diff --git a/docs/agents/reward_modelling.html b/docs/agents/reward_modelling.html new file mode 100644 index 000000000..3c761fd5f --- /dev/null +++ b/docs/agents/reward_modelling.html @@ -0,0 +1,1256 @@ + + + + + + + + + +reward_modelling – Axolotl + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    + + +
    + +
    + + +
    + + + +
    + + + + +
    +

    Reward Modelling — Agent Reference

    +

    Train models to score responses for use as reward signals in RL. For full docs, see reward_modelling.qmd.

    +
    +

    Types

    +
    +

    Outcome Reward Models (ORM)

    +

    Train a classifier to predict preference over entire interactions. Uses AutoModelForSequenceClassification.

    +
    base_model: google/gemma-2-2b
+model_type: AutoModelForSequenceClassification
+num_labels: 1
+reward_model: true
+chat_template: gemma
+datasets:
+  - path: argilla/distilabel-intel-orca-dpo-pairs
+    type: bradley_terry.chat_template
    +

    Dataset format: {"system": "...", "input": "...", "chosen": "...", "rejected": "..."}

    +
    +
    +

    Process Reward Models (PRM)

    +

    Train a token classifier to score each reasoning step. Uses AutoModelForTokenClassification.

    +
    base_model: Qwen/Qwen2.5-3B
+model_type: AutoModelForTokenClassification
+num_labels: 2
+process_reward_model: true
+datasets:
+  - path: trl-lib/math_shepherd
+    type: stepwise_supervised
    +

    Dataset format: see stepwise_supervised.qmd.

    +
    +
    +
    +

    File Map

    +
    src/axolotl/
+  core/builders/causal.py                    # Handles reward_model flag in trainer builder
+  prompt_strategies/bradley_terry/           # Bradley-Terry prompt strategies
+  prompt_strategies/stepwise_supervised.py   # PRM dataset strategy
+  utils/schemas/config.py                    # reward_model, process_reward_model config fields
    + + +
    +
    + +
    + +
    + + + + + \ No newline at end of file diff --git a/docs/agents/sft.html b/docs/agents/sft.html new file mode 100644 index 000000000..2cd787f6a --- /dev/null +++ b/docs/agents/sft.html @@ -0,0 +1,1443 @@ + + + + + + + + + +sft – Axolotl + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    + + +
    + +
    + + +
    + + + +
    + + + + +
    +

    SFT — Agent Reference

    +

    Supervised fine-tuning pipeline reference. For config templates and dataset format examples, see getting-started.qmd and dataset-formats/.

    +
    +

    Architecture

    +
    YAML Config → axolotl train config.yaml
+
+  1. Load base model (+ quantization if QLoRA/8-bit)
+  2. Apply adapter layers (LoRA/QLoRA) if configured
+  3. Load + tokenize dataset(s)
+     - Apply prompt template (chat_template / alpaca / custom)
+     - Mask inputs (train_on_inputs: false)
+     - Pack samples into sequences (sample_packing: true)
+  4. Training loop (HuggingFace Trainer)
+     - forward → loss → backward → optimizer step → lr scheduler step
+  5. Save model / adapter weights + tokenizer
+
+Multi-GPU: FSDP or DeepSpeed shards model across GPUs automatically.
    +
    +
    +

    Components Required

    +
      +
    1. A YAML config — model, dataset(s), adapter settings, hyperparameters
      2. +
    2. A dataset — HuggingFace Hub, local JSONL/JSON/Parquet, or S3/GCS path
      3. +
    3. (Optional) A custom prompt strategy — for non-standard dataset formats
      4. +
    +

    No external server processes needed (unlike GRPO which requires vLLM).

    +
    +
    +

    Dataset Format Decision Tree

    +
    Is your data in chat/message format?
+  ├─ YES: OpenAI message format (role/content)?
+  │   ├─ YES ──────────────────────> type: chat_template  (recommended)
+  │   └─ NO (custom field names) ──> type: chat_template + message_property_mappings
+  └─ NO: Instruction/response pairs?
+      ├─ YES ──> type: alpaca       (instruction, input, output)
+      └─ NO: Raw text?
+          ├─ YES with segments ─────> type: input_output  (template-free masking)
+          └─ YES continuous ────────> type: completion     (pretraining-style)
    +

    Full format specs: dataset-formats/

    +
    +
    +

    Model Size to Adapter Choice

    + +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Model SizeLoRAQLoRA (4-bit)Full Fine-TuneVRAM (approx)
    1-3BPreferredLow-budget optionSingle GPU OK8-16 GB (LoRA)
    7-8BPreferredGood balanceNeeds multi-GPU16-24 GB (LoRA)
    13-14BPreferredGood balanceMulti-GPU required24-40 GB (LoRA)
    30-70BLoRA or QLoRAPreferred for single GPUMulti-node40-80 GB (QLoRA)
    +
    +
    +

    Hyperparameter Ranges

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ParameterLoRAQLoRAFull FT
    learning_rate1e-4 to 3e-41e-4 to 3e-41e-5 to 5e-5
    lora_r16-6416-64N/A
    lora_alpha1-2x lora_r1-2x lora_rN/A
    micro_batch_size2-82-41-2
    gradient_accumulation_steps2-84-164-16
    num_epochs1-31-31-3
    optimizeradamw_8bitadamw_bnb_8bitadamw_torch_fused
    +

    Effective batch = micro_batch * grad_accum * num_gpus. Lower LR for larger models.

    +
    +
    +

    Healthy Training Indicators

    + +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    MetricHealthyProblem
    train_lossDecreasing, starting ~2-4 for chat modelsFlat or increasing from step 1 — data or LR issue
    eval_lossDecreasing, tracks train_lossIncreasing while train_loss decreases — overfitting
    grad_norm0.1-10, relatively stableSpikes >100 — instability. 0.0 — frozen weights
    learning_rateFollows scheduler curveFlat or NaN — config issue
    +

    Watch for: loss never decreasing (check train_on_inputs, dataset, LR), loss goes to 0 quickly (overfitting), eval_loss diverging (reduce epochs, add regularization). See training_stability.qmd.

    +
    +
    +

    Known Issues

    + ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    IssueFix
    OOM during trainingReduce micro_batch_size, enable gradient_checkpointing, reduce sequence_len
    sample_packing + SDPA + bf16 = 0.0 lossUse flash_attention: true or disable sample_packing
    Missing chat template errorSet chat_template: chatml explicitly
    Label masking wrongRun axolotl preprocess config.yaml --debug and inspect labels
    Loss NaNUse bf16: auto, lower LR, check data for empty samples
    Tokenizer pad token / infinite lossSet special_tokens: pad_token: "<\|end_of_text\|>"
    FSDP save hangsUse fsdp_state_dict_type: FULL_STATE_DICT
    DeepSpeed CheckpointErrorSet use_reentrant: true in gradient_checkpointing_kwargs
    +

    Full troubleshooting: training_stability.qmd, debugging.qmd

    +
    +
    +

    File Map

    +
    src/axolotl/
+  cli/train.py                     # Entry point for `axolotl train`
+  cli/preprocess.py                # Entry point for `axolotl preprocess`
+  core/builders/causal.py          # HFCausalTrainerBuilder — wires config → SFT trainer
+  core/trainers/base.py            # AxolotlTrainer — base trainer class
+  core/trainers/mixins/            # Packing, optimizer, scheduler, checkpoints
+  prompt_strategies/               # Format handlers: chat_template, alpaca, completion, input_output
+  utils/schemas/config.py          # AxolotlInputConfig — main config schema
+  utils/schemas/datasets.py        # SFTDataset, DatasetConfig
+  utils/schemas/peft.py            # LoraConfig — LoRA parameters
+  integrations/liger/              # Liger kernel plugin
+
+examples/llama-3/                  # LoRA, QLoRA, full FT example configs
+docs/getting-started.qmd           # Quickstart with config templates
+docs/optimizations.qmd             # Flash attention, gradient checkpointing, sample packing
+docs/multi-gpu.qmd                 # FSDP and DeepSpeed setup
    + + +
    +
    + +
    + +
    + + + + + \ No newline at end of file diff --git a/docs/amd_hpc.html b/docs/amd_hpc.html index 0b51799d3..a9b70db6a 100644 --- a/docs/amd_hpc.html +++ b/docs/amd_hpc.html @@ -177,6 +177,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -561,6 +567,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -732,6 +756,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.args.html b/docs/api/cli.args.html index f30fe22a3..774b841ac 100644 --- a/docs/api/cli.args.html +++ b/docs/api/cli.args.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.art.html b/docs/api/cli.art.html index b35548a0a..51ab343fc 100644 --- a/docs/api/cli.art.html +++ b/docs/api/cli.art.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.checks.html b/docs/api/cli.checks.html index 4ec1ac9ad..1e8218f22 100644 --- a/docs/api/cli.checks.html +++ b/docs/api/cli.checks.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.cloud.base.html b/docs/api/cli.cloud.base.html index 06c709fda..fd7c394da 100644 --- a/docs/api/cli.cloud.base.html +++ b/docs/api/cli.cloud.base.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.cloud.modal_.html b/docs/api/cli.cloud.modal_.html index c4b5e7e98..03856d2c1 100644 --- a/docs/api/cli.cloud.modal_.html +++ b/docs/api/cli.cloud.modal_.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.config.html b/docs/api/cli.config.html index 325cf94be..b174d37c3 100644 --- a/docs/api/cli.config.html +++ b/docs/api/cli.config.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.delinearize_llama4.html b/docs/api/cli.delinearize_llama4.html index ad114c568..08177d617 100644 --- a/docs/api/cli.delinearize_llama4.html +++ b/docs/api/cli.delinearize_llama4.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.evaluate.html b/docs/api/cli.evaluate.html index 00f9f9c11..bf0e66595 100644 --- a/docs/api/cli.evaluate.html +++ b/docs/api/cli.evaluate.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.inference.html b/docs/api/cli.inference.html index ea5ebf96f..5718cb3bd 100644 --- a/docs/api/cli.inference.html +++ b/docs/api/cli.inference.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.main.html b/docs/api/cli.main.html index 7def2fc21..edd9603a4 100644 --- a/docs/api/cli.main.html +++ b/docs/api/cli.main.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.merge_lora.html b/docs/api/cli.merge_lora.html index 16916a81e..f164a9f36 100644 --- a/docs/api/cli.merge_lora.html +++ b/docs/api/cli.merge_lora.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.merge_sharded_fsdp_weights.html b/docs/api/cli.merge_sharded_fsdp_weights.html index c7c2a92c8..cf7252525 100644 --- a/docs/api/cli.merge_sharded_fsdp_weights.html +++ b/docs/api/cli.merge_sharded_fsdp_weights.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.preprocess.html b/docs/api/cli.preprocess.html index fb6d0891f..f18809d25 100644 --- a/docs/api/cli.preprocess.html +++ b/docs/api/cli.preprocess.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.quantize.html b/docs/api/cli.quantize.html index 159065010..0ef85e3b8 100644 --- a/docs/api/cli.quantize.html +++ b/docs/api/cli.quantize.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.train.html b/docs/api/cli.train.html index 009f67743..952ea7c94 100644 --- a/docs/api/cli.train.html +++ b/docs/api/cli.train.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.utils.args.html b/docs/api/cli.utils.args.html index 039845830..9dc65b3bd 100644 --- a/docs/api/cli.utils.args.html +++ b/docs/api/cli.utils.args.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.utils.fetch.html b/docs/api/cli.utils.fetch.html index 83381cf4f..cdc76f8f1 100644 --- a/docs/api/cli.utils.fetch.html +++ b/docs/api/cli.utils.fetch.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.utils.html b/docs/api/cli.utils.html index 0d3e2871b..4b037bf49 100644 --- a/docs/api/cli.utils.html +++ b/docs/api/cli.utils.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.utils.load.html b/docs/api/cli.utils.load.html index 2bceb943f..756b3b78f 100644 --- a/docs/api/cli.utils.load.html +++ b/docs/api/cli.utils.load.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.utils.sweeps.html b/docs/api/cli.utils.sweeps.html index 24e573c0e..0940c0f05 100644 --- a/docs/api/cli.utils.sweeps.html +++ b/docs/api/cli.utils.sweeps.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.utils.train.html b/docs/api/cli.utils.train.html index e31ec1f9b..873c7158a 100644 --- a/docs/api/cli.utils.train.html +++ b/docs/api/cli.utils.train.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/cli.vllm_serve.html b/docs/api/cli.vllm_serve.html index d6808a059..4beffb9d7 100644 --- a/docs/api/cli.vllm_serve.html +++ b/docs/api/cli.vllm_serve.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/common.architectures.html b/docs/api/common.architectures.html index b9ef2d18a..e31086856 100644 --- a/docs/api/common.architectures.html +++ b/docs/api/common.architectures.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/common.const.html b/docs/api/common.const.html index 88eccb3b8..86804af05 100644 --- a/docs/api/common.const.html +++ b/docs/api/common.const.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/common.datasets.html b/docs/api/common.datasets.html index a9ceac733..6e67a6e4f 100644 --- a/docs/api/common.datasets.html +++ b/docs/api/common.datasets.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/convert.html b/docs/api/convert.html index 6507f2a08..725d59b0f 100644 --- a/docs/api/convert.html +++ b/docs/api/convert.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.builders.base.html b/docs/api/core.builders.base.html index 4dbb9d664..ad4ec1463 100644 --- a/docs/api/core.builders.base.html +++ b/docs/api/core.builders.base.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.builders.causal.html b/docs/api/core.builders.causal.html index b2cba8347..eb109b2b8 100644 --- a/docs/api/core.builders.causal.html +++ b/docs/api/core.builders.causal.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.builders.rl.html b/docs/api/core.builders.rl.html index 24dd983a0..b0aadffb6 100644 --- a/docs/api/core.builders.rl.html +++ b/docs/api/core.builders.rl.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.chat.format.chatml.html b/docs/api/core.chat.format.chatml.html index 4657e0d06..dd242d41f 100644 --- a/docs/api/core.chat.format.chatml.html +++ b/docs/api/core.chat.format.chatml.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.chat.format.llama3x.html b/docs/api/core.chat.format.llama3x.html index 749772165..e321a5eba 100644 --- a/docs/api/core.chat.format.llama3x.html +++ b/docs/api/core.chat.format.llama3x.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.chat.format.shared.html b/docs/api/core.chat.format.shared.html index 8984f19f4..9c4a0fa23 100644 --- a/docs/api/core.chat.format.shared.html +++ b/docs/api/core.chat.format.shared.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.chat.messages.html b/docs/api/core.chat.messages.html index 46665b6f4..56959f84c 100644 --- a/docs/api/core.chat.messages.html +++ b/docs/api/core.chat.messages.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.datasets.chat.html b/docs/api/core.datasets.chat.html index 48b2d6220..3c5897e29 100644 --- a/docs/api/core.datasets.chat.html +++ b/docs/api/core.datasets.chat.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.datasets.transforms.chat_builder.html b/docs/api/core.datasets.transforms.chat_builder.html index 0f6a39afb..f5e2b43f6 100644 --- a/docs/api/core.datasets.transforms.chat_builder.html +++ b/docs/api/core.datasets.transforms.chat_builder.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.trainers.base.html b/docs/api/core.trainers.base.html index 4743f186a..aba48dd09 100644 --- a/docs/api/core.trainers.base.html +++ b/docs/api/core.trainers.base.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.trainers.dpo.trainer.html b/docs/api/core.trainers.dpo.trainer.html index 83f397428..2d80931a3 100644 --- a/docs/api/core.trainers.dpo.trainer.html +++ b/docs/api/core.trainers.dpo.trainer.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.trainers.grpo.sampler.html b/docs/api/core.trainers.grpo.sampler.html index 1ba9cae69..6acf08070 100644 --- a/docs/api/core.trainers.grpo.sampler.html +++ b/docs/api/core.trainers.grpo.sampler.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.trainers.grpo.trainer.html b/docs/api/core.trainers.grpo.trainer.html index a9a416dde..31319e5b1 100644 --- a/docs/api/core.trainers.grpo.trainer.html +++ b/docs/api/core.trainers.grpo.trainer.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.trainers.mamba.html b/docs/api/core.trainers.mamba.html index 44281443a..07098c59c 100644 --- a/docs/api/core.trainers.mamba.html +++ b/docs/api/core.trainers.mamba.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.trainers.mixins.optimizer.html b/docs/api/core.trainers.mixins.optimizer.html index fb5f43e01..4b0576f99 100644 --- a/docs/api/core.trainers.mixins.optimizer.html +++ b/docs/api/core.trainers.mixins.optimizer.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.trainers.mixins.rng_state_loader.html b/docs/api/core.trainers.mixins.rng_state_loader.html index 14ade33f8..c16ac96a4 100644 --- a/docs/api/core.trainers.mixins.rng_state_loader.html +++ b/docs/api/core.trainers.mixins.rng_state_loader.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.trainers.mixins.scheduler.html b/docs/api/core.trainers.mixins.scheduler.html index de448c30e..ad6e0e4eb 100644 --- a/docs/api/core.trainers.mixins.scheduler.html +++ b/docs/api/core.trainers.mixins.scheduler.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.trainers.trl.html b/docs/api/core.trainers.trl.html index e56345b8b..8d6392964 100644 --- a/docs/api/core.trainers.trl.html +++ b/docs/api/core.trainers.trl.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.trainers.utils.html b/docs/api/core.trainers.utils.html index f2b24d317..c0cfae2d4 100644 --- a/docs/api/core.trainers.utils.html +++ b/docs/api/core.trainers.utils.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/core.training_args.html b/docs/api/core.training_args.html index 5bf6baad2..56a4e2fc2 100644 --- a/docs/api/core.training_args.html +++ b/docs/api/core.training_args.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/datasets.html b/docs/api/datasets.html index 78de36d36..b05e69a47 100644 --- a/docs/api/datasets.html +++ b/docs/api/datasets.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/evaluate.html b/docs/api/evaluate.html index a90b94534..5b2fc0f1a 100644 --- a/docs/api/evaluate.html +++ b/docs/api/evaluate.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/index.html b/docs/api/index.html index 02d34303b..13c0cb75c 100644 --- a/docs/api/index.html +++ b/docs/api/index.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/integrations.base.html b/docs/api/integrations.base.html index 55959e1ef..4c714218a 100644 --- a/docs/api/integrations.base.html +++ b/docs/api/integrations.base.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/integrations.cut_cross_entropy.args.html b/docs/api/integrations.cut_cross_entropy.args.html index 1ee6d1384..2bad49db9 100644 --- a/docs/api/integrations.cut_cross_entropy.args.html +++ b/docs/api/integrations.cut_cross_entropy.args.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/integrations.grokfast.optimizer.html b/docs/api/integrations.grokfast.optimizer.html index e2539242c..e803ef381 100644 --- a/docs/api/integrations.grokfast.optimizer.html +++ b/docs/api/integrations.grokfast.optimizer.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/integrations.kd.trainer.html b/docs/api/integrations.kd.trainer.html index 2fb6bf2e0..6986412fc 100644 --- a/docs/api/integrations.kd.trainer.html +++ b/docs/api/integrations.kd.trainer.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/integrations.liger.args.html b/docs/api/integrations.liger.args.html index c3257f9c9..1fa3b2e79 100644 --- a/docs/api/integrations.liger.args.html +++ b/docs/api/integrations.liger.args.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/integrations.lm_eval.args.html b/docs/api/integrations.lm_eval.args.html index 6119af8bb..8bf090d69 100644 --- a/docs/api/integrations.lm_eval.args.html +++ b/docs/api/integrations.lm_eval.args.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/integrations.spectrum.args.html b/docs/api/integrations.spectrum.args.html index 0b78fd01c..2b3a91256 100644 --- a/docs/api/integrations.spectrum.args.html +++ b/docs/api/integrations.spectrum.args.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/kernels.geglu.html b/docs/api/kernels.geglu.html index 9250d1d20..b29dd0813 100644 --- a/docs/api/kernels.geglu.html +++ b/docs/api/kernels.geglu.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/kernels.lora.html b/docs/api/kernels.lora.html index d7b9940e6..1aaf83d4d 100644 --- a/docs/api/kernels.lora.html +++ b/docs/api/kernels.lora.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/kernels.quantize.html b/docs/api/kernels.quantize.html index d665a1315..2b6f0f0b2 100644 --- a/docs/api/kernels.quantize.html +++ b/docs/api/kernels.quantize.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/kernels.swiglu.html b/docs/api/kernels.swiglu.html index 9c84e177c..ed0c60e50 100644 --- a/docs/api/kernels.swiglu.html +++ b/docs/api/kernels.swiglu.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/kernels.utils.html b/docs/api/kernels.utils.html index 887560ab4..d9d23d3ec 100644 --- a/docs/api/kernels.utils.html +++ b/docs/api/kernels.utils.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/loaders.adapter.html b/docs/api/loaders.adapter.html index c4e37d4a7..0afeb2925 100644 --- a/docs/api/loaders.adapter.html +++ b/docs/api/loaders.adapter.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/loaders.constants.html b/docs/api/loaders.constants.html index 2749a78c4..dd213dc32 100644 --- a/docs/api/loaders.constants.html +++ b/docs/api/loaders.constants.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/loaders.model.html b/docs/api/loaders.model.html index 6e4d66258..ad006919e 100644 --- a/docs/api/loaders.model.html +++ b/docs/api/loaders.model.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/loaders.patch_manager.html b/docs/api/loaders.patch_manager.html index c7518c844..fc42510bc 100644 --- a/docs/api/loaders.patch_manager.html +++ b/docs/api/loaders.patch_manager.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/loaders.processor.html b/docs/api/loaders.processor.html index 401caaa07..ec5278596 100644 --- a/docs/api/loaders.processor.html +++ b/docs/api/loaders.processor.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/loaders.tokenizer.html b/docs/api/loaders.tokenizer.html index 3f6fcaff1..34a0aa602 100644 --- a/docs/api/loaders.tokenizer.html +++ b/docs/api/loaders.tokenizer.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/logging_config.html b/docs/api/logging_config.html index 1b4bdb73c..ea5ee8938 100644 --- a/docs/api/logging_config.html +++ b/docs/api/logging_config.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/models.mamba.modeling_mamba.html b/docs/api/models.mamba.modeling_mamba.html index 138e3e0fe..02a0fc07e 100644 --- a/docs/api/models.mamba.modeling_mamba.html +++ b/docs/api/models.mamba.modeling_mamba.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.btlm_attn_hijack_flash.html b/docs/api/monkeypatch.btlm_attn_hijack_flash.html index 606e87c6d..d868955c9 100644 --- a/docs/api/monkeypatch.btlm_attn_hijack_flash.html +++ b/docs/api/monkeypatch.btlm_attn_hijack_flash.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.data.batch_dataset_fetcher.html b/docs/api/monkeypatch.data.batch_dataset_fetcher.html index 4a19bf777..fd106591a 100644 --- a/docs/api/monkeypatch.data.batch_dataset_fetcher.html +++ b/docs/api/monkeypatch.data.batch_dataset_fetcher.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.gradient_checkpointing.offload_cpu.html b/docs/api/monkeypatch.gradient_checkpointing.offload_cpu.html index dae9540c4..cb04753e6 100644 --- a/docs/api/monkeypatch.gradient_checkpointing.offload_cpu.html +++ b/docs/api/monkeypatch.gradient_checkpointing.offload_cpu.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.gradient_checkpointing.offload_disk.html b/docs/api/monkeypatch.gradient_checkpointing.offload_disk.html index 9138b09d5..323567e35 100644 --- a/docs/api/monkeypatch.gradient_checkpointing.offload_disk.html +++ b/docs/api/monkeypatch.gradient_checkpointing.offload_disk.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.llama_attn_hijack_flash.html b/docs/api/monkeypatch.llama_attn_hijack_flash.html index 0cf998f53..ef1c937c5 100644 --- a/docs/api/monkeypatch.llama_attn_hijack_flash.html +++ b/docs/api/monkeypatch.llama_attn_hijack_flash.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.llama_attn_hijack_xformers.html b/docs/api/monkeypatch.llama_attn_hijack_xformers.html index 922d053ec..0d8053cdf 100644 --- a/docs/api/monkeypatch.llama_attn_hijack_xformers.html +++ b/docs/api/monkeypatch.llama_attn_hijack_xformers.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.lora_kernels.html b/docs/api/monkeypatch.lora_kernels.html index 29c60e437..2872abac9 100644 --- a/docs/api/monkeypatch.lora_kernels.html +++ b/docs/api/monkeypatch.lora_kernels.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.mistral_attn_hijack_flash.html b/docs/api/monkeypatch.mistral_attn_hijack_flash.html index 72fb500a6..a8f627083 100644 --- a/docs/api/monkeypatch.mistral_attn_hijack_flash.html +++ b/docs/api/monkeypatch.mistral_attn_hijack_flash.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.mixtral.html b/docs/api/monkeypatch.mixtral.html index 8f1fbe476..542145d1a 100644 --- a/docs/api/monkeypatch.mixtral.html +++ b/docs/api/monkeypatch.mixtral.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.multipack.html b/docs/api/monkeypatch.multipack.html index 7be88633d..e1a904915 100644 --- a/docs/api/monkeypatch.multipack.html +++ b/docs/api/monkeypatch.multipack.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.relora.html b/docs/api/monkeypatch.relora.html index 6cb3f4006..650ccfe09 100644 --- a/docs/api/monkeypatch.relora.html +++ b/docs/api/monkeypatch.relora.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.stablelm_attn_hijack_flash.html b/docs/api/monkeypatch.stablelm_attn_hijack_flash.html index 6d8476fbc..c8bd40021 100644 --- a/docs/api/monkeypatch.stablelm_attn_hijack_flash.html +++ b/docs/api/monkeypatch.stablelm_attn_hijack_flash.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.trainer_fsdp_optim.html b/docs/api/monkeypatch.trainer_fsdp_optim.html index c6f9e1358..b54ce7d75 100644 --- a/docs/api/monkeypatch.trainer_fsdp_optim.html +++ b/docs/api/monkeypatch.trainer_fsdp_optim.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.transformers_fa_utils.html b/docs/api/monkeypatch.transformers_fa_utils.html index d495a9970..32e8a579f 100644 --- a/docs/api/monkeypatch.transformers_fa_utils.html +++ b/docs/api/monkeypatch.transformers_fa_utils.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.unsloth_.html b/docs/api/monkeypatch.unsloth_.html index c4421a02a..36fe89591 100644 --- a/docs/api/monkeypatch.unsloth_.html +++ b/docs/api/monkeypatch.unsloth_.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/monkeypatch.utils.html b/docs/api/monkeypatch.utils.html index e78a41d98..bb867160d 100644 --- a/docs/api/monkeypatch.utils.html +++ b/docs/api/monkeypatch.utils.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.alpaca_chat.html b/docs/api/prompt_strategies.alpaca_chat.html index 871589def..ced27ec8e 100644 --- a/docs/api/prompt_strategies.alpaca_chat.html +++ b/docs/api/prompt_strategies.alpaca_chat.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.alpaca_instruct.html b/docs/api/prompt_strategies.alpaca_instruct.html index fe43880a8..a407c16e0 100644 --- a/docs/api/prompt_strategies.alpaca_instruct.html +++ b/docs/api/prompt_strategies.alpaca_instruct.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.alpaca_w_system.html b/docs/api/prompt_strategies.alpaca_w_system.html index c6bfa27ae..0c648706d 100644 --- a/docs/api/prompt_strategies.alpaca_w_system.html +++ b/docs/api/prompt_strategies.alpaca_w_system.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.base.html b/docs/api/prompt_strategies.base.html index 9a04a187d..0b3473e0d 100644 --- a/docs/api/prompt_strategies.base.html +++ b/docs/api/prompt_strategies.base.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.bradley_terry.llama3.html b/docs/api/prompt_strategies.bradley_terry.llama3.html index 9ecf15cc4..bf20562fb 100644 --- a/docs/api/prompt_strategies.bradley_terry.llama3.html +++ b/docs/api/prompt_strategies.bradley_terry.llama3.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.chat_template.html b/docs/api/prompt_strategies.chat_template.html index cd364085c..3a9d35562 100644 --- a/docs/api/prompt_strategies.chat_template.html +++ b/docs/api/prompt_strategies.chat_template.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.completion.html b/docs/api/prompt_strategies.completion.html index 95a636634..3de4a38f7 100644 --- a/docs/api/prompt_strategies.completion.html +++ b/docs/api/prompt_strategies.completion.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.dpo.chat_template.html b/docs/api/prompt_strategies.dpo.chat_template.html index bcd2b3494..677dca5b2 100644 --- a/docs/api/prompt_strategies.dpo.chat_template.html +++ b/docs/api/prompt_strategies.dpo.chat_template.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.dpo.chatml.html b/docs/api/prompt_strategies.dpo.chatml.html index 4b296b0f1..1863b3f24 100644 --- a/docs/api/prompt_strategies.dpo.chatml.html +++ b/docs/api/prompt_strategies.dpo.chatml.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.dpo.llama3.html b/docs/api/prompt_strategies.dpo.llama3.html index b016952d4..188ed45cc 100644 --- a/docs/api/prompt_strategies.dpo.llama3.html +++ b/docs/api/prompt_strategies.dpo.llama3.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.dpo.passthrough.html b/docs/api/prompt_strategies.dpo.passthrough.html index 148150b15..3c7b3ad16 100644 --- a/docs/api/prompt_strategies.dpo.passthrough.html +++ b/docs/api/prompt_strategies.dpo.passthrough.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.dpo.user_defined.html b/docs/api/prompt_strategies.dpo.user_defined.html index a3121c684..634135636 100644 --- a/docs/api/prompt_strategies.dpo.user_defined.html +++ b/docs/api/prompt_strategies.dpo.user_defined.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.dpo.zephyr.html b/docs/api/prompt_strategies.dpo.zephyr.html index f889881a7..c850354eb 100644 --- a/docs/api/prompt_strategies.dpo.zephyr.html +++ b/docs/api/prompt_strategies.dpo.zephyr.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.input_output.html b/docs/api/prompt_strategies.input_output.html index f4966f7c9..b3e881814 100644 --- a/docs/api/prompt_strategies.input_output.html +++ b/docs/api/prompt_strategies.input_output.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.kto.chatml.html b/docs/api/prompt_strategies.kto.chatml.html index fddb387bb..d72dabdb9 100644 --- a/docs/api/prompt_strategies.kto.chatml.html +++ b/docs/api/prompt_strategies.kto.chatml.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.kto.llama3.html b/docs/api/prompt_strategies.kto.llama3.html index 3c310d8b4..8699db4f8 100644 --- a/docs/api/prompt_strategies.kto.llama3.html +++ b/docs/api/prompt_strategies.kto.llama3.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.kto.user_defined.html b/docs/api/prompt_strategies.kto.user_defined.html index 36c2f33fb..0ab9b8068 100644 --- a/docs/api/prompt_strategies.kto.user_defined.html +++ b/docs/api/prompt_strategies.kto.user_defined.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.llama2_chat.html b/docs/api/prompt_strategies.llama2_chat.html index 5e65e2ecd..593ed0d5a 100644 --- a/docs/api/prompt_strategies.llama2_chat.html +++ b/docs/api/prompt_strategies.llama2_chat.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.messages.chat.html b/docs/api/prompt_strategies.messages.chat.html index a16151ad7..5f6a7f707 100644 --- a/docs/api/prompt_strategies.messages.chat.html +++ b/docs/api/prompt_strategies.messages.chat.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.metharme.html b/docs/api/prompt_strategies.metharme.html index 6d9fff6f3..8a5ae8434 100644 --- a/docs/api/prompt_strategies.metharme.html +++ b/docs/api/prompt_strategies.metharme.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.orcamini.html b/docs/api/prompt_strategies.orcamini.html index 8a70cb9ea..b4ed89d2d 100644 --- a/docs/api/prompt_strategies.orcamini.html +++ b/docs/api/prompt_strategies.orcamini.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.orpo.chat_template.html b/docs/api/prompt_strategies.orpo.chat_template.html index 6609b4f50..c5a4f3b45 100644 --- a/docs/api/prompt_strategies.orpo.chat_template.html +++ b/docs/api/prompt_strategies.orpo.chat_template.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.pygmalion.html b/docs/api/prompt_strategies.pygmalion.html index de543326f..4dc11bf50 100644 --- a/docs/api/prompt_strategies.pygmalion.html +++ b/docs/api/prompt_strategies.pygmalion.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.stepwise_supervised.html b/docs/api/prompt_strategies.stepwise_supervised.html index 5cc6099bb..5ae86d85c 100644 --- a/docs/api/prompt_strategies.stepwise_supervised.html +++ b/docs/api/prompt_strategies.stepwise_supervised.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_strategies.user_defined.html b/docs/api/prompt_strategies.user_defined.html index 1f89747b4..3aae48e94 100644 --- a/docs/api/prompt_strategies.user_defined.html +++ b/docs/api/prompt_strategies.user_defined.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/prompt_tokenizers.html b/docs/api/prompt_tokenizers.html index 181449473..627faa3f7 100644 --- a/docs/api/prompt_tokenizers.html +++ b/docs/api/prompt_tokenizers.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/train.html b/docs/api/train.html index f849dbd65..aea57da9f 100644 --- a/docs/api/train.html +++ b/docs/api/train.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.bench.html b/docs/api/utils.bench.html index 50b58b3c3..30f67ce0c 100644 --- a/docs/api/utils.bench.html +++ b/docs/api/utils.bench.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.callbacks.comet_.html b/docs/api/utils.callbacks.comet_.html index ac2c34d5b..5f5b9aa5e 100644 --- a/docs/api/utils.callbacks.comet_.html +++ b/docs/api/utils.callbacks.comet_.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.callbacks.lisa.html b/docs/api/utils.callbacks.lisa.html index 3c46f558a..a5724590c 100644 --- a/docs/api/utils.callbacks.lisa.html +++ b/docs/api/utils.callbacks.lisa.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.callbacks.mlflow_.html b/docs/api/utils.callbacks.mlflow_.html index 0139aa88d..b6beb54f6 100644 --- a/docs/api/utils.callbacks.mlflow_.html +++ b/docs/api/utils.callbacks.mlflow_.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.callbacks.perplexity.html b/docs/api/utils.callbacks.perplexity.html index b330014c4..eee82c409 100644 --- a/docs/api/utils.callbacks.perplexity.html +++ b/docs/api/utils.callbacks.perplexity.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.callbacks.profiler.html b/docs/api/utils.callbacks.profiler.html index 43bdde2bc..5077d84a9 100644 --- a/docs/api/utils.callbacks.profiler.html +++ b/docs/api/utils.callbacks.profiler.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.callbacks.qat.html b/docs/api/utils.callbacks.qat.html index 5fdc08f5f..a3a1e225f 100644 --- a/docs/api/utils.callbacks.qat.html +++ b/docs/api/utils.callbacks.qat.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.chat_templates.html b/docs/api/utils.chat_templates.html index 57d3cb1df..22a610513 100644 --- a/docs/api/utils.chat_templates.html +++ b/docs/api/utils.chat_templates.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.collators.batching.html b/docs/api/utils.collators.batching.html index 457b9dcee..64c60749d 100644 --- a/docs/api/utils.collators.batching.html +++ b/docs/api/utils.collators.batching.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.collators.core.html b/docs/api/utils.collators.core.html index 49b1878b4..a3ef27c40 100644 --- a/docs/api/utils.collators.core.html +++ b/docs/api/utils.collators.core.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.collators.mamba.html b/docs/api/utils.collators.mamba.html index 739b29954..08565972a 100644 --- a/docs/api/utils.collators.mamba.html +++ b/docs/api/utils.collators.mamba.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.collators.mm_chat.html b/docs/api/utils.collators.mm_chat.html index a21c5e10f..e751cda6f 100644 --- a/docs/api/utils.collators.mm_chat.html +++ b/docs/api/utils.collators.mm_chat.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.ctx_managers.sequence_parallel.html b/docs/api/utils.ctx_managers.sequence_parallel.html index b536daca0..cbf8e8778 100644 --- a/docs/api/utils.ctx_managers.sequence_parallel.html +++ b/docs/api/utils.ctx_managers.sequence_parallel.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.data.sft.html b/docs/api/utils.data.sft.html index 2f80eee90..8b48104bf 100644 --- a/docs/api/utils.data.sft.html +++ b/docs/api/utils.data.sft.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.data.streaming.html b/docs/api/utils.data.streaming.html index 012852e29..e2b7f7d8d 100644 --- a/docs/api/utils.data.streaming.html +++ b/docs/api/utils.data.streaming.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.dict.html b/docs/api/utils.dict.html index b61b05594..5f40123d8 100644 --- a/docs/api/utils.dict.html +++ b/docs/api/utils.dict.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.distributed.html b/docs/api/utils.distributed.html index ac4e649df..48312fc12 100644 --- a/docs/api/utils.distributed.html +++ b/docs/api/utils.distributed.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.freeze.html b/docs/api/utils.freeze.html index 2b86036e4..6969e6b62 100644 --- a/docs/api/utils.freeze.html +++ b/docs/api/utils.freeze.html @@ -205,6 +205,12 @@ window.Quarto = { Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -589,6 +595,24 @@ window.Quarto = { RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -760,6 +784,12 @@ window.Quarto = { FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.lora.html b/docs/api/utils.lora.html index 9aec1813a..b354adbf2 100644 --- a/docs/api/utils.lora.html +++ b/docs/api/utils.lora.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.model_shard_quant.html b/docs/api/utils.model_shard_quant.html index b7cdc1a7b..6a4011113 100644 --- a/docs/api/utils.model_shard_quant.html +++ b/docs/api/utils.model_shard_quant.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.optimizers.adopt.html b/docs/api/utils.optimizers.adopt.html index 4ba2bedab..82c693760 100644 --- a/docs/api/utils.optimizers.adopt.html +++ b/docs/api/utils.optimizers.adopt.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.quantization.html b/docs/api/utils.quantization.html index 0e93c17bb..64616242e 100644 --- a/docs/api/utils.quantization.html +++ b/docs/api/utils.quantization.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.samplers.multipack.html b/docs/api/utils.samplers.multipack.html index 8fed337ce..7a02f63fc 100644 --- a/docs/api/utils.samplers.multipack.html +++ b/docs/api/utils.samplers.multipack.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.schedulers.html b/docs/api/utils.schedulers.html index 12d4dbc56..7257d957d 100644 --- a/docs/api/utils.schedulers.html +++ b/docs/api/utils.schedulers.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.schemas.config.html b/docs/api/utils.schemas.config.html index 25d7ecd5f..a53eaf6ce 100644 --- a/docs/api/utils.schemas.config.html +++ b/docs/api/utils.schemas.config.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.schemas.datasets.html b/docs/api/utils.schemas.datasets.html index 6d99a419e..bcc225f3b 100644 --- a/docs/api/utils.schemas.datasets.html +++ b/docs/api/utils.schemas.datasets.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.schemas.enums.html b/docs/api/utils.schemas.enums.html index 141523979..ce8b03485 100644 --- a/docs/api/utils.schemas.enums.html +++ b/docs/api/utils.schemas.enums.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.schemas.integrations.html b/docs/api/utils.schemas.integrations.html index f9d4e681a..17f0684f5 100644 --- a/docs/api/utils.schemas.integrations.html +++ b/docs/api/utils.schemas.integrations.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.schemas.model.html b/docs/api/utils.schemas.model.html index 7b3abe1a7..4b1b51645 100644 --- a/docs/api/utils.schemas.model.html +++ b/docs/api/utils.schemas.model.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.schemas.multimodal.html b/docs/api/utils.schemas.multimodal.html index 8fc258717..3ec3ac980 100644 --- a/docs/api/utils.schemas.multimodal.html +++ b/docs/api/utils.schemas.multimodal.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.schemas.peft.html b/docs/api/utils.schemas.peft.html index e86d5aa37..b88f05554 100644 --- a/docs/api/utils.schemas.peft.html +++ b/docs/api/utils.schemas.peft.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.schemas.training.html b/docs/api/utils.schemas.training.html index c98c4d9c1..0b46baae5 100644 --- a/docs/api/utils.schemas.training.html +++ b/docs/api/utils.schemas.training.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.schemas.trl.html b/docs/api/utils.schemas.trl.html index 632bb5e65..13e427827 100644 --- a/docs/api/utils.schemas.trl.html +++ b/docs/api/utils.schemas.trl.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.schemas.utils.html b/docs/api/utils.schemas.utils.html index 1af0096f9..c6553b1b2 100644 --- a/docs/api/utils.schemas.utils.html +++ b/docs/api/utils.schemas.utils.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.tokenization.html b/docs/api/utils.tokenization.html index e8bee2afe..54fb9bf32 100644 --- a/docs/api/utils.tokenization.html +++ b/docs/api/utils.tokenization.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/api/utils.trainer.html b/docs/api/utils.trainer.html index 7bf6389d4..331df27f2 100644 --- a/docs/api/utils.trainer.html +++ b/docs/api/utils.trainer.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/attention.html b/docs/attention.html index ee3a11115..ee2ff0c20 100644 --- a/docs/attention.html +++ b/docs/attention.html @@ -177,6 +177,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -561,6 +567,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -732,6 +756,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/batch_vs_grad.html b/docs/batch_vs_grad.html index 0708ccec9..fbe45875b 100644 --- a/docs/batch_vs_grad.html +++ b/docs/batch_vs_grad.html @@ -142,6 +142,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -526,6 +532,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -697,6 +721,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/checkpoint_saving.html b/docs/checkpoint_saving.html index 4942a2c4d..acb6e95c0 100644 --- a/docs/checkpoint_saving.html +++ b/docs/checkpoint_saving.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/choosing_method.html b/docs/choosing_method.html new file mode 100644 index 000000000..5281a39ef --- /dev/null +++ b/docs/choosing_method.html @@ -0,0 +1,1725 @@ + + + + + + + + + + +Which Fine-Tuning Method Should I Use? – Axolotl + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    + + +
    + +
    + + +
    + + + +
    + +
    +
    +

    Which Fine-Tuning Method Should I Use?

    +
    + +
    +
    + A decision guide for choosing the right fine-tuning method, adapter, and hardware configuration in Axolotl. +
    +
    + + +
    + + + + +
    + + + +
    + + +
    +

    1 Overview

    +

    Axolotl supports four broad categories of fine-tuning, each suited to different data types, objectives, and resource constraints.

    + +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    MethodWhat It DoesData You Need
    Supervised Fine-Tuning (SFT)Teaches the model to produce specific outputs given inputsInput-output pairs (instructions, conversations, completions)
    Preference Learning (DPO/KTO/ORPO)Steers the model toward preferred outputs and away from dispreferred onesChosen/rejected response pairs (DPO, ORPO) or binary labels (KTO)
    Reinforcement Learning (GRPO)Optimizes the model against a reward signal through online generationA reward function (code or model-based) and a prompt dataset
    Reward ModelingTrains a model to score responses, for use as a reward signal in RLPreference pairs ranked by quality
    +

    Each method is configured through a YAML file with rl: <method> (or omitted for SFT). All methods support LoRA, QLoRA, and full fine-tuning unless otherwise noted.

    +
    +
    +

    2 Decision Tree

    +

    Use the following flowchart to choose your method. Start at the top and follow the path that matches your situation.

    +
    Do you have a reward function (code-based or model-based)?
+├── YES
+│   └── Use GRPO (rl: grpo)
+│       The model generates its own completions and learns from reward scores.
+│       Best for: math, code, reasoning, tasks with verifiable answers.
+│       See: rlhf.qmd#grpo
+│
+└── NO
+    │
+    Do you have preference pairs (chosen vs. rejected responses)?
+    ├── YES
+    │   │
+    │   Are they paired (same prompt, one chosen, one rejected)?
+    │   ├── YES → Use DPO (rl: dpo)
+    │   │         Direct optimization without a separate reward model.
+    │   │         See: rlhf.qmd#dpo
+    │   │
+    │   └── NO (only binary good/bad labels)
+    │       └── Use KTO (rl: kto)
+    │           Works with unpaired preference data.
+    │           See: rlhf.qmd#kto
+    │
+    └── NO
+        │
+        Do you have input-output examples?
+        ├── YES → Use SFT
+        │         The simplest and most common method.
+        │         See: getting-started.qmd
+        │
+        └── NO
+            └── You need to create training data first.
+                Consider generating preference pairs with an LLM judge,
+                or writing a reward function for GRPO.
    +
    +
    +
    + +
    +
    +Tip +
    +
    +
    +

    When in doubt, start with SFT. It is the most straightforward method and works well for most tasks. You can always move to preference learning or RL later to further refine behavior.

    +
    +
    +
    +

    2.1 Method Comparison at a Glance

    + +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    CriterionSFTDPOKTOGRPO
    Data complexityLow (input-output pairs)Medium (preference pairs)Medium (binary labels)Low (prompts + reward code)
    Compute costLowMediumMediumHigh (requires vLLM server)
    Learning signalSupervisedContrastiveContrastiveOnline reward
    Online generationNoNoNoYes
    Reward model neededNoNoNoNo (uses reward functions)
    Best forTask adaptation, instruction followingSafety, style alignmentUnpaired preference dataReasoning, math, code
    +
    +
    +
    + +
    +
    +Note +
    +
    +
    +

    ORPO is an alternative to DPO that combines SFT and preference optimization in a single training stage, removing the need for a separate SFT step. Configure with rl: orpo. See rlhf.qmd for details.

    +
    +
    +
    +
    +
    +

    3 Adapter Selection

    +

    Once you have chosen a method, decide how to apply the parameter updates. The three main options trade off VRAM usage against model quality.

    +
    +

    3.1 QLoRA

    +
      +
    • How it works: The base model is loaded in 4-bit (NF4) quantization. Small low-rank adapter matrices are trained in higher precision on top.
      • +
    • VRAM savings: Roughly 4x reduction in model memory compared to full fine-tuning.
      • +
    • Quality: Slight degradation due to quantization noise, but often negligible for task-specific fine-tuning.
      • +
    • When to use: When your GPU cannot fit the model in full precision, or when you want fast experimentation.
      • +
    +
    adapter: qlora
+load_in_4bit: true
+lora_r: 32
+lora_alpha: 64
+lora_target_linear: true
    +
    +
    +

    3.2 LoRA

    +
      +
    • How it works: The base model is loaded at full precision (or 8-bit). Low-rank adapter matrices are trained alongside.
      • +
    • VRAM savings: Roughly 2-3x reduction compared to full fine-tuning (model weights are frozen, only adapters + optimizer states for adapters are stored).
      • +
    • Quality: Very close to full fine-tuning for most tasks, especially with higher rank values.
      • +
    • When to use: When you have enough VRAM for the base model but not for full optimizer states.
      • +
    +
    adapter: lora
+lora_r: 32
+lora_alpha: 64
+lora_target_linear: true
    +
    +
    +
    + +
    +
    +Tip +
    +
    +
    +

    For GRPO training, LoRA is strongly recommended. The vLLM server needs to sync weights from the trainer, and LoRA sync (trl.vllm_lora_sync: true) is far more efficient than syncing full merged weights. See vLLM Serving for details.

    +
    +
    +
    +
    +

    3.3 Full Fine-Tuning

    +
      +
    • How it works: All model parameters are updated during training. No adapters.
      • +
    • VRAM savings: None. Requires memory for model weights, gradients, and optimizer states (roughly 4x model size in bf16 with AdamW).
      • +
    • Quality: Highest potential quality, especially for large distribution shifts.
      • +
    • When to use: When you have ample GPU memory or multi-GPU setups, and need maximum performance. Also required for pre-training.
      • +
    +
    # No adapter or load_in_* lines needed
+micro_batch_size: 1
+gradient_accumulation_steps: 16
    +
    +
    +

    3.4 Quick Comparison

    + ++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    QLoRALoRAFull
    Trainable params~0.1-1%~0.1-1%100%
    Model memory~25% of full~50-100% of full100%
    Optimizer memoryTiny (adapters only)Tiny (adapters only)2x model size (AdamW)
    Training speedSlower (dequantization overhead)BaselineFaster per-step (no adapter overhead)
    InferenceMerge or serve with adapterMerge or serve with adapterDirect
    Multi-GPU required?RarelyFor 13B+ modelsFor 7B+ models
    +
    +
    +
    +

    4 Hardware Mapping

    +

    The tables below provide approximate GPU memory requirements. Actual usage depends on context length, batch size, and optimizer choice.

    +
    +

    4.1 SFT / Preference Learning

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Model SizeQLoRA (4-bit)LoRA (bf16)Full (bf16 + AdamW)
    1-3B6-8 GB8-12 GB24-32 GB
    7-8B10-14 GB16-24 GB60-80 GB
    13-14B16-20 GB28-40 GB120+ GB
    30-34B24-32 GB64-80 GB2-4x 80 GB
    70-72B40-48 GB2x 80 GB4-8x 80 GB
    +
    +
    +
    + +
    +
    +Important +
    +
    +
    +

    These estimates assume a short context length (512-2048 tokens) and micro_batch_size of 1-2. Longer sequences and larger batches increase memory significantly due to activations. Use gradient checkpointing to reduce activation memory at the cost of ~30% slower training.

    +
    +
    +
    +
    +

    4.2 GRPO (RL Training)

    +

    GRPO requires additional GPU(s) for the vLLM generation server. Plan for at least two GPUs: one for training, one for vLLM.

    + ++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Model SizeTraining GPU (LoRA, bf16)vLLM GPUTotal GPUs
    0.5-3B1x 24 GB1x 24 GB2x 24 GB
    7-8B1x 80 GB1x 80 GB2x 80 GB
    13-14B1-2x 80 GB1-2x 80 GB2-4x 80 GB
    30-72B2-4x 80 GB (FSDP/DeepSpeed)2-4x 80 GB (tensor parallel)4-8x 80 GB
    +
    +
    +
    + +
    +
    +Tip +
    +
    +
    +

    For single-GPU GRPO, use vllm_mode: colocate with vllm_enable_sleep_mode: true. The vLLM engine shares the GPU and offloads VRAM when not generating. This works for smaller models (up to ~3B on a 24 GB GPU) but is slower than the two-GPU server mode.

    +
    +
    +
    +
    +

    4.3 Multi-GPU Threshold

    +

    You need multi-GPU training when:

    +
      +
    • Full fine-tuning of models 7B+ (use FSDP or DeepSpeed ZeRO)
      • +
    • LoRA of models 30B+ (or 13B+ with long contexts)
      • +
    • GRPO almost always (separate vLLM server), unless using colocate mode
      • +
    +

    See Multi-GPU Training for FSDP and DeepSpeed configuration.

    +
    +
    + + +
    + +
    + + + + + \ No newline at end of file diff --git a/docs/cli.html b/docs/cli.html index 2bcbd1be5..521e07cc6 100644 --- a/docs/cli.html +++ b/docs/cli.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/config-reference.html b/docs/config-reference.html index f4d4537d1..af9a436b2 100644 --- a/docs/config-reference.html +++ b/docs/config-reference.html @@ -177,6 +177,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -561,6 +567,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -732,6 +756,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/custom_integrations.html b/docs/custom_integrations.html index 97fedf1ef..c7f7b587c 100644 --- a/docs/custom_integrations.html +++ b/docs/custom_integrations.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/dataset-formats/conversation.html b/docs/dataset-formats/conversation.html index eaa0bee9a..b9692bf40 100644 --- a/docs/dataset-formats/conversation.html +++ b/docs/dataset-formats/conversation.html @@ -177,6 +177,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -561,6 +567,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -732,6 +756,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/dataset-formats/index.html b/docs/dataset-formats/index.html index 154ec9738..dba8ddf0b 100644 --- a/docs/dataset-formats/index.html +++ b/docs/dataset-formats/index.html @@ -177,6 +177,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -561,6 +567,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -732,6 +756,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • @@ -759,15 +789,8 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});
  • Instruction Dataset +

    For help choosing between these methods, see Choosing a Fine-Tuning Method.

    RLHF using Axolotl

    @@ -1310,7 +1341,7 @@ Tip
    -

    Check out our GRPO cookbook.

    +

    Check out our GRPO cookbook. For a comprehensive guide covering async training, custom rewards, importance sampling, and scaling, see the GRPO deep dive.

    In the latest GRPO implementation, vLLM is used to significantly speedup trajectory generation during training. In this example, we’re using 4 GPUs - 2 for training, and 2 for vLLM:

    @@ -1683,7 +1714,7 @@ Note CUDA_VISIBLE_DEVICES=0 axolotl vllm-serve config.yaml # Terminal 2: Train on GPUs 0,1 -CUDA_VISIBLE_DEVICES=0,1 accelerate launch --num_processes 2 -m axolotl.cli.train config.yaml     +CUDA_VISIBLE_DEVICES=0,1 axolotl train config.yaml
    @@ -1823,6 +1854,19 @@ Tip

    EBFT

    +
    +
    +
    + +
    +
    +Tip +
    +
    +
    +

    For a detailed guide on EBFT modes, feature extraction, and configuration, see the EBFT guide.

    +
    +

    EBFT (Energy-Based Fine-Tuning) fine-tunes language models by optimizing a feature-matching loss rather than relying on external reward functions. A frozen copy of the model extracts embeddings from both generated and ground-truth completions, and the generator is updated via REINFORCE to match the ground-truth feature moments.

    Paper: “Matching Features, Not Tokens: Energy-Based Fine-Tuning of Language Models” (Jelassi et al., 2026)

    Key advantages:

    diff --git a/docs/sequence_parallelism.html b/docs/sequence_parallelism.html index e998eea73..2a68acebc 100644 --- a/docs/sequence_parallelism.html +++ b/docs/sequence_parallelism.html @@ -177,6 +177,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -561,6 +567,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -732,6 +756,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/streaming.html b/docs/streaming.html index c1cdaca04..78cd1f364 100644 --- a/docs/streaming.html +++ b/docs/streaming.html @@ -177,6 +177,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -561,6 +567,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -732,6 +756,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/telemetry.html b/docs/telemetry.html index 38edc6200..5cc6c242d 100644 --- a/docs/telemetry.html +++ b/docs/telemetry.html @@ -142,6 +142,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -526,6 +532,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -697,6 +721,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/torchao.html b/docs/torchao.html index 82cdcf3c4..0532580e6 100644 --- a/docs/torchao.html +++ b/docs/torchao.html @@ -177,6 +177,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -561,6 +567,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -732,6 +756,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/training_stability.html b/docs/training_stability.html new file mode 100644 index 000000000..378f879c8 --- /dev/null +++ b/docs/training_stability.html @@ -0,0 +1,1838 @@ + + + + + + + + + + +Training Stability & Debugging – Axolotl + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    + + +
    + +
    + + +
    + + + +
    + +
    +
    +

    Training Stability & Debugging

    +
    + +
    +
    + Guide to monitoring, debugging, and stabilizing training runs in axolotl +
    +
    + + +
    + + + + +
    + + + +
    + + +

    This guide covers practical techniques for monitoring training health, diagnosing instability, and resolving common failures in both supervised fine-tuning (SFT) and reinforcement learning (GRPO/EBFT) workflows.

    +
    +

    Monitoring Training

    +
    +

    Key Metrics for SFT

    +

    Every SFT run should be monitored through at least these four metrics:

    + +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    MetricWhat It Tells YouHealthy Range
    train/lossHow well the model fits training dataDecreasing; typically 0.5–2.0 for chat fine-tuning
    eval/lossGeneralization performanceTracks train loss with small gap; divergence signals overfitting
    grad_normGradient magnitude0.1–10.0; spikes above 100 indicate instability
    learning_rateCurrent LR from schedulerShould follow expected schedule (warmup then decay)
    +
    +
    +
    + +
    +
    +TipSet Up Logging Early +
    +
    +
    +

    Enable W&B or TensorBoard from the start. Debugging a failed run without metrics is guesswork.

    +
    wandb_project: my-project
+wandb_run_id:   # optional, for resuming
+logging_steps: 1
    +
    +
    +
    +
    +

    Key Metrics for RL (GRPO)

    +

    GRPO training logs a richer set of metrics. These are the critical ones:

    + +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    MetricHealthy RangeRed Flag
    rewards/<name>/mean> 0.15 within 20 stepsStays at 0 – reward function is broken or task is too hard
    reward_std> 0 on most stepsAlways 0 – no learning signal (all completions get the same reward)
    frac_reward_zero_std< 0.81.0 on every step – zero-advantage skip fires constantly, no gradient updates
    grad_norm0.001–1.00.0 is acceptable occasionally (zero-adv skip); > 10.0 is unstable
    entropy0.05–0.5< 0.01 suggests mode collapse; > 1.0 suggests the model is not converging
    kl0.0–0.5> 2.0 suggests policy has diverged too far from reference
    sampling/sampling_logp_difference/mean< 0.1> 1.0 means policy has diverged far from vLLM server weights
    sampling/importance_sampling_ratio/min> 0.1Near 0 indicates stale off-policy data; increase vllm_sync_interval
    clip_ratio/region_mean< 0.1> 0.3 means PPO clipping is too aggressive
    completions/mean_lengthTask-dependentMonotonically increasing to max length suggests reward hacking
    completions/clipped_ratio< 0.3> 0.8 means most completions hit max_completion_length – increase it
    +
    +
    +
    + +
    +
    +NoteEBFT-Specific Metrics +
    +
    +
    +

    For EBFT training, also monitor ebft/alignment (should trend upward, healthy 0.3–0.9), ebft/diversity (healthy 0.01–0.1; > 1.0 indicates mode collapse), and ebft/cfm_loss (should trend downward, < 10).

    +
    +
    +
    +
    +
    +

    SFT Stability

    +
    +

    Loss Plateau

    +

    Symptom: Loss stops decreasing early in training, well above expected values.

    +

    Causes and fixes:

    +
      +
    • Learning rate too low: Increase by 2–5x. Typical ranges: full fine-tune 1e-5 to 5e-5, LoRA 1e-4 to 3e-4.
      • +
    • Insufficient warmup: Set warmup_steps to 5–10% of total steps. Too-aggressive learning at the start can push the model into a flat region.
      • +
    • Data quality: Check that labels are correctly masked. Use axolotl preprocess and inspect tokenized samples to confirm only the target tokens are trainable.
      • +
    • Weight decay too high: Default 0.01 is usually fine. Values above 0.1 can suppress learning in LoRA.
      • +
    +
    +
    +

    Loss Spikes

    +

    Symptom: Loss suddenly jumps by 2–10x then (possibly) recovers.

    +

    Causes and fixes:

    +
      +
    • Bad data samples: A single malformed or extremely long example can cause a spike. Enable sample_packing: false temporarily and check if spikes correlate with specific batches.
      • +
    • Learning rate too high: Reduce by 2–5x, or increase warmup.
      • +
    • Gradient accumulation mismatch: Effective batch size = micro_batch_size * gradient_accumulation_steps * num_gpus. Very large effective batch sizes amplify gradient noise.
      • +
    • Mixed precision issues: With bf16: true, some operations can lose precision. If spikes are severe, try fp32 for diagnosis.
      • +
    +
    +
    +

    Overfitting

    +

    Symptom: Train loss keeps decreasing but eval loss starts increasing.

    +

    Fixes:

    +
      +
    • Increase val_set_size (e.g., 0.05) and monitor eval/loss.
      • +
    • Reduce num_epochs or max_steps.
      • +
    • Increase weight_decay (try 0.01–0.1).
      • +
    • Use a smaller LoRA rank (lora_r). Typical values: 8–32.
      • +
    • Increase dropout: lora_dropout: 0.05.
      • +
    +
    +
    +
    +

    RL/GRPO Stability

    +
    +

    Reward Never Increases

    +

    If rewards/*/mean stays at 0 for more than 20 steps:

    +
      +

    1. Test reward function standalone: Run it outside training with known inputs to verify it returns nonzero values.

      +
      cd experiments && python -c "import my_rewards; print(my_rewards.accuracy_reward(...))"
      2. +

    2. Check dataset columns: The reward function receives **kwargs containing dataset columns. Verify the columns it needs (e.g., answer) are not removed by the dataset transform.

      3. +

    3. Check completion content: Enable log_completions: true in the trl: config and inspect logged completions in W&B. If completions are empty or incoherent, the model may be too weak for the task.

      4. +

    4. Verify vLLM is serving the right model: Hit the vLLM health endpoint and confirm the model name matches your config.

      5. +
    +
    +
    +

    Entropy Collapse (Mode Collapse)

    +

    Symptom: entropy drops below 0.01; all completions become nearly identical.

    +

    Fixes:

    +
      +
    • Increase temperature in generation kwargs (try 0.8–1.0).
      • +
    • Reduce learning rate.
      • +
    • Add a KL penalty term (beta parameter in GRPO config).
      • +
    • Check that num_generations is sufficient (16+ gives better advantage estimates).
      • +
    +
    +
    +

    IS Ratio Divergence

    +

    Symptom: sampling/importance_sampling_ratio/min drops near 0, or sampling/sampling_logp_difference/mean exceeds 1.0.

    +

    This means the policy has diverged significantly from the weights used by vLLM for generation. The importance sampling correction becomes unreliable.

    +

    Fixes:

    +
      +
    • Decrease vllm_sync_interval (sync weights more often).
      • +
    • Enable off_policy_mask_threshold (e.g., 0.5) to mask stale off-policy samples.
      • +
    • Use importance_sampling_level: token for finer-grained correction.
      • +
    +
    +
    +

    Gradient Norm Instability

    +

    Symptom: grad_norm oscillates wildly or exceeds 10.0 regularly.

    +

    Fixes:

    +
      +
    • Enable gradient clipping: max_grad_norm: 1.0 (default in most configs).
      • +
    • Reduce learning rate.
      • +
    • Increase gradient_accumulation_steps to smooth out noisy batches.
      • +
    • Check for NaN issues (see next section).
      • +
    +
    +
    +
    +

    NaN and Inf Handling

    +
    +

    Common Causes

    + +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    CauseWhere It ManifestsDetection
    FP8 zero-scale divisionForward pass logitsgrad_norm: nan, loss becomes NaN immediately
    Gradient explosionBackward passgrad_norm spikes to inf, then loss goes NaN
    Bad data (empty sequences)Logprob computationNaN in specific batches only
    Numerical overflow in log-softmaxLoss computationLarge negative logprobs cause exp() overflow
    +
    +
    +

    FP8-Specific NaN Issues

    +

    FP8 quantization (fp8: true) can produce NaN when the activation quantization kernel divides by max(abs(x)) / 448. If the input tensor is all zeros (e.g., padding positions), the scale becomes 0, causing division by zero.

    +

    Fixes applied in axolotl:

    +
      +
    • The act_quant_kernel has a zero-guard: s = tl.where(s == 0, 1.0, s).
      • +
    • A safety net nan_to_num(logits, nan=0.0) is applied in _get_per_token_logps_and_entropies.
      • +
    • Embedding padding is zero-padded for FP8 compatibility.
      • +
    +
    +
    +
    + +
    +
    +ImportantAfter Modifying Triton Kernels +
    +
    +
    +

    If you patch any Triton JIT kernel (e.g., the FP8 quantization kernels in transformers), you must clear the Triton cache for changes to take effect:

    +
    rm -rf ~/.triton/cache
    +
    +
    +
    +
    +

    General NaN Debugging Steps

    +
      +

    1. Enable anomaly detection (slow, but pinpoints the source):

      +
      torch.autograd.set_detect_anomaly(True)
      2. +

    2. Check grad_norm: If it goes to NaN, the backward pass is the problem. If loss is NaN but grad_norm was fine on the previous step, the forward pass is the problem.

      3. +

    3. Reduce to single GPU, single batch: Eliminate distributed training variables.

      4. +

    4. Inspect data: Print the batch that triggers NaN. Look for empty sequences, extreme token IDs, or unexpected padding patterns.

      5. +
    +
    +
    +
    +

    OOM Debugging

    +

    Out-of-memory errors are the most common training failure. Use this systematic approach, from least to most disruptive:

    +
    +

    Step 1: Reduce Batch Size

    +

    The single highest-impact change. VRAM scales roughly linearly with batch size.

    +
    micro_batch_size: 1              # Start here
+gradient_accumulation_steps: 16  # Increase to maintain effective batch size
    +

    For GRPO specifically, the logits tensor for policy logprob computation can be very large. batch_size * num_generations * seq_len * vocab_size in bf16. For example, with num_generations: 16 and micro_batch_size: 8, the logits tensor alone is:

    +
    8 * 16 * 2048 * 151936 * 2 bytes = ~75 GB  (way too large)
    +

    Reduce micro_batch_size to 2–4 for GRPO.

    +
    +
    +

    Step 2: Enable Gradient Checkpointing

    +

    Trades compute for memory by recomputing activations during the backward pass instead of storing them.

    +
    gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+  use_reentrant: false     # Recommended default
    +
    +
    +
    + +
    +
    +WarningReentrant Checkpointing Exceptions +
    +
    +
    +

    Some configurations require use_reentrant: true:

    +
      +
    • DeepSpeed ZeRO-3 (non-reentrant causes CheckpointError)
      • +
    • EBFT strided mode with flex_attention
      • +
    +
    +
    +
    +
    +

    Step 3: Use Quantization

    +

    Load the base model in reduced precision:

    +
    # 4-bit QLoRA
+adapter: qlora
+load_in_4bit: true
+
+# 8-bit
+load_in_8bit: true
+
+# FP8 (saves ~50% model VRAM, same compute speed as bf16)
+fp8: true
    +
    +
    +

    Step 4: Reduce Sequence Length

    +
    sequence_len: 1024     # Down from 2048 or 4096
    +

    For GRPO, also reduce max_completion_length. Memory scales quadratically with sequence length when using standard attention.

    +
    +
    +

    Step 5: Use Flash Attention

    +

    Reduces attention memory from O(n^2) to O(n):

    +
    flash_attention: true
    +
    +
    +

    Step 6: Offload with DeepSpeed

    +

    For extreme cases, offload optimizer states or parameters to CPU:

    +
    deepspeed: deepspeed_configs/zero3_bf16.json
    +
    +
    +

    Diagnosing the Specific Culprit

    +

    Use the profiler_steps config option to capture GPU memory snapshots:

    +
    profiler_steps: [1, 2]
    +

    This generates PyTorch profiler traces you can inspect to see exactly which tensor allocation caused the OOM.

    +
    +
    +
    +

    Common Errors

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Error MessageLikely CauseFix
    exitcode: -9System RAM exhaustionReduce dataset size, dataset_num_proc, or number of data workers
    exitcode: -7 (DeepSpeed)DeepSpeed version issuepip install -U deepspeed
    CUDA out of memoryGPU VRAM exhaustionFollow OOM debugging steps above
    RuntimeError: NCCL communicator was abortedGPU communication failureSee NCCL docs; check NCCL_DEBUG=INFO output
    ValueError: Asking to pad but the tokenizer does not have a padding tokenMissing pad tokenAdd special_tokens: { pad_token: "<\|endoftext\|>" } to config
    'DummyOptim' object has no attribute 'step'DeepSpeed on single GPURemove deepspeed: section from config
    unable to load strategy X then None is not callableReward module not importableRun cd experiments && python -c "import my_rewards" to check
    generation_batch_size not divisible by num_generationsmicro_batch_size too smallSet micro_batch_size >= num_generations and make it divisible
    'weight' must be 2-DFSDP1 flattened parametersUse fsdp_version: 2 or skip unwrap_model when FSDP is enabled
    CheckpointError (tensor count mismatch)Non-reentrant checkpointing + ZeRO-3 or flex_attentionSet use_reentrant: true in gradient_checkpointing_kwargs
    BFloat16 TypeError during weight syncNumPy does not support bf16Fixed in axolotl’s weight_serde.py (auto bf16 to fp16 conversion)
    Content end boundary is before start boundaryChat template parsing issueCheck eos_token matches template; file a GitHub issue if persistent
    CAS service error during data processingHuggingFace XET issueSet export HF_HUB_DISABLE_XET=1
    Training hangs (multi-GPU)FSDP + async prefetch deadlockSet async_prefetch: false with FSDP
    +
    +
    +

    Profiling

    +
    +

    PyTorch Profiler

    +

    Axolotl supports PyTorch profiler integration via the config:

    +
    profiler_steps: [1, 2, 3]
    +

    This captures profiler traces for the specified steps. View them in TensorBoard:

    +
    tensorboard --logdir output_dir/runs
    +

    Or open the .json trace file in chrome://tracing.

    +
    +
    +

    CUDA Memory Snapshots

    +

    For detailed memory analysis, use PyTorch’s memory snapshot API. Add this to your training script or use it interactively:

    +
    import torch
+
+# Enable memory history tracking
+torch.cuda.memory._record_memory_history()
+
+# ... run your training step ...
+
+# Save snapshot
+torch.cuda.memory._dump_snapshot("memory_snapshot.pickle")
    +

    Visualize with PyTorch’s memory visualizer:

    +
    python -m torch.cuda.memory._viz memory_snapshot.pickle
    +
    +
    +

    Quick GPU Memory Check

    +

    During training, monitor GPU utilization in a separate terminal:

    +
    watch -n 1 nvidia-smi
    +

    For programmatic access within axolotl, the logged metrics memory/max_alloc and memory/max_reserved come from torch.cuda.max_memory_allocated() and torch.cuda.max_memory_reserved(). Note these report PyTorch’s view of memory, which may differ from nvidia-smi (see FAQ).

    +
    +
    +
    +

    W&B and Logging

    +
    +

    Enabling Logging

    +
    wandb_project: my-project
+wandb_entity: my-team          # optional
+wandb_run_id: run-123          # optional, for resuming
+wandb_name: experiment-name    # optional
+logging_steps: 1               # log every step (recommended for RL)
    +
    +
    +

    Debug Logging

    +

    For detailed axolotl-internal debug output:

    +
    AXOLOTL_LOG_LEVEL=DEBUG axolotl train config.yaml 2>&1 | tee /tmp/training.log
    +
    +
    +
    + +
    +
    +TipAlways Log to a File +
    +
    +
    +

    Pipe training output to a log file so you can inspect it after the run:

    +
    axolotl train config.yaml 2>&1 | tee /tmp/my_run.log
    +
    +
    +
    +
    +

    What Axolotl Logs

    +

    SFT metrics (logged every logging_steps):

    +
      +
    • train/loss, eval/loss – training and validation loss
      • +
    • train/grad_norm – gradient L2 norm (before clipping)
      • +
    • train/learning_rate – current learning rate
      • +
    • memory/max_alloc, memory/max_reserved – peak GPU memory
      • +
    +

    GRPO/RL metrics (logged every step):

    +
      +
    • rewards/<name>/mean, rewards/<name>/std – per-reward-function statistics
      • +
    • reward, reward_std – aggregated reward across all reward functions
      • +
    • frac_reward_zero_std – fraction of prompt groups where all completions got the same reward
      • +
    • completions/mean_length, completions/min_length, completions/max_length – completion token lengths
      • +
    • completions/clipped_ratio – fraction of completions that hit the max length
      • +
    • completions/mean_terminated_length, completions/min_terminated_length, completions/max_terminated_length – lengths of naturally terminated completions
      • +
    • kl – KL divergence between policy and reference
      • +
    • entropy – policy entropy (measure of output diversity)
      • +
    • clip_ratio/region_mean, clip_ratio/low_mean, clip_ratio/high_mean – PPO clipping statistics
      • +
    • sampling/sampling_logp_difference/mean, sampling/sampling_logp_difference/max – log-probability difference between policy and sampling distribution
      • +
    • sampling/importance_sampling_ratio/min, sampling/importance_sampling_ratio/mean, sampling/importance_sampling_ratio/max – IS ratio statistics for off-policy correction
      • +
    • num_tokens – total tokens processed
      • +
    +
    +
    +

    Reading W&B Charts

    +

    For a healthy GRPO run, expect to see:

    +
      +
    1. reward/mean: Gradual upward trend. May start near 0 and reach 0.3–0.8 depending on task difficulty. Not monotonic – fluctuations are normal.
      2. +
    2. entropy: Gradual decrease from initial values (often 0.3–0.6) as the model becomes more confident. Should not collapse to near-zero.
      3. +
    3. grad_norm: Mostly in the 0.001–1.0 range. Occasional 0.0 values are fine (zero-advantage skip). Persistent values above 10.0 need investigation.
      4. +
    4. kl: Starts near 0 and grows slowly. If it shoots up rapidly, the policy is diverging from the reference.
      5. +
    5. completions/mean_length: Should reflect the task’s natural answer length. If it steadily increases to max_completion_length, the model may be reward-hacking by generating longer outputs.
      6. +
    + + +
    +
    + +
    + +
    + + + + + \ No newline at end of file diff --git a/docs/unsloth.html b/docs/unsloth.html index 6f527cb44..55146b371 100644 --- a/docs/unsloth.html +++ b/docs/unsloth.html @@ -177,6 +177,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -561,6 +567,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -732,6 +756,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/docs/vllm_serving.html b/docs/vllm_serving.html new file mode 100644 index 000000000..01b84508a --- /dev/null +++ b/docs/vllm_serving.html @@ -0,0 +1,1825 @@ + + + + + + + + + + +vLLM Serving for GRPO Training – Axolotl + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    + + +
    + +
    + + +
    + + + +
    + +
    +
    +

    vLLM Serving for GRPO Training

    +
    + +
    +
    + How to configure and run vLLM as a generation backend for GRPO reinforcement learning in Axolotl. +
    +
    + + +
    + + + + +
    + + + +
    + + +
    +

    1 Overview

    +

    GRPO (Group Relative Policy Optimization) trains a language model by generating completions, scoring them with reward functions, and updating the policy to favor higher-reward outputs. The generation step is the bottleneck: producing thousands of tokens per training step with the policy model is slow using standard HuggingFace generation.

    +

    Axolotl uses vLLM as a high-throughput generation backend. vLLM runs as a separate process (either on a dedicated GPU or colocated on the training GPU) and serves completions via an HTTP API. The trainer sends prompts to vLLM, receives completions, scores them, and performs gradient updates.

    +
    ┌──────────────────────┐       HTTP        ┌──────────────────────┐
+│   Trainer (GPU 1)    │ ───────────────── │   vLLM Server (GPU 0)│
+│                      │  prompts/compls   │                      │
+│  - Policy model      │ ◄──────────────── │  - Same base model   │
+│  - Reward scoring    │                   │  - Fast generation   │
+│  - Gradient updates  │  weight sync      │  - LoRA adapter      │
+│  - LoRA adapter      │ ─────────────────►│    (periodically     │
+│                      │  (every N steps)  │     updated)         │
+└──────────────────────┘                   └──────────────────────┘
    +
    +
    +
    + +
    +
    +Important +
    +
    +
    +

    vLLM must serve the same base model specified in your training config. If the models do not match, weight synchronization will silently produce incorrect results.

    +
    +
    +
    +
    +

    2 Server Mode

    +

    Server mode runs vLLM as an external process on dedicated GPU(s). This is the recommended configuration for most setups.

    +
    +

    2.1 Starting the Server

    +

    Use the axolotl vllm-serve command with your training config:

    +
    # Terminal 1: Start vLLM on GPU 0
+CUDA_VISIBLE_DEVICES=0 axolotl vllm-serve grpo_config.yaml
    +
    # Terminal 2: Start training on GPU 1
+CUDA_VISIBLE_DEVICES=1 axolotl train grpo_config.yaml
    +

    The server reads vLLM settings from the vllm: section of your config and starts an HTTP server (default: http://0.0.0.0:8000).

    +
    +
    +
    + +
    +
    +Tip +
    +
    +
    +

    Use tmux or screen to manage the vLLM server process. Typical startup time is 30-90 seconds depending on model size and whether CUDA graphs are captured.

    +
    +
    +
    +
    +

    2.2 Minimal Server Config

    +
    base_model: Qwen/Qwen2.5-1.5B-Instruct
+
+vllm:
+  host: 0.0.0.0
+  port: 8000
+  gpu_memory_utilization: 0.85
+  dtype: auto
+  max_model_len: 4096
+
+rl: grpo
+trl:
+  use_vllm: true
+  vllm_server_host: 0.0.0.0
+  vllm_server_port: 8000
+  vllm_server_timeout: 300
    +
    +
    +

    2.3 Multi-GPU vLLM

    +

    For larger models, use tensor parallelism across multiple GPUs:

    +
    vllm:
+  tensor_parallel_size: 2
+  gpu_memory_utilization: 0.85
    +
    # vLLM on GPUs 2,3; training on GPUs 0,1
+CUDA_VISIBLE_DEVICES=2,3 axolotl vllm-serve grpo_config.yaml
+CUDA_VISIBLE_DEVICES=0,1 axolotl train grpo_config.yaml --num-processes 2
    +
    +
    +
    + +
    +
    +Note +
    +
    +
    +

    Due to how TRL maps vLLM device indices, the vLLM instance should use the last N GPUs (highest device indices), while training uses the first N.

    +
    +
    +
    +
    +
    +

    3 Colocate Mode

    +

    Colocate mode runs vLLM on the same GPU as the trainer. This is useful when you only have a single GPU.

    +
    trl:
+  use_vllm: true
+  vllm_mode: colocate
+  vllm_enable_sleep_mode: true
    +

    With vllm_enable_sleep_mode: true, vLLM offloads its VRAM allocation when not actively generating, freeing memory for training. When the trainer needs new completions, vLLM wakes up and reclaims VRAM.

    +
    +
    +
    + +
    +
    +Warning +
    +
    +
    +

    Colocate mode is significantly slower than server mode because generation and training cannot overlap. The GPU alternates between the two workloads. This mode is practical only for smaller models (up to ~3B on a 24 GB GPU).

    +
    +
    +

    When to use colocate mode:

    +
      +
    • You have exactly one GPU
      • +
    • The model fits in memory with both vLLM and training active (with sleep mode), or is small enough to time-share
      • +
    • You accept the performance tradeoff for simpler setup (no separate vLLM process to manage)
      • +
    +

    When to use server mode:

    +
      +
    • You have two or more GPUs
      • +
    • You want maximum throughput (generation overlaps with training via async prefetch)
      • +
    • You are running larger models (7B+)
      • +
    +
    +
    +

    4 LoRA Sync

    +

    LoRA sync is the recommended weight synchronization method when training with LoRA adapters. Instead of merging adapter weights into the base model and broadcasting the full merged weights over NCCL, it saves only the LoRA adapter files to the filesystem and tells vLLM to load them natively.

    +
    +

    4.1 How It Works

    +
      +
    1. The trainer calls model.save_pretrained() to write the LoRA adapter weights to a temporary directory
      2. +
    2. The trainer sends an HTTP POST to /set_lora_adapter/ on the vLLM server
      3. +
    3. vLLM loads the adapter using its native LoRA support (Punica kernels)
      4. +
    4. Generation uses the updated adapter on the next request
      5. +
    +
    +
    +

    4.2 Benefits

    +
      +
    • Smaller sync payload: Transfers ~40 MB of LoRA weights instead of ~1.4 GB+ of merged model weights (for a typical 0.5-3B model)
      • +
    • No NCCL communicator: Eliminates the need for a cross-GPU NCCL communication channel, removing GPU contention between vLLM generation and weight sync
      • +
    • Faster sync: ~200 ms per sync vs. 350 ms to 5+ seconds for NCCL merge sync
      • +
    • Simpler multi-GPU: No need to set up NCCL groups between trainer and vLLM processes
      • +
    +
    +
    +

    4.3 Configuration

    +
    adapter: lora
+lora_r: 32
+lora_alpha: 64
+lora_target_linear: true
+
+trl:
+  vllm_lora_sync: true    # Enables LoRA sync mode
+  vllm_sync_interval: 5   # Sync every 5 training steps
    +

    Setting vllm_lora_sync: true automatically selects the LoRA-aware vLLM serve script (axolotl.scripts.vllm_serve_lora). You do not need to set vllm.serve_module manually.

    +
    +
    +
    + +
    +
    +Important +
    +
    +
    +

    LoRA sync requires that you are training with a LoRA adapter (adapter: lora or adapter: qlora). It is not applicable to full fine-tuning.

    +
    +
    +
    +
    +
    +

    5 Weight Synchronization

    +

    During GRPO training, the policy model on the trainer is continuously updated via gradient steps. The vLLM server, however, still holds the old weights. Periodically, the trainer must push updated weights to vLLM so that future generations reflect the improved policy.

    +
    +

    5.1 Sync Interval

    +

    The vllm_sync_interval parameter controls how often weights are synced:

    +
    trl:
+  vllm_sync_interval: 5   # Sync every 5 optimizer steps
    +

    Tradeoffs:

    +
      +
    • Lower interval (e.g., 1-3): Fresher generations, better on-policy data, but more sync overhead per step
      • +
    • Higher interval (e.g., 5-10): Less overhead, but generations become increasingly off-policy between syncs
      • +
    • Recommended: 3-5 for most setups. Axolotl includes importance sampling correction (vllm_importance_sampling_correction: true) to handle mild distribution mismatch from stale vLLM weights.
      • +
    +
    +
    +

    5.2 Sync Methods

    + +++++++ + + + + + + + + + + + + + + + + + + + + + + + + + +
    MethodConfigPayloadMechanismTypical Time
    LoRA syncvllm_lora_sync: trueLoRA adapter only (~40 MB)Filesystem + HTTP~200 ms
    NCCL merge syncDefault (no lora_sync)Full merged weights (~1.4 GB+)HTTP trigger + NCCL broadcast350 ms - 5 s
    +
    +
    +
    + +
    +
    +Tip +
    +
    +
    +

    If you are training with LoRA (which is recommended for GRPO), always enable vllm_lora_sync: true. The performance difference is substantial, especially as training progresses and NCCL contention increases.

    +
    +
    +
    +
    +

    5.3 Importance Sampling Correction

    +

    When vLLM weights are stale (between syncs), the generated data is slightly off-policy. Axolotl can correct for this:

    +
    trl:
+  vllm_importance_sampling_correction: true
+  importance_sampling_level: token          # 'token' or 'sequence'
+  off_policy_mask_threshold: 0.5            # KL threshold for masking stale sequences
    +
      +
    • Token-level IS is recommended when using Liger kernel (sequence-level has numerical issues with chunked computation)
      • +
    • Off-policy sequence masking (OPSM) drops sequences that have diverged too far from the current policy, providing a safety net against stale data
      • +
    +
    +
    +
    +

    6 Restart Requirements

    +
    +
    +
    + +
    +
    +Warning +
    +
    +
    +

    vLLM must be restarted between training runs. Weight syncs from a previous run leave the server in a corrupted state. If you start a new training run against a stale vLLM server, the model may fail to learn.

    +
    +
    +
    +

    6.1 When to Restart

    +
      +
    • Before every new training experiment
      • +
    • After a training run crashes or is interrupted
      • +
    • If you change the base model in your config
      • +
    +
    +
    +

    6.2 How to Restart

    +

    Killing vLLM reliably requires terminating both the main process and its background EngineCore subprocess:

    +
    # Kill all vLLM-related processes
+pkill -9 -f "vllm|EngineCore"
+
+# Verify GPU memory is freed
+nvidia-smi
+
+# Restart the server
+CUDA_VISIBLE_DEVICES=0 axolotl vllm-serve grpo_config.yaml
    +
    +
    +
    + +
    +
    +Tip +
    +
    +
    +

    A single kill often does not fully stop vLLM. Always use kill -9 and verify with nvidia-smi that GPU memory has been released before restarting.

    +
    +
    +
    +
    +

    6.3 Health Check

    +

    The vLLM server exposes a health endpoint. Wait for it to return 200 before starting training:

    +
    # For the LoRA serve script (trailing slash required)
+curl http://localhost:8000/health/
+
+# For the default TRL serve script
+curl http://localhost:8000/health
    +
    +
    +
    +

    7 Configuration Reference

    +
    +

    7.1 vLLM Server Options (vllm: section)

    +

    These control the vLLM server process started by axolotl vllm-serve.

    + ++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    OptionTypeDefaultDescription
    hoststr0.0.0.0Host address for the vLLM server
    portint8000Port for the vLLM server
    devicestrautoDevice to use for vLLM
    tensor_parallel_sizeintNoneNumber of GPUs for tensor parallelism
    data_parallel_sizeintNoneNumber of data parallel replicas
    gpu_memory_utilizationfloat0.9Fraction of GPU memory for vLLM (0.0-1.0)
    dtypestrautoData type (auto, float16, bfloat16)
    max_model_lenintNoneMaximum model context length. Set explicitly if the default is too large for your GPU
    enable_prefix_cachingboolNoneEnable prefix caching for repeated prompt prefixes
    enable_reasoningboolNoneEnable reasoning mode for models with thinking tokens
    reasoning_parserstrNoneParser for reasoning output
    enforce_eagerboolNoneDisable CUDA graph capture (required for some architectures like Qwen3.5 hybrid attention)
    serve_modulestrNonePython module for vLLM serve script. Auto-set when vllm_lora_sync: true
    worker_extension_clsstrNonevLLM worker extension class for weight sync
    +
    +
    +

    7.2 Trainer vLLM Options (trl: section)

    +

    These control how the trainer interacts with vLLM.

    + ++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    OptionTypeDefaultDescription
    use_vllmboolfalseEnable vLLM for generation
    vllm_modestrNoneserver (external process) or colocate (same GPU)
    vllm_server_hoststr0.0.0.0Host of the vLLM server to connect to
    vllm_server_portint8000Port of the vLLM server to connect to
    vllm_server_timeoutintNoneTimeout in seconds for vLLM requests
    vllm_lora_syncboolfalseSync LoRA adapters via filesystem instead of NCCL merge
    vllm_sync_intervalintNoneSync weights every N optimizer steps
    vllm_enable_sleep_modeboolNoneOffload vLLM VRAM when idle (colocate mode)
    vllm_guided_decoding_regexstrNoneRegex constraint for guided decoding
    +

    For async pipeline and off-policy correction options, see the GRPO Configuration Reference.

    +
    +
    +
    +

    8 Complete Example

    +

    For a full working GRPO config including vLLM, LoRA sync, async generation, rewards, and dataset setup, see the GRPO Quick Start. That config includes all the vLLM settings covered in this guide.

    +
    # Terminal 1: Start vLLM
+CUDA_VISIBLE_DEVICES=0 axolotl vllm-serve grpo_config.yaml
+
+# Wait for health check to pass
+curl http://localhost:8000/health/
+
+# Terminal 2: Start training
+CUDA_VISIBLE_DEVICES=1 axolotl train grpo_config.yaml
    +
    +
    +

    9 Troubleshooting

    + +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ProblemLikely CauseSolution
    Training hangs waiting for vLLMServer not started or wrong portCheck curl http://localhost:8000/health/ and verify vllm_server_host/vllm_server_port match
    OOM on vLLM GPUgpu_memory_utilization too high or max_model_len too largeReduce gpu_memory_utilization to 0.7 or set max_model_len explicitly
    OOM on training GPUBatch too large for policy logprobsReduce micro_batch_size or num_generations
    Accuracy stays at zeroStale vLLM from previous runRestart vLLM: pkill -9 -f "vllm\|EngineCore", verify with nvidia-smi, restart
    ResponseValidationError from vLLMMissing logprobs in responseEnsure you are using the correct serve module (auto-selected with vllm_lora_sync: true)
    Weight sync takes 5+ secondsNCCL contention with vLLM generationSwitch to vllm_lora_sync: true to eliminate NCCL
    async_prefetch deadlocks with FSDPBackground threads run unsynchronized FSDP collectivesSet async_prefetch: false when using FSDP or DeepSpeed multi-GPU
    + + +
    + +
    + +
    + + + + + \ No newline at end of file diff --git a/examples/colab-notebooks/colab-axolotl-example.html b/examples/colab-notebooks/colab-axolotl-example.html index 361049cca..0c77e2b27 100644 --- a/examples/colab-notebooks/colab-axolotl-example.html +++ b/examples/colab-notebooks/colab-axolotl-example.html @@ -180,6 +180,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -564,6 +570,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -735,6 +759,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/index.html b/index.html index 2b56ccb84..d6c860000 100644 --- a/index.html +++ b/index.html @@ -176,6 +176,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -560,6 +566,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -731,6 +755,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/search.json b/search.json index 8eff88459..3d5d5fa91 100644 --- a/search.json +++ b/search.json @@ -88,7 +88,7 @@ "href": "docs/dataset-formats/pretraining.html", "title": "Pre-training", "section": "", - "text": "For pretraining, there is no prompt template or roles. The only required field is text:\n\n\ndata.jsonl\n\n{\"text\": \"first row\"}\n{\"text\": \"second row\"}\n...\n\n\n\n\n\n\n\nNoteStreaming is recommended for large datasets\n\n\n\nAxolotl usually loads the entire dataset into memory. This will be challenging for large datasets. Use the following config to enable streaming:\n\n\nconfig.yaml\n\npretraining_dataset:\n - name:\n path:\n split:\n text_column: # column in dataset with the data, usually `text`\n type: pretrain\n trust_remote_code:\n skip: # number of rows of data to skip over from the beginning", + "text": "Note\n\n\n\nPre-training documentation has been consolidated:\n\nStreaming pretraining (large datasets): See Streaming Datasets\nNon-streaming pretraining (type: completion): See Dataset Formats", "crumbs": [ "Dataset Formats", "Pre-training" @@ -109,7 +109,7 @@ "href": "docs/dataset-formats/index.html#pre-training", "title": "Dataset Formats", "section": "Pre-training", - "text": "Pre-training\nWhen aiming to train on large corpora of text datasets, pre-training is your go-to choice. Due to the size of these datasets, downloading the entire-datasets before beginning training would be prohibitively time-consuming. Axolotl supports streaming to only load batches into memory at a time.\nA sample format for a pre-training dataset is as follows:\n{\"text\": \"first row\"}\n{\"text\": \"second row\"}\n...\nIt is typically recommended to save your dataset as .jsonl due to its flexibility and simplicity.\nAxolotl supports loading from a Hugging Face hub repo or from local files.\n\nPre-training from Hugging Face hub datasets\nAs an example, to train using a Hugging Face dataset hf_org/name, you can pass the following config:\npretraining_dataset: hf_org/name\n\n\nPre-training from local dataset files\nGiven a few corpus files: A.jsonl, B.jsonl, and C.jsonl, your config will look like the below:\npretraining_dataset:\n - path: json\n data_files:\n - A.jsonl\n - B.jsonl\n - C.jsonl\nWhile we recommend .jsonl, you can also use the other formats (csv, parquet, arrow, SQL, Webdataset) that are supported by Dataset.load_dataset\n\n\nPre-training without streaming\nIn the case that the dataset is small and can be loaded entirely into memory, another approach to running pre-training is to use the completion format. This would mean that the entire dataset is pre-tokenized instead of on-demand in streaming.\nOne benefit of this is that the tokenization can be performed separately on a CPU-only machine, and then transferred to a GPU machine for training to save costs.\nFrom Hugging Face:\ndatasets:\n - path: hf_org/name\n type: completion\nFrom local files:\ndatasets:\n - path: A.jsonl\n type: completion\n\n - path: B.jsonl\n type: completion\n\n\n\n\n\n\nImportant\n\n\n\nFor completion only, Axolotl would split texts if it exceeds the context length into multiple smaller prompts. If you are interested in having this for pretraining_dataset too, please let us know or help make a PR!\n\n\n\n\nPre-training dataset configuration tips\n\nSetting max_steps\nWhen using streaming for large datasets, Axolotl does not know in advance how large the dataset is and does not know when to stop.\nTherefore, it is necessary to set max_steps: int in your config for pre-training to run, so that Axolotl knows when to stop training.\nOne step is equal to sequence_len * micro_batch_size * gradient_accumulation_steps * total_num_gpus tokens.\n\n\nGroup_by_length\nIt is recommended to leave this off if downloading from Hugging Face hub as it would download the entire dataset which can be very large.\n\n\n\nReference\nPlease see docs here.", + "text": "Pre-training\nPre-training trains on raw text corpora with no input masking. The dataset format is simple:\n{\"text\": \"first row\"}\n{\"text\": \"second row\"}\nAxolotl supports two approaches:\n\nStreaming (large datasets)\nFor large corpora that don’t fit in memory, use pretraining_dataset with streaming. Data is tokenized on-demand during training.\npretraining_dataset:\n - path: HuggingFaceFW/fineweb-edu\n type: pretrain\n text_column: text\n split: train\n\n\n\n\n\n\nImportant\n\n\n\nStreaming requires max_steps in your config — Axolotl cannot infer the dataset size. One step = sequence_len * micro_batch_size * gradient_accumulation_steps * num_gpus tokens.\n\n\nSee Streaming Datasets for full configuration details.\n\n\nNon-streaming (smaller datasets)\nFor datasets that fit in memory, use type: completion under datasets:. The entire dataset is pre-tokenized before training, which can be done on a CPU-only machine.\ndatasets:\n - path: my_corpus\n type: completion\n\n\n\n\n\n\nNote\n\n\n\nWith completion, texts exceeding sequence_len are split into multiple samples automatically.", "crumbs": [ "Dataset Formats" ] @@ -1237,70 +1237,214 @@ ] }, { - "objectID": "docs/optimizations.html", - "href": "docs/optimizations.html", - "title": "Optimizations Guide", + "objectID": "docs/checkpoint_saving.html", + "href": "docs/checkpoint_saving.html", + "title": "Checkpoint Saving", "section": "", - "text": "Axolotl includes numerous optimizations to speed up training, reduce memory usage, and handle large models.\nThis guide provides a high-level overview and directs you to the detailed documentation for each feature.", - "crumbs": [ - "How To Guides", - "Optimizations Guide" - ] + "text": "Axolotl supports on-demand checkpoint saving during training. You can trigger checkpoints via file-based triggers (for programmatic control) or Control+C (for interactive use)." }, { - "objectID": "docs/optimizations.html#speed-optimizations", - "href": "docs/optimizations.html#speed-optimizations", - "title": "Optimizations Guide", - "section": "Speed Optimizations", - "text": "Speed Optimizations\nThese optimizations focus on increasing training throughput and reducing total training time.\n\nSample Packing\nImproves GPU utilization by combining multiple short sequences into a single packed sequence for training. This requires enabling one of the attention implementations below.\n\nConfig: sample_packing: true\nLearn more: Sample Packing\n\n\n\nAttention Implementations\nUsing an optimized attention implementation is critical for training speed.\n\nFlash Attention 2: flash_attention: true. (Recommended) The industry standard for fast attention on modern GPUs. Requires Ampere or higher. For AMD, check AMD Support.\nFlex Attention: flex_attention: true.\nSDP Attention: sdp_attention: true. PyTorch’s native implementation.\nXformers: xformers_attention: true. Works with FP16.\n\nNote: You should only enable one attention backend.\n\n\nLoRA Optimizations\nLeverages optimized kernels to accelerate LoRA training and reduce memory usage.\n\nLearn more: LoRA Optimizations Documentation", - "crumbs": [ - "How To Guides", - "Optimizations Guide" - ] + "objectID": "docs/checkpoint_saving.html#overview", + "href": "docs/checkpoint_saving.html#overview", + "title": "Checkpoint Saving", + "section": "", + "text": "Axolotl supports on-demand checkpoint saving during training. You can trigger checkpoints via file-based triggers (for programmatic control) or Control+C (for interactive use)." }, { - "objectID": "docs/optimizations.html#memory-optimizations", - "href": "docs/optimizations.html#memory-optimizations", - "title": "Optimizations Guide", - "section": "Memory Optimizations", - "text": "Memory Optimizations\nThese techniques help you fit larger models or use bigger batch sizes on your existing hardware.\n\nParameter Efficient Finetuning (LoRA & QLoRA)\nDrastically reduces memory by training a small set of “adapter” parameters instead of the full model. This is the most common and effective memory-saving technique.\n\nExamples: Find configs with lora or qlora in the examples directory.\nConfig Reference: See adapter, load_in_4bit, and load_in_8bit in the Configuration Reference.\n\n\n\nGradient Checkpointing & Activation Offloading\nThese techniques save VRAM by changing how activations are handled.\n\nGradient Checkpointing: re-computes activations during the backward pass, trading compute time for VRAM.\nActivation Offloading: moves activations to CPU RAM or disk, trading I/O overhead for VRAM.\nLearn more: Gradient Checkpointing and Offloading Docs\n\n\n\nLayer Offloading\nOffloads frozen (non-trainable) decoder layer parameters to CPU and streams them back to GPU one layer at a time during forward/backward passes using CUDA stream prefetching. Especially effective for LoRA/QLoRA where most parameters are frozen.\n\nConfig: layer_offloading: true\nLearn more: Layer Offloading Docs\n\n\n\nCut Cross Entropy (CCE)\nReduces VRAM usage by using an optimized cross-entropy loss calculation.\n\nLearn more: Custom Integrations - CCE\n\n\n\nLiger Kernels\nProvides efficient Triton kernels to improve training speed and reduce memory usage.\n\nLearn more: Custom Integrations - Liger Kernels\n\n\n\nExpert Kernels\nOptimized kernel implementations for Mixture of Experts (MoE) model training.\n\nScatterMoE: Triton-based MoE kernels with fused LoRA support.\nSonicMoE: CUTLASS-based MoE kernels for NVIDIA Hopper and Blackwell GPUs.\nLearn more: Custom Integrations - Kernels Integration", - "crumbs": [ - "How To Guides", - "Optimizations Guide" - ] + "objectID": "docs/checkpoint_saving.html#file-based-checkpoint-trigger", + "href": "docs/checkpoint_saving.html#file-based-checkpoint-trigger", + "title": "Checkpoint Saving", + "section": "2 File-Based Checkpoint Trigger", + "text": "2 File-Based Checkpoint Trigger\n\n2.1 Configuration\nEnable in your config:\ndynamic_checkpoint:\n enabled: true\n check_interval: 100 # Optional: check every N steps (default: 100)\n trigger_file_path: \"axolotl_checkpoint.save\" # Optional: custom filename\nOptions:\n- enabled: true to enable (required)\n- check_interval: Steps between file checks. Default: 100. Lower = faster response, higher I/O overhead.\n- trigger_file_path: Custom trigger filename. Default: axolotl_checkpoint.save\n\n\n2.2 How It Works\n\nRank 0 checks for trigger file every check_interval steps in output_dir\nWhen detected, file is deleted and checkpoint is saved\nIn distributed training, rank 0 broadcasts to synchronize all ranks\n\n\n\n2.3 Usage\nCommand line:\ntouch /path/to/output_dir/axolotl_checkpoint.save\nProgrammatic:\nfrom pathlib import Path\nPath(\"/path/to/output_dir/axolotl_checkpoint.save\").touch()\nCheckpoint saves within the next check_interval steps. The trigger file is auto-deleted after detection, so you can create it multiple times.\nCustom filename:\ndynamic_checkpoint:\n enabled: true\n trigger_file_path: \"my_trigger.save\"\ntouch /path/to/output_dir/my_trigger.save" }, { - "objectID": "docs/optimizations.html#long-context-models", - "href": "docs/optimizations.html#long-context-models", - "title": "Optimizations Guide", - "section": "Long Context Models", - "text": "Long Context Models\nTechniques to train models on sequences longer than their original context window.\n\nRoPE Scaling\nExtends a model’s context window by interpolating its Rotary Position Embeddings.\n\nConfig: Pass the rope_scaling config under the overrides_of_model_config:. To learn how to set RoPE, check the respective model config.\n\n\n\nSequence Parallelism\nSplits long sequences across multiple GPUs, enabling training with sequence lengths that would not fit on a single device.\n\nLearn more: Sequence Parallelism Documentation\n\n\n\nArtic Long Sequence Training (ALST)\nALST is a recipe that combines several techniques to train long-context models efficiently. It typically involves:\n\nTiledMLP to reduce memory usage in MLP layers.\nTiled Loss functions (like CCE.\nActivation Offloading to CPU.\nExample: ALST Example Configuration", - "crumbs": [ - "How To Guides", - "Optimizations Guide" - ] + "objectID": "docs/checkpoint_saving.html#controlc-sigint-checkpoint", + "href": "docs/checkpoint_saving.html#controlc-sigint-checkpoint", + "title": "Checkpoint Saving", + "section": "3 Control+C (SIGINT) Checkpoint", + "text": "3 Control+C (SIGINT) Checkpoint\nPressing Ctrl+C during training saves the model state and exits gracefully. Note: This saves only the model weights, not optimizer state. For resumable checkpoints, use the file-based trigger." }, { - "objectID": "docs/optimizations.html#large-models-distributed-training", - "href": "docs/optimizations.html#large-models-distributed-training", - "title": "Optimizations Guide", - "section": "Large Models (Distributed Training)", - "text": "Large Models (Distributed Training)\nTo train models that don’t fit on a single GPU, you’ll need to use a distributed training strategy like FSDP or DeepSpeed. These frameworks shard the model weights, gradients, and optimizer states across multiple GPUs and nodes.\n\nLearn more: Multi-GPU Guide\nLearn more: Multi-Node Guide\n\n\nN-D Parallelism (Beta)\nFor advanced scaling, Axolotl allows you to compose different parallelism techniques (e.g., Data, Tensor, Sequence Parallelism). This is a powerful approach to train an extremely large model by overcoming multiple bottlenecks at once.\n\nLearn more: N-D Parallelism Guide", - "crumbs": [ - "How To Guides", - "Optimizations Guide" - ] + "objectID": "docs/checkpoint_saving.html#best-practices", + "href": "docs/checkpoint_saving.html#best-practices", + "title": "Checkpoint Saving", + "section": "4 Best Practices", + "text": "4 Best Practices\n\nCheck interval: Lower values (10-50) for fast training, default 100 for slower training\nDistributed training: Create trigger file once; rank 0 handles synchronization\nResume: Dynamic checkpoints can be resumed like regular checkpoints via resume_from_checkpoint" }, { - "objectID": "docs/optimizations.html#quantization", - "href": "docs/optimizations.html#quantization", - "title": "Optimizations Guide", - "section": "Quantization", - "text": "Quantization\nTechniques to reduce the precision of model weights for memory savings.\n\n4-bit Training (QLoRA)\nThe recommended approach for quantization-based training. It loads the base model in 4-bit using bitsandbytes and then trains QLoRA adapters. See Adapter Finetuning for details.\n\n\nFP8 Training\nEnables training with 8-bit floating point precision on supported hardware (e.g., NVIDIA Hopper series GPUs) for significant speed and memory gains.\n\nExample: Llama 3 FP8 FSDP Example\n\n\n\nQuantization Aware Training (QAT)\nSimulates quantization effects during training, helping the model adapt and potentially improving the final accuracy of the quantized model.\n\nLearn more: QAT Documentation\n\n\n\nGPTQ\nAllows you to finetune LoRA adapters on top of a model that has already been quantized using the GPTQ method.\n\nExample: GPTQ LoRA Example\n\n\n\nMoE Expert Quantization\nQuantizes MoE expert weights on load to reduce VRAM when training MoE models with adapters. Required for Transformers v5+ MoE models where experts use fused nn.Parameter tensors.\n\nConfig: quantize_moe_experts: true\nLearn more: MoE Expert Quantization", - "crumbs": [ - "How To Guides", - "Optimizations Guide" - ] + "objectID": "docs/checkpoint_saving.html#example", + "href": "docs/checkpoint_saving.html#example", + "title": "Checkpoint Saving", + "section": "5 Example", + "text": "5 Example\noutput_dir: ./outputs/lora-out\nsave_steps: 500 # Scheduled checkpoints\n\ndynamic_checkpoint:\n enabled: true\n check_interval: 50\nThis enables scheduled checkpoints every 500 steps plus on-demand saves via file trigger (checked every 50 steps)." + }, + { + "objectID": "docs/agents/pretraining.html", + "href": "docs/agents/pretraining.html", + "title": "Pretraining / Continual Pretraining — Agent Reference", + "section": "", + "text": "Train on raw text with no input masking. Two approaches depending on dataset size.\n\n\n\nContinual pretraining on domain-specific corpora\nAdapting a base model to a new language or domain before fine-tuning\nPretraining-style data where the entire text is the training signal\n\n\n\n\n\n\n\n\n\n\n\n\n\nNon-streaming (type: completion)\nStreaming (pretraining_dataset)\n\n\n\n\nDataset size\nFits in memory\nToo large to fit in memory\n\n\nTokenization\nPre-tokenized before training\nOn-demand during training\n\n\nConfig key\ndatasets:\npretraining_dataset:\n\n\nLong text handling\nSplits texts exceeding sequence_len\nConcatenates into fixed-length sequences\n\n\nBenefit\nCan preprocess on CPU, transfer to GPU\nStart training immediately, no preprocessing\n\n\n\n\n\n\nFor smaller datasets that fit in memory. Pre-tokenizes the entire dataset.\ndatasets:\n - path: my_corpus\n type: completion\n # field: text # Column name (default: \"text\")\n\n\n\nFor large corpora. Streams data on-demand without loading everything into memory.\npretraining_dataset:\n - path: HuggingFaceFW/fineweb-edu\n type: pretrain\n text_column: text\n split: train\n\nmax_steps: 1000 # Required — axolotl can't infer dataset size\nstreaming_multipack_buffer_size: 10000 # Buffer for sample packing\npretrain_multipack_attn: true # Prevent cross-attention between packed samples\nmax_steps is required for streaming — one step = sequence_len * micro_batch_size * gradient_accumulation_steps * num_gpus tokens.\nFull streaming docs: streaming.qmd\n\n\n\n{\"text\": \"The complete document text goes here.\"}\n\n\n\n\nsample_packing: true + pad_to_sequence_len: true — pack documents into fixed-length sequences\nflash_attention: true — required for sample packing\nNo adapter — typically full fine-tune for pretraining\ntrain_on_inputs: true — default for completion (all tokens trained on)\n\n\n\n\nsrc/axolotl/\n prompt_strategies/completion.py # Non-streaming: completion prompt strategy (no masking)\n utils/data/sft.py # Non-streaming: dataset loading and processing\n utils/data/streaming.py # Streaming: encode_streaming(), wrap_streaming_dataset()\n utils/schemas/config.py # Config fields: pretraining_dataset, pretrain_multipack_attn, etc.\n\nexamples/streaming/pretrain.yaml # Full streaming pretraining example config" + }, + { + "objectID": "docs/agents/pretraining.html#when-to-use", + "href": "docs/agents/pretraining.html#when-to-use", + "title": "Pretraining / Continual Pretraining — Agent Reference", + "section": "", + "text": "Continual pretraining on domain-specific corpora\nAdapting a base model to a new language or domain before fine-tuning\nPretraining-style data where the entire text is the training signal" + }, + { + "objectID": "docs/agents/pretraining.html#choosing-an-approach", + "href": "docs/agents/pretraining.html#choosing-an-approach", + "title": "Pretraining / Continual Pretraining — Agent Reference", + "section": "", + "text": "Non-streaming (type: completion)\nStreaming (pretraining_dataset)\n\n\n\n\nDataset size\nFits in memory\nToo large to fit in memory\n\n\nTokenization\nPre-tokenized before training\nOn-demand during training\n\n\nConfig key\ndatasets:\npretraining_dataset:\n\n\nLong text handling\nSplits texts exceeding sequence_len\nConcatenates into fixed-length sequences\n\n\nBenefit\nCan preprocess on CPU, transfer to GPU\nStart training immediately, no preprocessing" + }, + { + "objectID": "docs/agents/pretraining.html#non-streaming-type-completion", + "href": "docs/agents/pretraining.html#non-streaming-type-completion", + "title": "Pretraining / Continual Pretraining — Agent Reference", + "section": "", + "text": "For smaller datasets that fit in memory. Pre-tokenizes the entire dataset.\ndatasets:\n - path: my_corpus\n type: completion\n # field: text # Column name (default: \"text\")" + }, + { + "objectID": "docs/agents/pretraining.html#streaming-pretraining_dataset", + "href": "docs/agents/pretraining.html#streaming-pretraining_dataset", + "title": "Pretraining / Continual Pretraining — Agent Reference", + "section": "", + "text": "For large corpora. Streams data on-demand without loading everything into memory.\npretraining_dataset:\n - path: HuggingFaceFW/fineweb-edu\n type: pretrain\n text_column: text\n split: train\n\nmax_steps: 1000 # Required — axolotl can't infer dataset size\nstreaming_multipack_buffer_size: 10000 # Buffer for sample packing\npretrain_multipack_attn: true # Prevent cross-attention between packed samples\nmax_steps is required for streaming — one step = sequence_len * micro_batch_size * gradient_accumulation_steps * num_gpus tokens.\nFull streaming docs: streaming.qmd" + }, + { + "objectID": "docs/agents/pretraining.html#dataset-format", + "href": "docs/agents/pretraining.html#dataset-format", + "title": "Pretraining / Continual Pretraining — Agent Reference", + "section": "", + "text": "{\"text\": \"The complete document text goes here.\"}" + }, + { + "objectID": "docs/agents/pretraining.html#key-settings", + "href": "docs/agents/pretraining.html#key-settings", + "title": "Pretraining / Continual Pretraining — Agent Reference", + "section": "", + "text": "sample_packing: true + pad_to_sequence_len: true — pack documents into fixed-length sequences\nflash_attention: true — required for sample packing\nNo adapter — typically full fine-tune for pretraining\ntrain_on_inputs: true — default for completion (all tokens trained on)" + }, + { + "objectID": "docs/agents/pretraining.html#file-map", + "href": "docs/agents/pretraining.html#file-map", + "title": "Pretraining / Continual Pretraining — Agent Reference", + "section": "", + "text": "src/axolotl/\n prompt_strategies/completion.py # Non-streaming: completion prompt strategy (no masking)\n utils/data/sft.py # Non-streaming: dataset loading and processing\n utils/data/streaming.py # Streaming: encode_streaming(), wrap_streaming_dataset()\n utils/schemas/config.py # Config fields: pretraining_dataset, pretrain_multipack_attn, etc.\n\nexamples/streaming/pretrain.yaml # Full streaming pretraining example config" + }, + { + "objectID": "docs/agents/grpo.html", + "href": "docs/agents/grpo.html", + "title": "GRPO — Agent Reference", + "section": "", + "text": "Online RL with verifiable reward functions. For full config reference, async features, and scaling, see grpo.qmd. For vLLM setup, see vllm_serving.qmd.\n\n\nTerminal 1 (GPU 0) Terminal 2 (GPU 1)\n┌──────────────────────┐ ┌──────────────────────────────────┐\n│ vLLM Server │ HTTP │ Trainer │\n│ Serves base model │◄────────────►│ 1. Send prompts to vLLM │\n│ + LoRA adapter │ /generate │ 2. Score completions (rewards) │\n│ │ /set_lora │ 3. Compute advantages │\n│ Punica kernels for │ │ 4. PPO-clip gradient update │\n│ LoRA inference │ │ 5. Sync LoRA weights to vLLM │\n└──────────────────────┘ └──────────────────────────────────┘\n\n\n\n\nA YAML config with rl: grpo\nA reward module (Python file with reward functions)\nA running vLLM server (axolotl vllm-serve config.yaml)\n\n\n\n\ndef my_reward(completions, **kwargs) -> list[float]:\n # completions[i][0][\"content\"] = text of i-th completion\n # **kwargs contains dataset columns not removed by transform\n return [score_for_each_completion]\nMultiple rewards: reward_funcs: [r1, r2] with reward_weights: [1.0, 0.5].\n\n\n\n\n\n\n\n\n\n\n\nFeature\nConfig\nPurpose\n\n\n\n\nAsync prefetch\nasync_prefetch: true\nOverlap generation with training\n\n\nLoRA sync\nvllm_lora_sync: true\nFast adapter sync via filesystem\n\n\nStreaming scoring\nstreaming_partial_batch: true\nScore one group at a time\n\n\nZero-adv skip\nskip_zero_advantage_batches: true\nSkip batches with no learning signal\n\n\nReplay buffer\nreplay_buffer_size: 100\nCache high-signal groups\n\n\nIS correction\nvllm_importance_sampling_correction: true\nFix off-policy distribution shift\n\n\n\n\n\n\n\nrewards/*/mean > 0.15 within 20 steps (else: test reward function standalone)\nreward_std > 0 on most steps (else: no learning signal)\nentropy 0.05-0.5 (< 0.01 = mode collapse)\ngrad_norm 0.001-1.0 (> 10 = unstable, 0.0 = zero-advantage skip)\n\nSee training_stability.qmd for detailed diagnostics.\n\n\n\nsrc/axolotl/\n cli/train.py # Entry point\n cli/vllm_serve.py # Entry point for vLLM server\n core/trainers/grpo/\n trainer.py # AxolotlGRPOTrainer\n sampler.py # Sampling utilities\n core/builders/rl.py # HFRLTrainerBuilder — routes rl type → trainer\n scripts/vllm_serve_lora.py # vLLM serve script with LoRA sync support\n utils/schemas/trl.py # TRL config schema (all trl: options)\n\ndocs/grpo.qmd # Full user docs: async, rewards, scaling, config reference\ndocs/vllm_serving.qmd # vLLM server modes, LoRA sync, weight sync" + }, + { + "objectID": "docs/agents/grpo.html#architecture", + "href": "docs/agents/grpo.html#architecture", + "title": "GRPO — Agent Reference", + "section": "", + "text": "Terminal 1 (GPU 0) Terminal 2 (GPU 1)\n┌──────────────────────┐ ┌──────────────────────────────────┐\n│ vLLM Server │ HTTP │ Trainer │\n│ Serves base model │◄────────────►│ 1. Send prompts to vLLM │\n│ + LoRA adapter │ /generate │ 2. Score completions (rewards) │\n│ │ /set_lora │ 3. Compute advantages │\n│ Punica kernels for │ │ 4. PPO-clip gradient update │\n│ LoRA inference │ │ 5. Sync LoRA weights to vLLM │\n└──────────────────────┘ └──────────────────────────────────┘" + }, + { + "objectID": "docs/agents/grpo.html#components-required", + "href": "docs/agents/grpo.html#components-required", + "title": "GRPO — Agent Reference", + "section": "", + "text": "A YAML config with rl: grpo\nA reward module (Python file with reward functions)\nA running vLLM server (axolotl vllm-serve config.yaml)" + }, + { + "objectID": "docs/agents/grpo.html#reward-function-signature", + "href": "docs/agents/grpo.html#reward-function-signature", + "title": "GRPO — Agent Reference", + "section": "", + "text": "def my_reward(completions, **kwargs) -> list[float]:\n # completions[i][0][\"content\"] = text of i-th completion\n # **kwargs contains dataset columns not removed by transform\n return [score_for_each_completion]\nMultiple rewards: reward_funcs: [r1, r2] with reward_weights: [1.0, 0.5]." + }, + { + "objectID": "docs/agents/grpo.html#key-async-features", + "href": "docs/agents/grpo.html#key-async-features", + "title": "GRPO — Agent Reference", + "section": "", + "text": "Feature\nConfig\nPurpose\n\n\n\n\nAsync prefetch\nasync_prefetch: true\nOverlap generation with training\n\n\nLoRA sync\nvllm_lora_sync: true\nFast adapter sync via filesystem\n\n\nStreaming scoring\nstreaming_partial_batch: true\nScore one group at a time\n\n\nZero-adv skip\nskip_zero_advantage_batches: true\nSkip batches with no learning signal\n\n\nReplay buffer\nreplay_buffer_size: 100\nCache high-signal groups\n\n\nIS correction\nvllm_importance_sampling_correction: true\nFix off-policy distribution shift" + }, + { + "objectID": "docs/agents/grpo.html#health-checks", + "href": "docs/agents/grpo.html#health-checks", + "title": "GRPO — Agent Reference", + "section": "", + "text": "rewards/*/mean > 0.15 within 20 steps (else: test reward function standalone)\nreward_std > 0 on most steps (else: no learning signal)\nentropy 0.05-0.5 (< 0.01 = mode collapse)\ngrad_norm 0.001-1.0 (> 10 = unstable, 0.0 = zero-advantage skip)\n\nSee training_stability.qmd for detailed diagnostics." + }, + { + "objectID": "docs/agents/grpo.html#file-map", + "href": "docs/agents/grpo.html#file-map", + "title": "GRPO — Agent Reference", + "section": "", + "text": "src/axolotl/\n cli/train.py # Entry point\n cli/vllm_serve.py # Entry point for vLLM server\n core/trainers/grpo/\n trainer.py # AxolotlGRPOTrainer\n sampler.py # Sampling utilities\n core/builders/rl.py # HFRLTrainerBuilder — routes rl type → trainer\n scripts/vllm_serve_lora.py # vLLM serve script with LoRA sync support\n utils/schemas/trl.py # TRL config schema (all trl: options)\n\ndocs/grpo.qmd # Full user docs: async, rewards, scaling, config reference\ndocs/vllm_serving.qmd # vLLM server modes, LoRA sync, weight sync" + }, + { + "objectID": "docs/agents/sft.html", + "href": "docs/agents/sft.html", + "title": "SFT — Agent Reference", + "section": "", + "text": "Supervised fine-tuning pipeline reference. For config templates and dataset format examples, see getting-started.qmd and dataset-formats/.\n\n\nYAML Config → axolotl train config.yaml\n\n 1. Load base model (+ quantization if QLoRA/8-bit)\n 2. Apply adapter layers (LoRA/QLoRA) if configured\n 3. Load + tokenize dataset(s)\n - Apply prompt template (chat_template / alpaca / custom)\n - Mask inputs (train_on_inputs: false)\n - Pack samples into sequences (sample_packing: true)\n 4. Training loop (HuggingFace Trainer)\n - forward → loss → backward → optimizer step → lr scheduler step\n 5. Save model / adapter weights + tokenizer\n\nMulti-GPU: FSDP or DeepSpeed shards model across GPUs automatically.\n\n\n\n\nA YAML config — model, dataset(s), adapter settings, hyperparameters\nA dataset — HuggingFace Hub, local JSONL/JSON/Parquet, or S3/GCS path\n(Optional) A custom prompt strategy — for non-standard dataset formats\n\nNo external server processes needed (unlike GRPO which requires vLLM).\n\n\n\nIs your data in chat/message format?\n ├─ YES: OpenAI message format (role/content)?\n │ ├─ YES ──────────────────────> type: chat_template (recommended)\n │ └─ NO (custom field names) ──> type: chat_template + message_property_mappings\n └─ NO: Instruction/response pairs?\n ├─ YES ──> type: alpaca (instruction, input, output)\n └─ NO: Raw text?\n ├─ YES with segments ─────> type: input_output (template-free masking)\n └─ YES continuous ────────> type: completion (pretraining-style)\nFull format specs: dataset-formats/\n\n\n\n\n\n\n\n\n\n\n\n\n\nModel Size\nLoRA\nQLoRA (4-bit)\nFull Fine-Tune\nVRAM (approx)\n\n\n\n\n1-3B\nPreferred\nLow-budget option\nSingle GPU OK\n8-16 GB (LoRA)\n\n\n7-8B\nPreferred\nGood balance\nNeeds multi-GPU\n16-24 GB (LoRA)\n\n\n13-14B\nPreferred\nGood balance\nMulti-GPU required\n24-40 GB (LoRA)\n\n\n30-70B\nLoRA or QLoRA\nPreferred for single GPU\nMulti-node\n40-80 GB (QLoRA)\n\n\n\n\n\n\n\n\n\nParameter\nLoRA\nQLoRA\nFull FT\n\n\n\n\nlearning_rate\n1e-4 to 3e-4\n1e-4 to 3e-4\n1e-5 to 5e-5\n\n\nlora_r\n16-64\n16-64\nN/A\n\n\nlora_alpha\n1-2x lora_r\n1-2x lora_r\nN/A\n\n\nmicro_batch_size\n2-8\n2-4\n1-2\n\n\ngradient_accumulation_steps\n2-8\n4-16\n4-16\n\n\nnum_epochs\n1-3\n1-3\n1-3\n\n\noptimizer\nadamw_8bit\nadamw_bnb_8bit\nadamw_torch_fused\n\n\n\nEffective batch = micro_batch * grad_accum * num_gpus. Lower LR for larger models.\n\n\n\n\n\n\n\n\n\n\n\nMetric\nHealthy\nProblem\n\n\n\n\ntrain_loss\nDecreasing, starting ~2-4 for chat models\nFlat or increasing from step 1 — data or LR issue\n\n\neval_loss\nDecreasing, tracks train_loss\nIncreasing while train_loss decreases — overfitting\n\n\ngrad_norm\n0.1-10, relatively stable\nSpikes >100 — instability. 0.0 — frozen weights\n\n\nlearning_rate\nFollows scheduler curve\nFlat or NaN — config issue\n\n\n\nWatch for: loss never decreasing (check train_on_inputs, dataset, LR), loss goes to 0 quickly (overfitting), eval_loss diverging (reduce epochs, add regularization). See training_stability.qmd.\n\n\n\n\n\n\n\n\n\n\nIssue\nFix\n\n\n\n\nOOM during training\nReduce micro_batch_size, enable gradient_checkpointing, reduce sequence_len\n\n\nsample_packing + SDPA + bf16 = 0.0 loss\nUse flash_attention: true or disable sample_packing\n\n\nMissing chat template error\nSet chat_template: chatml explicitly\n\n\nLabel masking wrong\nRun axolotl preprocess config.yaml --debug and inspect labels\n\n\nLoss NaN\nUse bf16: auto, lower LR, check data for empty samples\n\n\nTokenizer pad token / infinite loss\nSet special_tokens: pad_token: \"<\\|end_of_text\\|>\"\n\n\nFSDP save hangs\nUse fsdp_state_dict_type: FULL_STATE_DICT\n\n\nDeepSpeed CheckpointError\nSet use_reentrant: true in gradient_checkpointing_kwargs\n\n\n\nFull troubleshooting: training_stability.qmd, debugging.qmd\n\n\n\nsrc/axolotl/\n cli/train.py # Entry point for `axolotl train`\n cli/preprocess.py # Entry point for `axolotl preprocess`\n core/builders/causal.py # HFCausalTrainerBuilder — wires config → SFT trainer\n core/trainers/base.py # AxolotlTrainer — base trainer class\n core/trainers/mixins/ # Packing, optimizer, scheduler, checkpoints\n prompt_strategies/ # Format handlers: chat_template, alpaca, completion, input_output\n utils/schemas/config.py # AxolotlInputConfig — main config schema\n utils/schemas/datasets.py # SFTDataset, DatasetConfig\n utils/schemas/peft.py # LoraConfig — LoRA parameters\n integrations/liger/ # Liger kernel plugin\n\nexamples/llama-3/ # LoRA, QLoRA, full FT example configs\ndocs/getting-started.qmd # Quickstart with config templates\ndocs/optimizations.qmd # Flash attention, gradient checkpointing, sample packing\ndocs/multi-gpu.qmd # FSDP and DeepSpeed setup" + }, + { + "objectID": "docs/agents/sft.html#architecture", + "href": "docs/agents/sft.html#architecture", + "title": "SFT — Agent Reference", + "section": "", + "text": "YAML Config → axolotl train config.yaml\n\n 1. Load base model (+ quantization if QLoRA/8-bit)\n 2. Apply adapter layers (LoRA/QLoRA) if configured\n 3. Load + tokenize dataset(s)\n - Apply prompt template (chat_template / alpaca / custom)\n - Mask inputs (train_on_inputs: false)\n - Pack samples into sequences (sample_packing: true)\n 4. Training loop (HuggingFace Trainer)\n - forward → loss → backward → optimizer step → lr scheduler step\n 5. Save model / adapter weights + tokenizer\n\nMulti-GPU: FSDP or DeepSpeed shards model across GPUs automatically." + }, + { + "objectID": "docs/agents/sft.html#components-required", + "href": "docs/agents/sft.html#components-required", + "title": "SFT — Agent Reference", + "section": "", + "text": "A YAML config — model, dataset(s), adapter settings, hyperparameters\nA dataset — HuggingFace Hub, local JSONL/JSON/Parquet, or S3/GCS path\n(Optional) A custom prompt strategy — for non-standard dataset formats\n\nNo external server processes needed (unlike GRPO which requires vLLM)." + }, + { + "objectID": "docs/agents/sft.html#dataset-format-decision-tree", + "href": "docs/agents/sft.html#dataset-format-decision-tree", + "title": "SFT — Agent Reference", + "section": "", + "text": "Is your data in chat/message format?\n ├─ YES: OpenAI message format (role/content)?\n │ ├─ YES ──────────────────────> type: chat_template (recommended)\n │ └─ NO (custom field names) ──> type: chat_template + message_property_mappings\n └─ NO: Instruction/response pairs?\n ├─ YES ──> type: alpaca (instruction, input, output)\n └─ NO: Raw text?\n ├─ YES with segments ─────> type: input_output (template-free masking)\n └─ YES continuous ────────> type: completion (pretraining-style)\nFull format specs: dataset-formats/" + }, + { + "objectID": "docs/agents/sft.html#model-size-to-adapter-choice", + "href": "docs/agents/sft.html#model-size-to-adapter-choice", + "title": "SFT — Agent Reference", + "section": "", + "text": "Model Size\nLoRA\nQLoRA (4-bit)\nFull Fine-Tune\nVRAM (approx)\n\n\n\n\n1-3B\nPreferred\nLow-budget option\nSingle GPU OK\n8-16 GB (LoRA)\n\n\n7-8B\nPreferred\nGood balance\nNeeds multi-GPU\n16-24 GB (LoRA)\n\n\n13-14B\nPreferred\nGood balance\nMulti-GPU required\n24-40 GB (LoRA)\n\n\n30-70B\nLoRA or QLoRA\nPreferred for single GPU\nMulti-node\n40-80 GB (QLoRA)" + }, + { + "objectID": "docs/agents/sft.html#hyperparameter-ranges", + "href": "docs/agents/sft.html#hyperparameter-ranges", + "title": "SFT — Agent Reference", + "section": "", + "text": "Parameter\nLoRA\nQLoRA\nFull FT\n\n\n\n\nlearning_rate\n1e-4 to 3e-4\n1e-4 to 3e-4\n1e-5 to 5e-5\n\n\nlora_r\n16-64\n16-64\nN/A\n\n\nlora_alpha\n1-2x lora_r\n1-2x lora_r\nN/A\n\n\nmicro_batch_size\n2-8\n2-4\n1-2\n\n\ngradient_accumulation_steps\n2-8\n4-16\n4-16\n\n\nnum_epochs\n1-3\n1-3\n1-3\n\n\noptimizer\nadamw_8bit\nadamw_bnb_8bit\nadamw_torch_fused\n\n\n\nEffective batch = micro_batch * grad_accum * num_gpus. Lower LR for larger models." + }, + { + "objectID": "docs/agents/sft.html#healthy-training-indicators", + "href": "docs/agents/sft.html#healthy-training-indicators", + "title": "SFT — Agent Reference", + "section": "", + "text": "Metric\nHealthy\nProblem\n\n\n\n\ntrain_loss\nDecreasing, starting ~2-4 for chat models\nFlat or increasing from step 1 — data or LR issue\n\n\neval_loss\nDecreasing, tracks train_loss\nIncreasing while train_loss decreases — overfitting\n\n\ngrad_norm\n0.1-10, relatively stable\nSpikes >100 — instability. 0.0 — frozen weights\n\n\nlearning_rate\nFollows scheduler curve\nFlat or NaN — config issue\n\n\n\nWatch for: loss never decreasing (check train_on_inputs, dataset, LR), loss goes to 0 quickly (overfitting), eval_loss diverging (reduce epochs, add regularization). See training_stability.qmd." + }, + { + "objectID": "docs/agents/sft.html#known-issues", + "href": "docs/agents/sft.html#known-issues", + "title": "SFT — Agent Reference", + "section": "", + "text": "Issue\nFix\n\n\n\n\nOOM during training\nReduce micro_batch_size, enable gradient_checkpointing, reduce sequence_len\n\n\nsample_packing + SDPA + bf16 = 0.0 loss\nUse flash_attention: true or disable sample_packing\n\n\nMissing chat template error\nSet chat_template: chatml explicitly\n\n\nLabel masking wrong\nRun axolotl preprocess config.yaml --debug and inspect labels\n\n\nLoss NaN\nUse bf16: auto, lower LR, check data for empty samples\n\n\nTokenizer pad token / infinite loss\nSet special_tokens: pad_token: \"<\\|end_of_text\\|>\"\n\n\nFSDP save hangs\nUse fsdp_state_dict_type: FULL_STATE_DICT\n\n\nDeepSpeed CheckpointError\nSet use_reentrant: true in gradient_checkpointing_kwargs\n\n\n\nFull troubleshooting: training_stability.qmd, debugging.qmd" + }, + { + "objectID": "docs/agents/sft.html#file-map", + "href": "docs/agents/sft.html#file-map", + "title": "SFT — Agent Reference", + "section": "", + "text": "src/axolotl/\n cli/train.py # Entry point for `axolotl train`\n cli/preprocess.py # Entry point for `axolotl preprocess`\n core/builders/causal.py # HFCausalTrainerBuilder — wires config → SFT trainer\n core/trainers/base.py # AxolotlTrainer — base trainer class\n core/trainers/mixins/ # Packing, optimizer, scheduler, checkpoints\n prompt_strategies/ # Format handlers: chat_template, alpaca, completion, input_output\n utils/schemas/config.py # AxolotlInputConfig — main config schema\n utils/schemas/datasets.py # SFTDataset, DatasetConfig\n utils/schemas/peft.py # LoraConfig — LoRA parameters\n integrations/liger/ # Liger kernel plugin\n\nexamples/llama-3/ # LoRA, QLoRA, full FT example configs\ndocs/getting-started.qmd # Quickstart with config templates\ndocs/optimizations.qmd # Flash attention, gradient checkpointing, sample packing\ndocs/multi-gpu.qmd # FSDP and DeepSpeed setup" }, { "objectID": "docs/multi-gpu.html", @@ -2298,6 +2442,39 @@ "NCCL" ] }, + { + "objectID": "docs/multipack.html", + "href": "docs/multipack.html", + "title": "Multipack (Sample Packing)", + "section": "", + "text": "Because Flash Attention simply drops the attention mask, we do not need to\nconstruct a 4d attention mask. We only need to concatenate the sequences into\na single batch and let flash attention know where each new sequence begins.\n4k context, bsz =4,\neach character represents 256 tokens\nX represents a padding token\n 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n[[ A A A A A A A A A A A ]\n B B B B B B ]\n C C C C C C C ]\n D D D D ]]\n\n[[ E E E E E E E E ]\n [ F F F F ]\n [ G G G ]\n [ H H H H ]]\n\n[[ I I I ]\n [ J J J ]\n [ K K K K K]\n [ L L L ]]\nafter padding to longest input in each step\n 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n[[ A A A A A A A A A A A ]\n B B B B B B X X X X X X ]\n C C C C C C C X X X X ]\n D D D D X X X X X X X ]]\n\n[[ E E E E E E E E ]\n [ F F F F X X X X ]\n [ G G G X X X X X ]\n [ H H H H X X X X ]]\n\n[[ I I I X X ]\n [ J J J X X ]\n [ K K K K K ]\n [ L L L X X ]]\nw packing ( note it’s the same effective number of tokens per step, but a true bsz of 1)\n 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n[[ A A A A A A A A A A A B B B B B\n B C C C C C C C D D D D E E E E\n E E E E F F F F F G G G H H H H\n I I I J J J J K K K K K L L L X ]]\ncu_seqlens:\n[[ 0, 11, 17, 24, 28, 36, 41 44, 48, 51, 55, 60, 64]]", + "crumbs": [ + "Core Concepts", + "Multipack (Sample Packing)" + ] + }, + { + "objectID": "docs/multipack.html#visualization-of-multipack-with-flash-attention", + "href": "docs/multipack.html#visualization-of-multipack-with-flash-attention", + "title": "Multipack (Sample Packing)", + "section": "", + "text": "Because Flash Attention simply drops the attention mask, we do not need to\nconstruct a 4d attention mask. We only need to concatenate the sequences into\na single batch and let flash attention know where each new sequence begins.\n4k context, bsz =4,\neach character represents 256 tokens\nX represents a padding token\n 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n[[ A A A A A A A A A A A ]\n B B B B B B ]\n C C C C C C C ]\n D D D D ]]\n\n[[ E E E E E E E E ]\n [ F F F F ]\n [ G G G ]\n [ H H H H ]]\n\n[[ I I I ]\n [ J J J ]\n [ K K K K K]\n [ L L L ]]\nafter padding to longest input in each step\n 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n[[ A A A A A A A A A A A ]\n B B B B B B X X X X X X ]\n C C C C C C C X X X X ]\n D D D D X X X X X X X ]]\n\n[[ E E E E E E E E ]\n [ F F F F X X X X ]\n [ G G G X X X X X ]\n [ H H H H X X X X ]]\n\n[[ I I I X X ]\n [ J J J X X ]\n [ K K K K K ]\n [ L L L X X ]]\nw packing ( note it’s the same effective number of tokens per step, but a true bsz of 1)\n 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n[[ A A A A A A A A A A A B B B B B\n B C C C C C C C D D D D E E E E\n E E E E F F F F F G G G H H H H\n I I I J J J J K K K K K L L L X ]]\ncu_seqlens:\n[[ 0, 11, 17, 24, 28, 36, 41 44, 48, 51, 55, 60, 64]]", + "crumbs": [ + "Core Concepts", + "Multipack (Sample Packing)" + ] + }, + { + "objectID": "docs/multipack.html#multipack-without-flash-attention", + "href": "docs/multipack.html#multipack-without-flash-attention", + "title": "Multipack (Sample Packing)", + "section": "Multipack without Flash Attention", + "text": "Multipack without Flash Attention\nMultipack can still be achieved without Flash attention, but with lower packing\nefficiency as we are not able to join multiple batches into a single batch due to\ncontext length limits without flash attention. We can use either Pytorch’s Scaled\nDot Product Attention implementation or native Pytorch attention implementation\nalong with 4d attention masks\nto pack sequences together and avoid cross attention.", + "crumbs": [ + "Core Concepts", + "Multipack (Sample Packing)" + ] + }, { "objectID": "docs/debugging.html", "href": "docs/debugging.html", @@ -2336,7 +2513,7 @@ "href": "docs/debugging.html#debugging-with-vscode", "title": "Debugging", "section": "Debugging with VSCode", - "text": "Debugging with VSCode\n\nBackground\nThe below example shows how to configure VSCode to debug data preprocessing of the chat_template format. This is the format used when you have the following in your axolotl config:\ndatasets:\n - path: <path to your chat_template formatted dataset> # example on HF Hub: fozziethebeat/alpaca_messages_2k_test\n type: chat_template\n\n[!Important]\nIf you are already familiar with advanced VSCode debugging, you can skip the below explanation and look at the files .vscode/launch.json and .vscode/tasks.json for an example configuration.\n\n\n[!Tip]\nIf you prefer to watch a video, rather than read, you can skip to the video tutorial below (but doing both is recommended).\n\n\n\nSetup\nMake sure you have an editable install of Axolotl, which ensures that changes you make to the code are reflected at runtime. Run the following commands from the root of this project:\npip3 install packaging\npip3 install --no-build-isolation -e '.[flash-attn,deepspeed]'\n\nRemote Hosts\nIf you developing on a remote host, you can easily use VSCode to debug remotely. To do so, you will need to follow this remote - SSH guide. You can also see the video below on Docker and Remote SSH debugging.\n\n\n\nConfiguration\nThe easiest way to get started is to modify the .vscode/launch.json file in this project. This is just an example configuration, so you may need to modify or copy it to suit your needs.\nFor example, to mimic the command cd devtools && CUDA_VISIBLE_DEVICES=0 accelerate launch -m axolotl.cli.train dev_chat_template.yml, you would use the below configuration1. Note that we add additional flags that override the axolotl config and incorporate the tips above (see the comments). We also set the working directory to devtools and set the env variable HF_HOME to a temporary folder that is later partially deleted. This is because we want to delete the HF dataset cache before each run in order to ensure that the data preprocessing code is run from scratch.\n// .vscode/launch.json\n{\n \"version\": \"0.2.0\",\n \"configurations\": [\n {\n \"name\": \"Debug axolotl prompt - chat_template\",\n \"type\": \"python\",\n \"module\": \"accelerate.commands.launch\",\n \"request\": \"launch\",\n \"args\": [\n \"-m\", \"axolotl.cli.train\", \"dev_chat_template.yml\",\n // The flags below simplify debugging by overriding the axolotl config\n // with the debugging tips above. Modify as needed.\n \"--dataset_num_proc=1\", // limits data preprocessing to one process\n \"--max_steps=1\", // limits training to just one step\n \"--batch_size=1\", // minimizes batch size\n \"--micro_batch_size=1\", // minimizes batch size\n \"--val_set_size=0\", // disables validation\n \"--sample_packing=False\", // disables sample packing which is necessary for small datasets\n \"--eval_sample_packing=False\",// disables sample packing on eval set\n \"--dataset_prepared_path=temp_debug/axolotl_outputs/data\", // send data outputs to a temp folder\n \"--output_dir=temp_debug/axolotl_outputs/model\" // send model outputs to a temp folder\n ],\n \"console\": \"integratedTerminal\", // show output in the integrated terminal\n \"cwd\": \"${workspaceFolder}/devtools\", // set working directory to devtools from the root of the project\n \"justMyCode\": true, // step through only axolotl code\n \"env\": {\"CUDA_VISIBLE_DEVICES\": \"0\", // Since we aren't doing distributed training, we need to limit to one GPU\n \"HF_HOME\": \"${workspaceFolder}/devtools/temp_debug/.hf-cache\"}, // send HF cache to a temp folder\n \"preLaunchTask\": \"cleanup-for-dataprep\", // delete temp folders (see below)\n }\n ]\n}\nAdditional notes about this configuration:\n\nThe argument justMyCode is set to true such that you step through only the axolotl code. If you want to step into dependencies, set this to false.\nThe preLaunchTask: cleanup-for-dataprep is defined in .vscode/tasks.json and is used to delete the following folders before debugging, which is essential to ensure that the data pre-processing code is run from scratch:\n\n./devtools/temp_debug/axolotl_outputs\n./devtools/temp_debug/.hf-cache/datasets\n\n\n\n[!Tip]\nYou may not want to delete these folders. For example, if you are debugging model training instead of data pre-processing, you may NOT want to delete the cache or output folders. You may also need to add additional tasks to the tasks.json file depending on your use case.\n\nBelow is the ./vscode/tasks.json file that defines the cleanup-for-dataprep task. This task is run before each debugging session when you use the above configuration. Note how there are two tasks that delete the two folders mentioned above. The third task cleanup-for-dataprep is a composite task that combines the two tasks. A composite task is necessary because VSCode does not allow you to specify multiple tasks in the preLaunchTask argument of the launch.json file.\n// .vscode/tasks.json\n// this file is used by launch.json\n{\n \"version\": \"2.0.0\",\n \"tasks\": [\n // this task changes into the devtools directory and deletes the temp_debug/axolotl_outputs folder\n {\n \"label\": \"delete-outputs\",\n \"type\": \"shell\",\n \"command\": \"rm -rf temp_debug/axolotl_outputs\",\n \"options\":{ \"cwd\": \"${workspaceFolder}/devtools\"},\n \"problemMatcher\": []\n },\n // this task changes into the devtools directory and deletes the `temp_debug/.hf-cache/datasets` folder\n {\n \"label\": \"delete-temp-hf-dataset-cache\",\n \"type\": \"shell\",\n \"command\": \"rm -rf temp_debug/.hf-cache/datasets\",\n \"options\":{ \"cwd\": \"${workspaceFolder}/devtools\"},\n \"problemMatcher\": []\n },\n // this task combines the two tasks above\n {\n \"label\": \"cleanup-for-dataprep\",\n \"dependsOn\": [\"delete-outputs\", \"delete-temp-hf-dataset-cache\"],\n }\n ]\n}\n\n\nCustomizing your debugger\nYour debugging use case may differ from the example above. The easiest thing to do is to put your own axolotl config in the devtools folder and modify the launch.json file to use your config. You may also want to modify the preLaunchTask to delete different folders or not delete anything at all.\n\n\nVideo Tutorial\nThe following video tutorial walks through the above configuration and demonstrates how to debug with VSCode, (click the image below to watch):\n\n\n\nHamel Husain’s tutorial: Debugging Axolotl w/VSCode", + "text": "Debugging with VSCode\n\nBackground\nThe below example shows how to configure VSCode to debug data preprocessing of the chat_template format. This is the format used when you have the following in your axolotl config:\ndatasets:\n - path: <path to your chat_template formatted dataset> # example on HF Hub: fozziethebeat/alpaca_messages_2k_test\n type: chat_template\n\n[!Important]\nIf you are already familiar with advanced VSCode debugging, you can skip the below explanation and look at the files .vscode/launch.json and .vscode/tasks.json for an example configuration.\n\n\n[!Tip]\nIf you prefer to watch a video, rather than read, you can skip to the video tutorial below (but doing both is recommended).\n\n\n\nSetup\nMake sure you have an editable install of Axolotl, which ensures that changes you make to the code are reflected at runtime. Run the following commands from the root of this project:\npip3 install packaging\npip3 install --no-build-isolation -e '.[flash-attn,deepspeed]'\n\nRemote Hosts\nIf you developing on a remote host, you can easily use VSCode to debug remotely. To do so, you will need to follow this remote - SSH guide. You can also see the video below on Docker and Remote SSH debugging.\n\n\n\nConfiguration\nThe easiest way to get started is to modify the .vscode/launch.json file in this project. This is just an example configuration, so you may need to modify or copy it to suit your needs.\nFor example, to mimic the command cd devtools && CUDA_VISIBLE_DEVICES=0 axolotl train dev_chat_template.yml, you would use the below configuration1. Note that we add additional flags that override the axolotl config and incorporate the tips above (see the comments). We also set the working directory to devtools and set the env variable HF_HOME to a temporary folder that is later partially deleted. This is because we want to delete the HF dataset cache before each run in order to ensure that the data preprocessing code is run from scratch.\n// .vscode/launch.json\n{\n \"version\": \"0.2.0\",\n \"configurations\": [\n {\n \"name\": \"Debug axolotl prompt - chat_template\",\n \"type\": \"python\",\n \"module\": \"accelerate.commands.launch\",\n \"request\": \"launch\",\n \"args\": [\n \"-m\", \"axolotl.cli.train\", \"dev_chat_template.yml\",\n // The flags below simplify debugging by overriding the axolotl config\n // with the debugging tips above. Modify as needed.\n \"--dataset_num_proc=1\", // limits data preprocessing to one process\n \"--max_steps=1\", // limits training to just one step\n \"--batch_size=1\", // minimizes batch size\n \"--micro_batch_size=1\", // minimizes batch size\n \"--val_set_size=0\", // disables validation\n \"--sample_packing=False\", // disables sample packing which is necessary for small datasets\n \"--eval_sample_packing=False\",// disables sample packing on eval set\n \"--dataset_prepared_path=temp_debug/axolotl_outputs/data\", // send data outputs to a temp folder\n \"--output_dir=temp_debug/axolotl_outputs/model\" // send model outputs to a temp folder\n ],\n \"console\": \"integratedTerminal\", // show output in the integrated terminal\n \"cwd\": \"${workspaceFolder}/devtools\", // set working directory to devtools from the root of the project\n \"justMyCode\": true, // step through only axolotl code\n \"env\": {\"CUDA_VISIBLE_DEVICES\": \"0\", // Since we aren't doing distributed training, we need to limit to one GPU\n \"HF_HOME\": \"${workspaceFolder}/devtools/temp_debug/.hf-cache\"}, // send HF cache to a temp folder\n \"preLaunchTask\": \"cleanup-for-dataprep\", // delete temp folders (see below)\n }\n ]\n}\nAdditional notes about this configuration:\n\nThe argument justMyCode is set to true such that you step through only the axolotl code. If you want to step into dependencies, set this to false.\nThe preLaunchTask: cleanup-for-dataprep is defined in .vscode/tasks.json and is used to delete the following folders before debugging, which is essential to ensure that the data pre-processing code is run from scratch:\n\n./devtools/temp_debug/axolotl_outputs\n./devtools/temp_debug/.hf-cache/datasets\n\n\n\n[!Tip]\nYou may not want to delete these folders. For example, if you are debugging model training instead of data pre-processing, you may NOT want to delete the cache or output folders. You may also need to add additional tasks to the tasks.json file depending on your use case.\n\nBelow is the ./vscode/tasks.json file that defines the cleanup-for-dataprep task. This task is run before each debugging session when you use the above configuration. Note how there are two tasks that delete the two folders mentioned above. The third task cleanup-for-dataprep is a composite task that combines the two tasks. A composite task is necessary because VSCode does not allow you to specify multiple tasks in the preLaunchTask argument of the launch.json file.\n// .vscode/tasks.json\n// this file is used by launch.json\n{\n \"version\": \"2.0.0\",\n \"tasks\": [\n // this task changes into the devtools directory and deletes the temp_debug/axolotl_outputs folder\n {\n \"label\": \"delete-outputs\",\n \"type\": \"shell\",\n \"command\": \"rm -rf temp_debug/axolotl_outputs\",\n \"options\":{ \"cwd\": \"${workspaceFolder}/devtools\"},\n \"problemMatcher\": []\n },\n // this task changes into the devtools directory and deletes the `temp_debug/.hf-cache/datasets` folder\n {\n \"label\": \"delete-temp-hf-dataset-cache\",\n \"type\": \"shell\",\n \"command\": \"rm -rf temp_debug/.hf-cache/datasets\",\n \"options\":{ \"cwd\": \"${workspaceFolder}/devtools\"},\n \"problemMatcher\": []\n },\n // this task combines the two tasks above\n {\n \"label\": \"cleanup-for-dataprep\",\n \"dependsOn\": [\"delete-outputs\", \"delete-temp-hf-dataset-cache\"],\n }\n ]\n}\n\n\nCustomizing your debugger\nYour debugging use case may differ from the example above. The easiest thing to do is to put your own axolotl config in the devtools folder and modify the launch.json file to use your config. You may also want to modify the preLaunchTask to delete different folders or not delete anything at all.\n\n\nVideo Tutorial\nThe following video tutorial walks through the above configuration and demonstrates how to debug with VSCode, (click the image below to watch):\n\n\n\nHamel Husain’s tutorial: Debugging Axolotl w/VSCode", "crumbs": [ "Troubleshooting", "Debugging" @@ -2358,7 +2535,7 @@ "href": "docs/debugging.html#footnotes", "title": "Debugging", "section": "Footnotes", - "text": "Footnotes\n\n\nThe config actually mimics the command CUDA_VISIBLE_DEVICES=0 python -m accelerate.commands.launch -m axolotl.cli.train devtools/chat_template.yml, but this is the same thing.↩︎\nMany of the below flags are recommended best practices by Nvidia when using nvidia-container-toolkit. You can read more about these flags here.↩︎", + "text": "Footnotes\n\n\nThe VSCode config uses accelerate.commands.launch as the Python module entry point, which is what axolotl train invokes under the hood.↩︎\nMany of the below flags are recommended best practices by Nvidia when using nvidia-container-toolkit. You can read more about these flags here.↩︎", "crumbs": [ "Troubleshooting", "Debugging" @@ -2387,212 +2564,256 @@ ] }, { - "objectID": "docs/ray-integration.html", - "href": "docs/ray-integration.html", - "title": "Ray Train", + "objectID": "docs/vllm_serving.html", + "href": "docs/vllm_serving.html", + "title": "vLLM Serving for GRPO Training", "section": "", - "text": "Axolotl supports using Ray as an alternative to accelerate for orchestrating training. This is especially useful for multi-node training since you only have to setup code and dependencies in a single node and launch training as if you were using a single node.\nWith the --use-ray CLI flag, Axolotl will use Ray Train’s TorchTrainer to run training.", + "text": "GRPO (Group Relative Policy Optimization) trains a language model by generating completions, scoring them with reward functions, and updating the policy to favor higher-reward outputs. The generation step is the bottleneck: producing thousands of tokens per training step with the policy model is slow using standard HuggingFace generation.\nAxolotl uses vLLM as a high-throughput generation backend. vLLM runs as a separate process (either on a dedicated GPU or colocated on the training GPU) and serves completions via an HTTP API. The trainer sends prompts to vLLM, receives completions, scores them, and performs gradient updates.\n┌──────────────────────┐ HTTP ┌──────────────────────┐\n│ Trainer (GPU 1) │ ───────────────── │ vLLM Server (GPU 0)│\n│ │ prompts/compls │ │\n│ - Policy model │ ◄──────────────── │ - Same base model │\n│ - Reward scoring │ │ - Fast generation │\n│ - Gradient updates │ weight sync │ - LoRA adapter │\n│ - LoRA adapter │ ─────────────────►│ (periodically │\n│ │ (every N steps) │ updated) │\n└──────────────────────┘ └──────────────────────┘\n\n\n\n\n\n\nImportant\n\n\n\nvLLM must serve the same base model specified in your training config. If the models do not match, weight synchronization will silently produce incorrect results.", "crumbs": [ - "Deployments", - "Ray Train" + "How To Guides", + "vLLM Serving for GRPO Training" ] }, { - "objectID": "docs/ray-integration.html#ray-cluster-setup", - "href": "docs/ray-integration.html#ray-cluster-setup", - "title": "Ray Train", - "section": "Ray cluster setup", - "text": "Ray cluster setup\nA prerequisite using the Ray Train integration is to setup a Ray cluster on your desired node(s). For a detailed guide on how you can get started with ray clusters, check the official Ray docs here.\nEvery Ray cluster has one head node and a set of worker nodes. The head node is just like any other worker node, but it also runs certain special processes related to scheduling and orchestration. Ray-enabled scripts are run on the head node and depending on the resources (number of CPUs, GPUs, etc) they request, will be scheduled to run certain tasks on the worker nodes. For more on key concepts behind a Ray cluster, you can refer this doc.", - "crumbs": [ - "Deployments", - "Ray Train" - ] - }, - { - "objectID": "docs/ray-integration.html#sanity-check", - "href": "docs/ray-integration.html#sanity-check", - "title": "Ray Train", - "section": "Sanity check", - "text": "Sanity check\nTo run a sanity check on whether your ray cluster is setup properly, execute the following on the head node:\nray status\nThe output should have a summary of your Ray cluster - list of all the nodes in your cluster, the number of CPUs and GPUs in your cluster, etc. For example, if you have a cluster with 1 CPU-only head node and 2 4xL40S worker nodes, the output can look like this:\nNode status\n---------------------------------------------------------------\nActive:\n 1 head\nIdle:\n 2 4xL40S:48CPU-384GB\nPending:\n (no pending nodes)\nRecent failures:\n (no failures)\n\nResources\n---------------------------------------------------------------\nUsage:\n 0.0/96.0 CPU\n 0.0/8.0 GPU\n 0B/800.00GiB memory\n 0B/229.57GiB object_store_memory\n\nDemands:\n (no resource demands)\nYou should also be able to see the same on the Ray dashboard.", - "crumbs": [ - "Deployments", - "Ray Train" - ] - }, - { - "objectID": "docs/ray-integration.html#configuring-training-with-ray-train", - "href": "docs/ray-integration.html#configuring-training-with-ray-train", - "title": "Ray Train", - "section": "Configuring training with Ray Train", - "text": "Configuring training with Ray Train\nYou can find an example configuration at configs/llama-3/lora-1b-ray.yaml.\nThe key parameters to note here are:\nuse_ray: true\nray_num_workers: 4\n# optional\nresources_per_worker:\n GPU: 1\n\nuse_ray: This is the flag that enables the Ray Train integration. You can either use the corresponding --use-ray flag in the CLI or set use_ray in the config file.\nray_num_workers: This is the number of workers/GPUs to use for training.\nresources_per_worker: This is the Ray resource request for each worker. This can be used to request a specific GPU type or a custom resource for each worker. For example, if your ray cluster has GPUs of different types, and you only want to use NVIDIA L40S GPUs, you can do\n\nresources_per_worker:\n accelerator_type:L40S: 0.001", - "crumbs": [ - "Deployments", - "Ray Train" - ] - }, - { - "objectID": "docs/ray-integration.html#launching-training", - "href": "docs/ray-integration.html#launching-training", - "title": "Ray Train", - "section": "Launching training", - "text": "Launching training\nYou can simply run the following command on the head node:\naxolotl train examples/llama-3/lora-1b-ray.yml --use-ray\nThis will launch training on the head node and workers will be scheduled automatically by Ray Train to run on the appropriate head or worker nodes.\nYou can also monitor training progress on the Ray dashboard.\nComing back to the example on a Ray cluster with 1 head node and 2 4xL40S worker nodes, let’s say you want to make use of all 8 GPUs. You would be able to just set ray_num_workers: 8 and run the previous command. The Cluster tab will show the following:\n\n\n\nRay dashboard", - "crumbs": [ - "Deployments", - "Ray Train" - ] - }, - { - "objectID": "docs/custom_integrations.html", - "href": "docs/custom_integrations.html", - "title": "Custom Integrations", + "objectID": "docs/vllm_serving.html#sec-overview", + "href": "docs/vllm_serving.html#sec-overview", + "title": "vLLM Serving for GRPO Training", "section": "", - "text": "Axolotl adds custom features through integrations. They are located within the src/axolotl/integrations directory.\nTo enable them, please check the respective documentations.", + "text": "GRPO (Group Relative Policy Optimization) trains a language model by generating completions, scoring them with reward functions, and updating the policy to favor higher-reward outputs. The generation step is the bottleneck: producing thousands of tokens per training step with the policy model is slow using standard HuggingFace generation.\nAxolotl uses vLLM as a high-throughput generation backend. vLLM runs as a separate process (either on a dedicated GPU or colocated on the training GPU) and serves completions via an HTTP API. The trainer sends prompts to vLLM, receives completions, scores them, and performs gradient updates.\n┌──────────────────────┐ HTTP ┌──────────────────────┐\n│ Trainer (GPU 1) │ ───────────────── │ vLLM Server (GPU 0)│\n│ │ prompts/compls │ │\n│ - Policy model │ ◄──────────────── │ - Same base model │\n│ - Reward scoring │ │ - Fast generation │\n│ - Gradient updates │ weight sync │ - LoRA adapter │\n│ - LoRA adapter │ ─────────────────►│ (periodically │\n│ │ (every N steps) │ updated) │\n└──────────────────────┘ └──────────────────────┘\n\n\n\n\n\n\nImportant\n\n\n\nvLLM must serve the same base model specified in your training config. If the models do not match, weight synchronization will silently produce incorrect results.", "crumbs": [ - "Advanced Features", - "Custom Integrations" + "How To Guides", + "vLLM Serving for GRPO Training" ] }, { - "objectID": "docs/custom_integrations.html#cut-cross-entropy", - "href": "docs/custom_integrations.html#cut-cross-entropy", - "title": "Custom Integrations", - "section": "Cut Cross Entropy", - "text": "Cut Cross Entropy\nCut Cross Entropy (CCE) reduces VRAM usage through optimization on the cross-entropy operation during loss calculation.\nSee https://github.com/apple/ml-cross-entropy\n\nRequirements\n\nPyTorch 2.4.0 or higher\n\n\n\nInstallation\nRun the following command to install cut_cross_entropy[transformers] if you don’t have it already.\n\nIf you are in dev environment\n\npython scripts/cutcrossentropy_install.py | sh\n\nIf you are installing from pip\n\npip3 uninstall -y cut-cross-entropy && pip3 install \"cut-cross-entropy[transformers] @ git+https://github.com/axolotl-ai-cloud/ml-cross-entropy.git@63b15e6\"\n\n\nUsage\nplugins:\n - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin\n\n\nSupported Models\n\nafmoe\napertus\narcee\ncohere\ncohere2\ndeepseek_v3\nexaone4\ngemma\ngemma2\ngemma3\ngemma3_text\ngemma3n\ngemma3n_text\nglm\nglm4\nglm4_moe\nglm4_moe_lite\nglm46v\nglm4v\nglm4v_moe\nglm_image\nglm_moe_dsa\ngpt_oss\ngranite\ngranitemoe\ngranitemoehybrid\ngranitemoeshared\nhunyuan_v1_dense\nhunyuan_v1_moe\ninternvl\nkimi_linear\nlfm2\nlfm2_moe\nlfm2_vl\nllama\nllama4\nllama4_text\nllava\nministral\nministral3\nmistral\nmistral3\nmistral4\nmixtral\nmllama\nnemotron_h\nolmo\nolmo2\nolmo3\nolmoe\nphi\nphi3\nphi4_multimodal\nqwen2\nqwen2_5_vl\nqwen2_moe\nqwen2_vl\nqwen3\nqwen3_5\nqwen3_5_text\nqwen3_5_moe\nqwen3_5_moe_text\nqwen3_moe\nqwen3_next\nqwen3_vl\nqwen3_vl_moe\nseed_oss\nsmollm3\nstep3p5\nvoxtral\n\n\n\nCitation\n@article{wijmans2024cut,\n author = {Erik Wijmans and\n Brody Huval and\n Alexander Hertzberg and\n Vladlen Koltun and\n Philipp Kr\\\"ahenb\\\"uhl},\n title = {Cut Your Losses in Large-Vocabulary Language Models},\n journal = {arXiv},\n year = {2024},\n url = {https://arxiv.org/abs/2411.09009},\n}\nPlease see reference here", + "objectID": "docs/vllm_serving.html#sec-server-mode", + "href": "docs/vllm_serving.html#sec-server-mode", + "title": "vLLM Serving for GRPO Training", + "section": "2 Server Mode", + "text": "2 Server Mode\nServer mode runs vLLM as an external process on dedicated GPU(s). This is the recommended configuration for most setups.\n\n2.1 Starting the Server\nUse the axolotl vllm-serve command with your training config:\n# Terminal 1: Start vLLM on GPU 0\nCUDA_VISIBLE_DEVICES=0 axolotl vllm-serve grpo_config.yaml\n# Terminal 2: Start training on GPU 1\nCUDA_VISIBLE_DEVICES=1 axolotl train grpo_config.yaml\nThe server reads vLLM settings from the vllm: section of your config and starts an HTTP server (default: http://0.0.0.0:8000).\n\n\n\n\n\n\nTip\n\n\n\nUse tmux or screen to manage the vLLM server process. Typical startup time is 30-90 seconds depending on model size and whether CUDA graphs are captured.\n\n\n\n\n2.2 Minimal Server Config\nbase_model: Qwen/Qwen2.5-1.5B-Instruct\n\nvllm:\n host: 0.0.0.0\n port: 8000\n gpu_memory_utilization: 0.85\n dtype: auto\n max_model_len: 4096\n\nrl: grpo\ntrl:\n use_vllm: true\n vllm_server_host: 0.0.0.0\n vllm_server_port: 8000\n vllm_server_timeout: 300\n\n\n2.3 Multi-GPU vLLM\nFor larger models, use tensor parallelism across multiple GPUs:\nvllm:\n tensor_parallel_size: 2\n gpu_memory_utilization: 0.85\n# vLLM on GPUs 2,3; training on GPUs 0,1\nCUDA_VISIBLE_DEVICES=2,3 axolotl vllm-serve grpo_config.yaml\nCUDA_VISIBLE_DEVICES=0,1 axolotl train grpo_config.yaml --num-processes 2\n\n\n\n\n\n\nNote\n\n\n\nDue to how TRL maps vLLM device indices, the vLLM instance should use the last N GPUs (highest device indices), while training uses the first N.", "crumbs": [ - "Advanced Features", - "Custom Integrations" + "How To Guides", + "vLLM Serving for GRPO Training" ] }, { - "objectID": "docs/custom_integrations.html#densemixer", - "href": "docs/custom_integrations.html#densemixer", - "title": "Custom Integrations", - "section": "DenseMixer", - "text": "DenseMixer\nSee DenseMixer\nSimply add the following to your axolotl YAML config:\nplugins:\n - axolotl.integrations.densemixer.DenseMixerPlugin\nPlease see reference here", + "objectID": "docs/vllm_serving.html#sec-colocate-mode", + "href": "docs/vllm_serving.html#sec-colocate-mode", + "title": "vLLM Serving for GRPO Training", + "section": "3 Colocate Mode", + "text": "3 Colocate Mode\nColocate mode runs vLLM on the same GPU as the trainer. This is useful when you only have a single GPU.\ntrl:\n use_vllm: true\n vllm_mode: colocate\n vllm_enable_sleep_mode: true\nWith vllm_enable_sleep_mode: true, vLLM offloads its VRAM allocation when not actively generating, freeing memory for training. When the trainer needs new completions, vLLM wakes up and reclaims VRAM.\n\n\n\n\n\n\nWarning\n\n\n\nColocate mode is significantly slower than server mode because generation and training cannot overlap. The GPU alternates between the two workloads. This mode is practical only for smaller models (up to ~3B on a 24 GB GPU).\n\n\nWhen to use colocate mode:\n\nYou have exactly one GPU\nThe model fits in memory with both vLLM and training active (with sleep mode), or is small enough to time-share\nYou accept the performance tradeoff for simpler setup (no separate vLLM process to manage)\n\nWhen to use server mode:\n\nYou have two or more GPUs\nYou want maximum throughput (generation overlaps with training via async prefetch)\nYou are running larger models (7B+)", "crumbs": [ - "Advanced Features", - "Custom Integrations" + "How To Guides", + "vLLM Serving for GRPO Training" ] }, { - "objectID": "docs/custom_integrations.html#diffusion-lm-training-plugin-for-axolotl", - "href": "docs/custom_integrations.html#diffusion-lm-training-plugin-for-axolotl", - "title": "Custom Integrations", - "section": "Diffusion LM Training Plugin for Axolotl", - "text": "Diffusion LM Training Plugin for Axolotl\nThis plugin enables diffusion language model training using an approach inspired by\nLLaDA (Large Language Diffusion Models) within Axolotl.\n\nOverview\nLLaDA is a diffusion-based approach to language model training that uses:\n- Random token masking during training instead of next-token prediction\n- Bidirectional attention to allow the model to attend to the full context\n- Importance weighting based on masking probabilities for stable training\nThis approach can lead to more robust language models with better understanding of\nbidirectional context.\n\n\nInstallation\nThe plugin is included with Axolotl. See our\ninstallation docs.\n\n\nQuickstart\nTrain with an example config (Llama‑3.2 1B):\n- Pretrain: axolotl train examples/llama-3/diffusion-3.2-1b-pretrain.yaml\n- SFT: axolotl train examples/llama-3/diffusion-3.2-1b-sft.yaml\n\n\nBasic Configuration\nYou can also modify your existing configs to enable / customize diffusion training.\nAdd the following to your Axolotl config:\nplugins:\n - axolotl.integrations.diffusion.DiffusionPlugin\nAnd, configure the nested diffusion block (defaults shown):\ndiffusion:\n noise_schedule: linear # or \"cosine\"\n min_mask_ratio: 0.1\n max_mask_ratio: 0.9\n num_diffusion_steps: 128\n eps: 1e-3\n importance_weighting: true\n\n # Mask token (training auto-adds if missing, avoid pad/eos)\n mask_token_str: \"<|diffusion_mask|>\"\n # Or use an existing special token id (e.g., 128002 for Llama-3.x)\n # mask_token_id: 128002\n\n # Sample generation during training (optional)\n generate_samples: true\n generation_interval: 100\n num_generation_samples: 3\n generation_steps: 128\n generation_temperature: 0.0\n generation_max_length: 100\n\n\nSupported Models\nAny models that support 4D attention masks should work out of the box. If not, please\ncreate an issue or open a\nPR!\n\n\nHow It Works\n\n\nRandom Masking\nDuring training, tokens are randomly masked:\n- Sample timestep t uniformly from [0, 1]\n- Calculate masking probability: p = (1 - eps) * t + eps\n- Randomly mask tokens with probability p\n\n\nDiffusion Loss\nLoss is computed only on masked tokens with (optional) importance weighting:\nloss = sum(cross_entropy(pred, target) / p_mask) / total_tokens\n\n\nSample Generation\nWhen diffusion.generate_samples: true, the plugin generates samples during training:\nSample 1:\n Original (45 tokens): The quick brown fox jumps over the lazy dog...\n Masked (18/45 tokens, 40.0%): The [MASK] [MASK] fox [MASK] over [MASK] lazy [MASK]...\n Generated: The quick brown fox jumps over the lazy dog...\nSamples are logged to console and wandb (if enabled).\n\n\nInference\nDiffusion inference is integrated into the standard Axolotl CLI. Use the same config\nyou trained with and run:\naxolotl inference path/to/your-config.yaml\nOptionally, pass --gradio to use a simple web interface.\nInteractive controls (prefix the prompt with commands):\n- :complete N → completion mode with N new masked tokens appended (default 64)\n- :mask R → random masking mode with target mask ratio R in [0.0, 1.0]\nExample session:\n================================================================================\nCommands:\n:complete N -> completion mode with N tokens (default 64)\n:mask R -> random masking with ratio R (0.0–1.0)\n================================================================================\nGive me an instruction (Ctrl + D to submit):\n\n:mask 0.4 The quick brown fox jumps over the lazy dog\n\nMasked (40.0%):\nThe [MASK] brown [MASK] jumps over the [MASK] dog\n\nGenerated:\nThe quick brown fox jumps over the loud dog\n\n\nMetrics and Monitoring\nThe plugin adds (or modifies) several metrics to track diffusion training:\n\ntrain/loss: Weighted diffusion loss\ntrain/accuracy: Accuracy on masked tokens\ntrain/mask_ratio: Average fraction of tokens masked\ntrain/num_masked_tokens: Number of tokens masked\ntrain/avg_p_mask: Average masking probability\ntrain/ce_loss: Unweighted cross-entropy loss\ntrain/importance_weight_avg: Average importance weight\n\n\n\nLimitations\n\nNo flash attention support\nNo RL training support\n\n\n\nReferences\n\nLLaDA Paper\nAxolotl Documentation\nAPI reference for plugin\n\nPlease see reference here", + "objectID": "docs/vllm_serving.html#sec-lora-sync", + "href": "docs/vllm_serving.html#sec-lora-sync", + "title": "vLLM Serving for GRPO Training", + "section": "4 LoRA Sync", + "text": "4 LoRA Sync\nLoRA sync is the recommended weight synchronization method when training with LoRA adapters. Instead of merging adapter weights into the base model and broadcasting the full merged weights over NCCL, it saves only the LoRA adapter files to the filesystem and tells vLLM to load them natively.\n\n4.1 How It Works\n\nThe trainer calls model.save_pretrained() to write the LoRA adapter weights to a temporary directory\nThe trainer sends an HTTP POST to /set_lora_adapter/ on the vLLM server\nvLLM loads the adapter using its native LoRA support (Punica kernels)\nGeneration uses the updated adapter on the next request\n\n\n\n4.2 Benefits\n\nSmaller sync payload: Transfers ~40 MB of LoRA weights instead of ~1.4 GB+ of merged model weights (for a typical 0.5-3B model)\nNo NCCL communicator: Eliminates the need for a cross-GPU NCCL communication channel, removing GPU contention between vLLM generation and weight sync\nFaster sync: ~200 ms per sync vs. 350 ms to 5+ seconds for NCCL merge sync\nSimpler multi-GPU: No need to set up NCCL groups between trainer and vLLM processes\n\n\n\n4.3 Configuration\nadapter: lora\nlora_r: 32\nlora_alpha: 64\nlora_target_linear: true\n\ntrl:\n vllm_lora_sync: true # Enables LoRA sync mode\n vllm_sync_interval: 5 # Sync every 5 training steps\nSetting vllm_lora_sync: true automatically selects the LoRA-aware vLLM serve script (axolotl.scripts.vllm_serve_lora). You do not need to set vllm.serve_module manually.\n\n\n\n\n\n\nImportant\n\n\n\nLoRA sync requires that you are training with a LoRA adapter (adapter: lora or adapter: qlora). It is not applicable to full fine-tuning.", "crumbs": [ - "Advanced Features", - "Custom Integrations" + "How To Guides", + "vLLM Serving for GRPO Training" ] }, { - "objectID": "docs/custom_integrations.html#grokfast", - "href": "docs/custom_integrations.html#grokfast", - "title": "Custom Integrations", - "section": "Grokfast", - "text": "Grokfast\nSee https://github.com/ironjr/grokfast\n\nUsage\nplugins:\n - axolotl.integrations.grokfast.GrokfastPlugin\n\ngrokfast_alpha: 2.0\ngrokfast_lamb: 0.98\n\n\nCitation\n@article{lee2024grokfast,\n title={{Grokfast}: Accelerated Grokking by Amplifying Slow Gradients},\n author={Lee, Jaerin and Kang, Bong Gyun and Kim, Kihoon and Lee, Kyoung Mu},\n journal={arXiv preprint arXiv:2405.20233},\n year={2024}\n}\nPlease see reference here", + "objectID": "docs/vllm_serving.html#sec-weight-sync", + "href": "docs/vllm_serving.html#sec-weight-sync", + "title": "vLLM Serving for GRPO Training", + "section": "5 Weight Synchronization", + "text": "5 Weight Synchronization\nDuring GRPO training, the policy model on the trainer is continuously updated via gradient steps. The vLLM server, however, still holds the old weights. Periodically, the trainer must push updated weights to vLLM so that future generations reflect the improved policy.\n\n5.1 Sync Interval\nThe vllm_sync_interval parameter controls how often weights are synced:\ntrl:\n vllm_sync_interval: 5 # Sync every 5 optimizer steps\nTradeoffs:\n\nLower interval (e.g., 1-3): Fresher generations, better on-policy data, but more sync overhead per step\nHigher interval (e.g., 5-10): Less overhead, but generations become increasingly off-policy between syncs\nRecommended: 3-5 for most setups. Axolotl includes importance sampling correction (vllm_importance_sampling_correction: true) to handle mild distribution mismatch from stale vLLM weights.\n\n\n\n5.2 Sync Methods\n\n\n\n\n\n\n\n\n\n\nMethod\nConfig\nPayload\nMechanism\nTypical Time\n\n\n\n\nLoRA sync\nvllm_lora_sync: true\nLoRA adapter only (~40 MB)\nFilesystem + HTTP\n~200 ms\n\n\nNCCL merge sync\nDefault (no lora_sync)\nFull merged weights (~1.4 GB+)\nHTTP trigger + NCCL broadcast\n350 ms - 5 s\n\n\n\n\n\n\n\n\n\nTip\n\n\n\nIf you are training with LoRA (which is recommended for GRPO), always enable vllm_lora_sync: true. The performance difference is substantial, especially as training progresses and NCCL contention increases.\n\n\n\n\n5.3 Importance Sampling Correction\nWhen vLLM weights are stale (between syncs), the generated data is slightly off-policy. Axolotl can correct for this:\ntrl:\n vllm_importance_sampling_correction: true\n importance_sampling_level: token # 'token' or 'sequence'\n off_policy_mask_threshold: 0.5 # KL threshold for masking stale sequences\n\nToken-level IS is recommended when using Liger kernel (sequence-level has numerical issues with chunked computation)\nOff-policy sequence masking (OPSM) drops sequences that have diverged too far from the current policy, providing a safety net against stale data", "crumbs": [ - "Advanced Features", - "Custom Integrations" + "How To Guides", + "vLLM Serving for GRPO Training" ] }, { - "objectID": "docs/custom_integrations.html#kernels-integration", - "href": "docs/custom_integrations.html#kernels-integration", - "title": "Custom Integrations", - "section": "Kernels Integration", - "text": "Kernels Integration\nMoE (Mixture of Experts) kernels speed up training for MoE layers and reduce VRAM costs. In transformers v5, batched_mm and grouped_mm were integrated as built-in options via the experts_implementation config kwarg:\nclass ExpertsInterface(GeneralInterface):\n _global_mapping = {\n \"batched_mm\": batched_mm_experts_forward,\n \"grouped_mm\": grouped_mm_experts_forward,\n }\nIn our custom integration, we add support for ScatterMoE and SonicMoE, which are more efficient and faster than grouped_mm.\n\nUsage\nAdd the following to your axolotl YAML config:\nplugins:\n - axolotl.integrations.kernels.KernelsPlugin\n\nuse_kernels: true\n\nuse_scattermoe: true\nuse_sonicmoe: true\nImportant: Setting experts_implementation is incompatible with custom kernel options.\n\n\nSonicMoE installation\nPrerequisites:\n- NVIDIA Hopper (H100, H200) or Blackwell (B200, GB200) GPU\n- CUDA 12.9+ (13.0+ for B300)\n- PyTorch 2.7+ (2.9.1 recommended)\n- For B300: Triton 3.6.0\npip install --ignore-requires-python --no-deps \"sonic-moe @ git+https://github.com/Dao-AILab/sonic-moe.git@116e2df0a41874f77fa0ad269ce7df3f0cfcb956\" && pip install nvidia-cutlass-dsl==4.4.0 quack-kernels==0.2.5\nSee the SonicMoE installation guide for the latest prerequisite details.\nNote: Blackwell support is in upstream beta. On Blackwell GPUs, Axolotl automatically sets USE_QUACK_GEMM=1 to enable the Blackwell kernels.\n\n\nHow It Works\nThe KernelsPlugin runs before model loading and:\n\n\nScatterMoE\n\nRegisters the ScatterMoE kernel from the local libs/scattermoe_lora package (includes fused LoRA support via Triton kernels).\nPatches the model’s SparseMoeBlock forward method with the optimized ScatterMoE implementation.\n\n\n\nSonicMoE\n\nResolves the model’s MoE block class(es) from constants.py.\nPatches the forward method with SonicMoE’s optimized kernels and registers a weight converter for the interleaved gate/up projection format.\nSupports both softmax->topk and sigmoid->topk routing strategies.\n\nBoth paths use the shared resolve_moe_block_classes utility in constants.py for model-type-to-class resolution.\n\nSupported Models\nSee constants.py for the full list of supported model types (Qwen2-MoE, Qwen3-MoE, OLMoE, Mixtral, DeepSeek-V3, GLM-MoE, MiniMax, etc.).\n\n\n\nLimitations\nScatterMoE uses a softmax -> topk routing, so results may be different for some model architectures as baseline (GPT-OSS, etc). Incompatible with GLM_MOE_DSA (GLM 5) and GLM4_MOE_LITE (GLM 4.7 Flash) at the moment.\nSonicMoE supports both softmax->topk and sigmoid->topk routing, covering a wider range of architectures.\nScatterMoE does not work for GLM4.7 Flash (glm4_moe_lite) atm.\n\n\nNote on MegaBlocks\nWe tested MegaBlocks but were unable to ensure numerical accuracy, so we did not integrate it. It was also incompatible with many newer model architectures in transformers.\nPlease see reference here", + "objectID": "docs/vllm_serving.html#sec-restart", + "href": "docs/vllm_serving.html#sec-restart", + "title": "vLLM Serving for GRPO Training", + "section": "6 Restart Requirements", + "text": "6 Restart Requirements\n\n\n\n\n\n\nWarning\n\n\n\nvLLM must be restarted between training runs. Weight syncs from a previous run leave the server in a corrupted state. If you start a new training run against a stale vLLM server, the model may fail to learn.\n\n\n\n6.1 When to Restart\n\nBefore every new training experiment\nAfter a training run crashes or is interrupted\nIf you change the base model in your config\n\n\n\n6.2 How to Restart\nKilling vLLM reliably requires terminating both the main process and its background EngineCore subprocess:\n# Kill all vLLM-related processes\npkill -9 -f \"vllm|EngineCore\"\n\n# Verify GPU memory is freed\nnvidia-smi\n\n# Restart the server\nCUDA_VISIBLE_DEVICES=0 axolotl vllm-serve grpo_config.yaml\n\n\n\n\n\n\nTip\n\n\n\nA single kill often does not fully stop vLLM. Always use kill -9 and verify with nvidia-smi that GPU memory has been released before restarting.\n\n\n\n\n6.3 Health Check\nThe vLLM server exposes a health endpoint. Wait for it to return 200 before starting training:\n# For the LoRA serve script (trailing slash required)\ncurl http://localhost:8000/health/\n\n# For the default TRL serve script\ncurl http://localhost:8000/health", "crumbs": [ - "Advanced Features", - "Custom Integrations" + "How To Guides", + "vLLM Serving for GRPO Training" ] }, { - "objectID": "docs/custom_integrations.html#knowledge-distillation-kd", - "href": "docs/custom_integrations.html#knowledge-distillation-kd", - "title": "Custom Integrations", - "section": "Knowledge Distillation (KD)", - "text": "Knowledge Distillation (KD)\n\nUsage\nplugins:\n - \"axolotl.integrations.kd.KDPlugin\"\n\nkd_trainer: True\nkd_ce_alpha: 0.1\nkd_alpha: 0.9\nkd_temperature: 1.0\n\ntorch_compile: True # torch>=2.6.0, recommended to reduce vram\n\ndatasets:\n - path: ...\n type: \"axolotl.integrations.kd.chat_template\"\n field_messages: \"messages_combined\"\n logprobs_field: \"llm_text_generation_vllm_logprobs\" # for kd only, field of logprobs\nAn example dataset can be found at axolotl-ai-co/evolkit-logprobs-pipeline-75k-v2-sample\nPlease see reference here", + "objectID": "docs/vllm_serving.html#sec-config-reference", + "href": "docs/vllm_serving.html#sec-config-reference", + "title": "vLLM Serving for GRPO Training", + "section": "7 Configuration Reference", + "text": "7 Configuration Reference\n\n7.1 vLLM Server Options (vllm: section)\nThese control the vLLM server process started by axolotl vllm-serve.\n\n\n\n\n\n\n\n\n\nOption\nType\nDefault\nDescription\n\n\n\n\nhost\nstr\n0.0.0.0\nHost address for the vLLM server\n\n\nport\nint\n8000\nPort for the vLLM server\n\n\ndevice\nstr\nauto\nDevice to use for vLLM\n\n\ntensor_parallel_size\nint\nNone\nNumber of GPUs for tensor parallelism\n\n\ndata_parallel_size\nint\nNone\nNumber of data parallel replicas\n\n\ngpu_memory_utilization\nfloat\n0.9\nFraction of GPU memory for vLLM (0.0-1.0)\n\n\ndtype\nstr\nauto\nData type (auto, float16, bfloat16)\n\n\nmax_model_len\nint\nNone\nMaximum model context length. Set explicitly if the default is too large for your GPU\n\n\nenable_prefix_caching\nbool\nNone\nEnable prefix caching for repeated prompt prefixes\n\n\nenable_reasoning\nbool\nNone\nEnable reasoning mode for models with thinking tokens\n\n\nreasoning_parser\nstr\nNone\nParser for reasoning output\n\n\nenforce_eager\nbool\nNone\nDisable CUDA graph capture (required for some architectures like Qwen3.5 hybrid attention)\n\n\nserve_module\nstr\nNone\nPython module for vLLM serve script. Auto-set when vllm_lora_sync: true\n\n\nworker_extension_cls\nstr\nNone\nvLLM worker extension class for weight sync\n\n\n\n\n\n7.2 Trainer vLLM Options (trl: section)\nThese control how the trainer interacts with vLLM.\n\n\n\n\n\n\n\n\n\nOption\nType\nDefault\nDescription\n\n\n\n\nuse_vllm\nbool\nfalse\nEnable vLLM for generation\n\n\nvllm_mode\nstr\nNone\nserver (external process) or colocate (same GPU)\n\n\nvllm_server_host\nstr\n0.0.0.0\nHost of the vLLM server to connect to\n\n\nvllm_server_port\nint\n8000\nPort of the vLLM server to connect to\n\n\nvllm_server_timeout\nint\nNone\nTimeout in seconds for vLLM requests\n\n\nvllm_lora_sync\nbool\nfalse\nSync LoRA adapters via filesystem instead of NCCL merge\n\n\nvllm_sync_interval\nint\nNone\nSync weights every N optimizer steps\n\n\nvllm_enable_sleep_mode\nbool\nNone\nOffload vLLM VRAM when idle (colocate mode)\n\n\nvllm_guided_decoding_regex\nstr\nNone\nRegex constraint for guided decoding\n\n\n\nFor async pipeline and off-policy correction options, see the GRPO Configuration Reference.", "crumbs": [ - "Advanced Features", - "Custom Integrations" + "How To Guides", + "vLLM Serving for GRPO Training" ] }, { - "objectID": "docs/custom_integrations.html#llmcompressor", - "href": "docs/custom_integrations.html#llmcompressor", - "title": "Custom Integrations", - "section": "LLMCompressor", - "text": "LLMCompressor\nFine-tune sparsified models in Axolotl using Neural Magic’s LLMCompressor.\nThis integration enables fine-tuning of models sparsified using LLMCompressor within the Axolotl training framework. By combining LLMCompressor’s model compression capabilities with Axolotl’s distributed training pipelines, users can efficiently fine-tune sparse models at scale.\nIt uses Axolotl’s plugin system to hook into the fine-tuning flows while maintaining sparsity throughout training.\n\n\nRequirements\n\nAxolotl with llmcompressor extras:\npip install \"axolotl[llmcompressor]\"\nRequires llmcompressor >= 0.5.1\n\nThis will install all necessary dependencies to fine-tune sparsified models using the integration.\n\n\n\nUsage\nTo enable sparse fine-tuning with this integration, include the plugin in your Axolotl config:\nplugins:\n - axolotl.integrations.llm_compressor.LLMCompressorPlugin\n\nllmcompressor:\n recipe:\n finetuning_stage:\n finetuning_modifiers:\n ConstantPruningModifier:\n targets: [\n 're:.*q_proj.weight',\n 're:.*k_proj.weight',\n 're:.*v_proj.weight',\n 're:.*o_proj.weight',\n 're:.*gate_proj.weight',\n 're:.*up_proj.weight',\n 're:.*down_proj.weight',\n ]\n start: 0\n save_compressed: true\nThis plugin does not apply pruning or sparsification itself — it is intended for fine-tuning models that have already been sparsified.\nPre-sparsified checkpoints can be:\n- Generated using LLMCompressor\n- Downloaded from Neural Magic’s Hugging Face page\n- Any custom LLM with compatible sparsity patterns that you’ve created yourself\nTo learn more about writing and customizing LLMCompressor recipes, refer to the official documentation:\nhttps://github.com/vllm-project/llm-compressor/blob/main/README.md\n\n\nStorage Optimization with save_compressed\nSetting save_compressed: true in your configuration enables saving models in a compressed format, which:\n- Reduces disk space usage by approximately 40%\n- Maintains compatibility with vLLM for accelerated inference\n- Maintains compatibility with llmcompressor for further optimization (example: quantization)\nThis option is highly recommended when working with sparse models to maximize the benefits of model compression.\n\n\nExample Config\nSee examples/llama-3/sparse-finetuning.yaml for a complete example.\n\n\n\nInference with vLLM\nAfter fine-tuning your sparse model, you can leverage vLLM for efficient inference.\nYou can also use LLMCompressor to apply additional quantization to your fine-tuned\nsparse model before inference for even greater performance benefits.:\nfrom vllm import LLM, SamplingParams\n\nprompts = [\n \"Hello, my name is\",\n \"The president of the United States is\",\n \"The capital of France is\",\n \"The future of AI is\",\n]\nsampling_params = SamplingParams(temperature=0.8, top_p=0.95)\nllm = LLM(\"path/to/your/sparse/model\")\noutputs = llm.generate(prompts, sampling_params)\n\nfor output in outputs:\n prompt = output.prompt\n generated_text = output.outputs[0].text\n print(f\"Prompt: {prompt!r}, Generated text: {generated_text!r}\")\nFor more details on vLLM’s capabilities and advanced configuration options, see the official vLLM documentation.\n\n\nLearn More\nFor details on available sparsity and quantization schemes, fine-tuning recipes, and usage examples, visit the official LLMCompressor repository:\nhttps://github.com/vllm-project/llm-compressor\nPlease see reference here", + "objectID": "docs/vllm_serving.html#sec-complete-example", + "href": "docs/vllm_serving.html#sec-complete-example", + "title": "vLLM Serving for GRPO Training", + "section": "8 Complete Example", + "text": "8 Complete Example\nFor a full working GRPO config including vLLM, LoRA sync, async generation, rewards, and dataset setup, see the GRPO Quick Start. That config includes all the vLLM settings covered in this guide.\n# Terminal 1: Start vLLM\nCUDA_VISIBLE_DEVICES=0 axolotl vllm-serve grpo_config.yaml\n\n# Wait for health check to pass\ncurl http://localhost:8000/health/\n\n# Terminal 2: Start training\nCUDA_VISIBLE_DEVICES=1 axolotl train grpo_config.yaml", "crumbs": [ - "Advanced Features", - "Custom Integrations" + "How To Guides", + "vLLM Serving for GRPO Training" ] }, { - "objectID": "docs/custom_integrations.html#language-model-evaluation-harness-lm-eval", - "href": "docs/custom_integrations.html#language-model-evaluation-harness-lm-eval", - "title": "Custom Integrations", - "section": "Language Model Evaluation Harness (LM Eval)", - "text": "Language Model Evaluation Harness (LM Eval)\nRun evaluation on model using the popular lm-evaluation-harness library.\nSee https://github.com/EleutherAI/lm-evaluation-harness\n\nUsage\nThere are two ways to use the LM Eval integration:\n\n\n1. Post-Training Evaluation\nWhen training with the plugin enabled, evaluation runs automatically after training completes:\nplugins:\n - axolotl.integrations.lm_eval.LMEvalPlugin\n\nlm_eval_tasks:\n - gsm8k\n - hellaswag\n - arc_easy\n\nlm_eval_batch_size: # Batch size for evaluation\n\noutput_dir:\nRun training as usual:\naxolotl train config.yml\n\n\n2. Standalone CLI Evaluation\nEvaluate any model directly without training:\nlm_eval_model: meta-llama/Llama-2-7b-hf\n\nplugins:\n - axolotl.integrations.lm_eval.LMEvalPlugin\n\nlm_eval_tasks:\n - gsm8k\n - hellaswag\n - arc_easy\n\nlm_eval_batch_size: 8\noutput_dir: ./outputs\nRun evaluation:\naxolotl lm-eval config.yml\n\n\nModel Selection Priority\nThe model to evaluate is selected in the following priority order:\n\nlm_eval_model - Explicit model path or HuggingFace repo (highest priority)\nhub_model_id - Trained model pushed to HuggingFace Hub\noutput_dir - Local checkpoint directory containing trained model weights\n\n\n\nCitation\n@misc{eval-harness,\n author = {Gao, Leo and Tow, Jonathan and Abbasi, Baber and Biderman, Stella and Black, Sid and DiPofi, Anthony and Foster, Charles and Golding, Laurence and Hsu, Jeffrey and Le Noac'h, Alain and Li, Haonan and McDonell, Kyle and Muennighoff, Niklas and Ociepa, Chris and Phang, Jason and Reynolds, Laria and Schoelkopf, Hailey and Skowron, Aviya and Sutawika, Lintang and Tang, Eric and Thite, Anish and Wang, Ben and Wang, Kevin and Zou, Andy},\n title = {A framework for few-shot language model evaluation},\n month = 07,\n year = 2024,\n publisher = {Zenodo},\n version = {v0.4.3},\n doi = {10.5281/zenodo.12608602},\n url = {https://zenodo.org/records/12608602}\n}\nPlease see reference here", + "objectID": "docs/vllm_serving.html#sec-troubleshooting", + "href": "docs/vllm_serving.html#sec-troubleshooting", + "title": "vLLM Serving for GRPO Training", + "section": "9 Troubleshooting", + "text": "9 Troubleshooting\n\n\n\n\n\n\n\n\nProblem\nLikely Cause\nSolution\n\n\n\n\nTraining hangs waiting for vLLM\nServer not started or wrong port\nCheck curl http://localhost:8000/health/ and verify vllm_server_host/vllm_server_port match\n\n\nOOM on vLLM GPU\ngpu_memory_utilization too high or max_model_len too large\nReduce gpu_memory_utilization to 0.7 or set max_model_len explicitly\n\n\nOOM on training GPU\nBatch too large for policy logprobs\nReduce micro_batch_size or num_generations\n\n\nAccuracy stays at zero\nStale vLLM from previous run\nRestart vLLM: pkill -9 -f \"vllm\\|EngineCore\", verify with nvidia-smi, restart\n\n\nResponseValidationError from vLLM\nMissing logprobs in response\nEnsure you are using the correct serve module (auto-selected with vllm_lora_sync: true)\n\n\nWeight sync takes 5+ seconds\nNCCL contention with vLLM generation\nSwitch to vllm_lora_sync: true to eliminate NCCL\n\n\nasync_prefetch deadlocks with FSDP\nBackground threads run unsynchronized FSDP collectives\nSet async_prefetch: false when using FSDP or DeepSpeed multi-GPU", "crumbs": [ - "Advanced Features", - "Custom Integrations" + "How To Guides", + "vLLM Serving for GRPO Training" ] }, { - "objectID": "docs/custom_integrations.html#liger-kernels", - "href": "docs/custom_integrations.html#liger-kernels", - "title": "Custom Integrations", - "section": "Liger Kernels", - "text": "Liger Kernels\nLiger Kernel provides efficient Triton kernels for LLM training, offering:\n\n20% increase in multi-GPU training throughput\n60% reduction in memory usage\nCompatibility with both FSDP and DeepSpeed\n\nSee https://github.com/linkedin/Liger-Kernel\n\nUsage\nplugins:\n - axolotl.integrations.liger.LigerPlugin\nliger_rope: true\nliger_rms_norm: true\nliger_glu_activation: true\nliger_layer_norm: true\nliger_fused_linear_cross_entropy: true\n\nliger_use_token_scaling: true\n\n\nSupported Models\n\ndeepseek_v2\ngemma\ngemma2\ngemma3\ngranite\njamba\nllama\nmistral\nmixtral\nmllama\nmllama_text_model\nolmo2\npaligemma\nphi3\nqwen2\nqwen2_5_vl\nqwen2_vl\n\n\n\nCitation\n@article{hsu2024ligerkernelefficienttriton,\n title={Liger Kernel: Efficient Triton Kernels for LLM Training},\n author={Pin-Lun Hsu and Yun Dai and Vignesh Kothapalli and Qingquan Song and Shao Tang and Siyu Zhu and Steven Shimizu and Shivam Sahni and Haowen Ning and Yanning Chen},\n year={2024},\n eprint={2410.10989},\n archivePrefix={arXiv},\n primaryClass={cs.LG},\n url={https://arxiv.org/abs/2410.10989},\n journal={arXiv preprint arXiv:2410.10989},\n}\nPlease see reference here", + "objectID": "docs/optimizers.html", + "href": "docs/optimizers.html", + "title": "Optimizers", + "section": "", + "text": "Axolotl supports all optimizers supported by transformers OptimizerNames\nHere is a list of optimizers supported by transformers as of v4.54.0:\n\nadamw_torch\nadamw_torch_fused\nadamw_torch_xla\nadamw_torch_npu_fused\nadamw_apex_fused\nadafactor\nadamw_anyprecision\nadamw_torch_4bit\nadamw_torch_8bit\nademamix\nsgd\nadagrad\nadamw_bnb_8bit\nadamw_8bit # alias for adamw_bnb_8bit\nademamix_8bit\nlion_8bit\nlion_32bit\npaged_adamw_32bit\npaged_adamw_8bit\npaged_ademamix_32bit\npaged_ademamix_8bit\npaged_lion_32bit\npaged_lion_8bit\nrmsprop\nrmsprop_bnb\nrmsprop_bnb_8bit\nrmsprop_bnb_32bit\ngalore_adamw\ngalore_adamw_8bit\ngalore_adafactor\ngalore_adamw_layerwise\ngalore_adamw_8bit_layerwise\ngalore_adafactor_layerwise\nlomo\nadalomo\ngrokadamw\nschedule_free_radam\nschedule_free_adamw\nschedule_free_sgd\napollo_adamw\napollo_adamw_layerwise\nstable_adamw", "crumbs": [ - "Advanced Features", - "Custom Integrations" + "Core Concepts", + "Optimizers" ] }, { - "objectID": "docs/custom_integrations.html#nemo-gym-integration-for-axolotl", - "href": "docs/custom_integrations.html#nemo-gym-integration-for-axolotl", - "title": "Custom Integrations", - "section": "NeMo Gym Integration for Axolotl", - "text": "NeMo Gym Integration for Axolotl\nTrain LLMs with reinforcement learning using NVIDIA NeMo Gym environments as reward sources. NeMo Gym provides 50+ verified RL environments spanning math, coding, tool-use, reasoning, and safety — each with deterministic reward signals.\n\nValidated Training Paths\n\n\n\n\n\n\n\n\n\nPath\nSpeed\nMulti-turn\nArchitecture\n\n\n\n\nAsync GRPO + Data Producer\nFastest (3x)\nYes\nNemoGymDataProducer replaces vLLM generation\n\n\nStandard GRPO + Data Producer\nBaseline\nYes\nSame producer, no async prefetch\n\n\nStandard GRPO + /verify\nSimplest\nNo\nReward function calls /verify directly\n\n\nFSDP2 + /verify (2 GPU)\nDistributed\nNo\nfsdp_version: 2\n\n\n\nMulti-turn uses nemo_gym_multi_turn: true which auto-enables the async trainer’s\ndata producer protocol. The plugin’s NemoGymDataProducer calls NeMo Gym agent /run\nendpoints and returns RolloutDataset with proper IS correction, env_mask, and rewards.\nAll paths tested end-to-end with Qwen3-0.6B + LoRA, logged to wandb project nemo-gym-rl.\n\n\nQuick Start\n\n\nPrerequisites\n\nuv package manager (for NeMo Gym’s venv)\nTwo GPUs recommended (one for vLLM server, one for training)\n\n\n\n1. Set Up NeMo Gym\ngit clone https://github.com/NVIDIA-NeMo/Gym.git ~/Gym\ncd ~/Gym\nuv venv --python 3.12 && source .venv/bin/activate && uv sync\n\nCFLAGS=\"\" uv pip install pycosat --python .venv/bin/python --no-build-isolation\n\nfor dir in resources_servers/reasoning_gym resources_servers/example_single_tool_call responses_api_models/vllm_model responses_api_agents/simple_agent; do\n uv venv --seed --allow-existing --python 3.12 $dir/.venv\n CFLAGS=\"\" uv pip install --python $dir/.venv/bin/python pycosat --no-build-isolation 2>/dev/null\n uv pip install --python $dir/.venv/bin/python -e . \"ray[default]==2.52.1\"\ndone\n\nuv pip install --python resources_servers/reasoning_gym/.venv/bin/python \\\n reasoning-gym matplotlib pillow cycler contourpy kiwisolver\n\n\n2. Multi-Turn with Async GRPO (Recommended — Fastest Path)\nThis is the fully validated, highest-performance path. NeMo Gym’s agent server handles\nmulti-turn tool execution while axolotl’s async GRPO prefetches data in background threads.\nStep 1: Create the NeMo Gym agent config\nCreate ~/Gym/configs/axolotl_tool_calling.yaml:\nexample_single_tool_call:\n resources_servers:\n example_single_tool_call:\n entrypoint: app.py\n domain: agent\n verified: false\n\npolicy_model:\n responses_api_models:\n vllm_model:\n entrypoint: app.py\n base_url: http://localhost:8000/v1\n api_key: dummy_key\n model: Qwen/Qwen3-0.6B # Must match your training model\n return_token_id_information: true\n uses_reasoning_parser: false\n\nexample_single_tool_call_simple_agent:\n responses_api_agents:\n simple_agent:\n entrypoint: app.py\n resources_server:\n type: resources_servers\n name: example_single_tool_call\n model_server:\n type: responses_api_models\n name: policy_model\n datasets:\n - name: weather\n type: example\n jsonl_fpath: resources_servers/example_single_tool_call/data/weather_tool_calling.jsonl\nStep 2: Start three services\nCUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \\\n --model Qwen/Qwen3-0.6B --max-model-len 2048 --gpu-memory-utilization 0.85\n\ncd ~/Gym && .venv/bin/ng_run \\\n \"+config_paths=[configs/axolotl_tool_calling.yaml]\" \"+skip_venv_if_present=true\"\n\ncd experiments && CUDA_VISIBLE_DEVICES=1 CUDA_HOME=$HOME/env-claude-cu130/cuda_shim \\\n axolotl train nemo_gym_async_agent.yaml\nStep 3: Training config (nemo_gym_async_agent.yaml):\nbase_model: Qwen/Qwen3-0.6B\nadapter: lora\nlora_r: 16\nlora_alpha: 32\nlora_target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]\nsequence_len: 2048\n\nrl: grpo\nchat_template: tokenizer_default\n\ntrl:\n use_vllm: true\n vllm_mode: server\n vllm_server_host: localhost\n vllm_server_port: 8000\n vllm_lora_sync: true\n vllm_sync_interval: 5\n # Async GRPO — 3x faster than standard\n use_data_producer: true\n async_prefetch: true\n num_generations: 4\n max_completion_length: 512\n temperature: 0.8\n reward_funcs:\n - axolotl.integrations.nemo_gym.rewards.reward_env\n\nplugins:\n - axolotl.integrations.nemo_gym.NemoGymPlugin\n\nnemo_gym_enabled: true\nnemo_gym_auto_start: false\nnemo_gym_head_port: 11000\nnemo_gym_multi_turn: true\nnemo_gym_verify_timeout: 120\nnemo_gym_datasets:\n - path: ~/Gym/resources_servers/example_single_tool_call/data/weather_tool_calling.jsonl\n server_name: example_single_tool_call\n\ndatasets:\n - path: ~/Gym/resources_servers/example_single_tool_call/data/weather_tool_calling.jsonl\n type: chat_template\n field_messages: responses_create_params.input\n message_field_content: content\n message_field_role: role\n\nvllm:\n gpu_memory_utilization: 0.85\n max_model_len: 2048\n tensor_parallel_size: 1\n\nlearning_rate: 5e-6\nmicro_batch_size: 1\ngradient_accumulation_steps: 4\nmax_steps: 30\ngradient_checkpointing: true\nbf16: true\noutput_dir: ./outputs/nemo_gym_async\n\nuse_wandb: true\nwandb_project: nemo-gym-rl\n\n\n3. Single-Turn Training (Simplest — No Agent Server Needed)\nFor environments that only need single-turn verify (math, coding challenges), you don’t need\nan agent server. The plugin’s reward function calls /verify directly.\nbase_model: Qwen/Qwen2.5-0.5B-Instruct\nrl: grpo\nchat_template: tokenizer_default\n\ntrl:\n use_vllm: true\n vllm_mode: colocate\n vllm_enable_sleep_mode: false\n num_generations: 8\n max_completion_length: 128\n temperature: 0.9\n reward_funcs:\n - axolotl.integrations.nemo_gym.rewards.reward_nemo_gym_verify\n\nplugins:\n - axolotl.integrations.nemo_gym.NemoGymPlugin\n\nnemo_gym_enabled: true\nnemo_gym_auto_start: false\nnemo_gym_head_port: 11000\nnemo_gym_datasets:\n - path: ~/Gym/resources_servers/reasoning_gym/data/train_basic_arithmetic.jsonl\n server_name: reasoning_gym\n\ndatasets:\n - path: ~/Gym/resources_servers/reasoning_gym/data/train_basic_arithmetic.jsonl\n type: chat_template\n field_messages: responses_create_params.input\n message_field_content: content\n message_field_role: role\n\nvllm:\n gpu_memory_utilization: 0.3\n max_model_len: 512\n tensor_parallel_size: 1\n\nlearning_rate: 1e-5\nmicro_batch_size: 4\ngradient_accumulation_steps: 2\nmax_steps: 50\noutput_dir: ./outputs/nemo_gym_arithmetic\nOnly needs ng_run with resource servers (no agent config):\ncd ~/Gym && ng_run \"+config_paths=[resources_servers/reasoning_gym/configs/resources_only.yaml]\" \"+skip_venv_if_present=true\"\n\n\nHow It Works\n\n\nSingle-Turn\naxolotl train → GRPO Trainer generates completions\n → NeMo Gym plugin reward_fn calls POST /verify on resource server\n → reward flows back to GRPO for advantage computation\n\n\nMulti-Turn (Agent /run)\n┌─────────────┐ ┌──────────────┐ ┌──────────────────┐\n│ axolotl │ │ NeMo Gym │────▶│ vLLM OpenAI │\n│ train │────▶│ Agent /run │◀────│ Server (GPU 0) │\n│ (GPU 1) │ │ │ │ /v1/completions │\n└─────────────┘ └──────┬───────┘ └──────────────────┘\n │\n ▼\n ┌──────────────┐\n │ Resource │\n │ Server │\n │ (tools + │\n │ verify) │\n └─────────────┘\nThe agent server orchestrates the entire multi-turn loop:\n1. Calls our vLLM server for model generation\n2. Parses tool calls from model output\n3. Executes tools against resource servers\n4. Feeds tool results back to the model\n5. Repeats until done, then calls /verify for reward\n6. Returns token IDs + logprobs + reward to our rollout_func\n\n\nData Producer Architecture (Multi-Turn)\nWhen nemo_gym_multi_turn: true, the plugin automatically forces use_data_producer: true\nwhich selects the AxolotlAsyncGRPOTrainer. The plugin then swaps the trainer’s data\nproducer with NemoGymDataProducer, which:\n\nGets a prompt batch from the dataset iterator\nExpands by num_generations (one agent call per rollout)\nCalls NeMo Gym agents via async HTTP (aiohttp.gather)\nParses responses into padded tensors (RolloutDataset)\nReturns with _pending_policy_logps=True for deferred scoring\n\nThe main thread then runs _compute_deferred_scores() which:\n- Computes policy logprobs on the training model (GPU forward pass)\n- Computes IS correction using agent’s sampling logprobs vs training model logprobs\n- Computes advantages with group-level normalization\n- All downstream features work: replay buffer, re-roll, streaming, zero-adv skip\nWith async_prefetch: true, the data producer runs in a background thread — giving ~3x\nspeedup as generation and training overlap. With async_prefetch: false, it runs\nsynchronously on the main thread (still uses the data producer protocol).\n\n\nWeight Sync (LoRA Mode)\nWith vllm_lora_sync: true, the plugin (or async trainer) replaces NCCL-based weight\nsync with filesystem + HTTP:\n\naccelerator.get_state_dict() gathers LoRA weights from all ranks\nRank 0 saves adapter to /tmp/lora_sync_*/vN/\nRank 0 POSTs to /set_lora_adapter/ on vLLM server\nvLLM loads adapter natively via Punica kernels\nOnly ~40MB transferred (vs multiple GBs for full model weights)\n\n\n\nMulti-Environment Support\nDatasets support per-row environment routing via agent_ref:\n{\"agent_ref\": {\"name\": \"reasoning_gym\"}, \"responses_create_params\": {...}}\n{\"agent_ref\": {\"name\": \"instruction_following\"}, \"responses_create_params\": {...}}\nOr use the simpler per-dataset routing:\nnemo_gym_datasets:\n - path: reasoning_data.jsonl\n server_name: reasoning_gym\n - path: tool_data.jsonl\n server_name: example_single_tool_call\n\n\nConfiguration Reference\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nnemo_gym_enabled\nbool\nnull\nEnable the NeMo Gym integration\n\n\nnemo_gym_dir\nstr\n~/Gym\nPath to NeMo Gym repo\n\n\nnemo_gym_auto_clone\nbool\ntrue\nAuto-clone NeMo Gym repo if missing\n\n\nnemo_gym_auto_start\nbool\ntrue\nAuto-start resource servers\n\n\nnemo_gym_config_paths\nlist[str]\n—\nServer config YAMLs (relative to gym_dir)\n\n\nnemo_gym_datasets\nlist[dict]\nrequired\nDataset configs with path and optional server_name\n\n\nnemo_gym_head_port\nint\n11000\nHead server port\n\n\nnemo_gym_server_timeout\nint\n360\nServer startup timeout (seconds)\n\n\nnemo_gym_verify_timeout\nint\n30\nPer-request timeout (seconds)\n\n\nnemo_gym_multi_turn\nbool\nfalse\nEnable multi-turn via agent /run\n\n\n\n\n\nDataset JSONL Format\nEach line must have responses_create_params with input messages:\n{\n \"responses_create_params\": {\n \"input\": [{\"role\": \"user\", \"content\": \"What's the weather in SF?\"}],\n \"tools\": [{\"name\": \"get_weather\", \"type\": \"function\", \"strict\": true, \"parameters\": {...}}]\n }\n}\nFor multi-turn agent routing, include agent_ref:\n{\"agent_ref\": {\"name\": \"my_agent\"}, \"responses_create_params\": {...}}\nNote: Tool definitions MUST include \"strict\": true and \"additionalProperties\": false for NeMo Gym agent compatibility.\n\n\nReward Functions\nThe plugin provides two built-in reward functions — no user code needed:\ntrl:\n reward_funcs:\n # Multi-turn (nemo_gym_multi_turn: true):\n # Passthrough — agent /run already computed the reward\n - axolotl.integrations.nemo_gym.rewards.reward_env\n\n # Single-turn (nemo_gym_multi_turn: false):\n # Calls /verify endpoints on NeMo Gym resource servers\n - axolotl.integrations.nemo_gym.rewards.reward_nemo_gym_verify\nBoth are also importable from Python:\nfrom axolotl.integrations.nemo_gym import reward_env, reward_nemo_gym_verify\n\n\nKnown Issues / Troubleshooting\n\n\nNeMo Gym Server Setup\n\npycosat build failure: CFLAGS=\"\" uv pip install pycosat --no-build-isolation\nRay version mismatch: Pin ray[default]==2.52.1 in all server venvs\nPre-build venvs: ng_run creates per-server venvs via Ray. Pre-build them and use +skip_venv_if_present=true\nTool strict field required: Agent server validates tool definitions require strict: true\n\n\n\nvLLM / Weight Sync\n\nStart vLLM with LoRA + tool calling + runtime loading:\nVLLM_ALLOW_RUNTIME_LORA_UPDATING=1 \\\nCUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \\\n --model Qwen/Qwen3-4B-Instruct-2507 \\\n --max-model-len 4096 \\\n --gpu-memory-utilization 0.7 \\\n --enable-lora --max-lora-rank 64 \\\n --enable-auto-tool-choice --tool-call-parser hermes\nVLLM_ALLOW_RUNTIME_LORA_UPDATING=1: Required for vllm_lora_sync: true. Without it, vLLM won’t expose the /v1/load_lora_adapter endpoint and weight sync will fail silently. The plugin warns if this endpoint is missing.\n--enable-lora: Enables LoRA adapter support in vLLM\n--enable-auto-tool-choice --tool-call-parser hermes: Required for Qwen3 tool calling\nmax_model_len must be > max_completion_length: Leave room for prompt tokens (~200). If equal, the NeMo Gym model proxy gets a 400 error and returns empty completions.\nCUDA_HOME required: DeepSpeed import needs it for the nvcc shim\nNCCL weight sync broken with vLLM 0.17: Use vllm_lora_sync: true (filesystem + HTTP via /v1/load_lora_adapter)\n\n\n\nMulti-Turn\n\nAgent server required: Multi-turn delegates to NeMo Gym’s agent server /run endpoint. Without an agent, the plugin falls back to single-turn /verify\nModel server proxy: NeMo Gym needs a responses_api_models server that proxies to your vLLM. See the agent config example above\n\n\n\nFSDP2\n\nValidated on 2 GPUs with single-turn + LoRA\nAsync field filtering: The builder automatically filters async-only config fields when using the standard GRPO trainer\n\n\n\nComparison with Other Integrations\n\n\n\n\n\n\n\n\n\nFeature\nAxolotl + NeMo Gym\nUnsloth + NeMo Gym\nNeMo RL (native)\n\n\n\n\nServer management\nAutomatic\nManual (notebook)\nBuilt-in\n\n\nMulti-environment\nPer-row routing\nManual code\nYAML config\n\n\nMulti-turn / tool use\nAgent /run delegation\nNo\nAgent /run (Ray)\n\n\nAsync GRPO (3x speedup)\nYes\nNo\nYes\n\n\nLoRA sync\nFilesystem + HTTP\nN/A\nNCCL\n\n\nMulti-GPU (FSDP2)\nYes\nNo\nYes (Ray)\n\n\nConfig-driven\nYes\nNo (code)\nYes\n\n\n\nPlease see reference here", + "objectID": "docs/optimizers.html#overview", + "href": "docs/optimizers.html#overview", + "title": "Optimizers", + "section": "", + "text": "Axolotl supports all optimizers supported by transformers OptimizerNames\nHere is a list of optimizers supported by transformers as of v4.54.0:\n\nadamw_torch\nadamw_torch_fused\nadamw_torch_xla\nadamw_torch_npu_fused\nadamw_apex_fused\nadafactor\nadamw_anyprecision\nadamw_torch_4bit\nadamw_torch_8bit\nademamix\nsgd\nadagrad\nadamw_bnb_8bit\nadamw_8bit # alias for adamw_bnb_8bit\nademamix_8bit\nlion_8bit\nlion_32bit\npaged_adamw_32bit\npaged_adamw_8bit\npaged_ademamix_32bit\npaged_ademamix_8bit\npaged_lion_32bit\npaged_lion_8bit\nrmsprop\nrmsprop_bnb\nrmsprop_bnb_8bit\nrmsprop_bnb_32bit\ngalore_adamw\ngalore_adamw_8bit\ngalore_adafactor\ngalore_adamw_layerwise\ngalore_adamw_8bit_layerwise\ngalore_adafactor_layerwise\nlomo\nadalomo\ngrokadamw\nschedule_free_radam\nschedule_free_adamw\nschedule_free_sgd\napollo_adamw\napollo_adamw_layerwise\nstable_adamw", "crumbs": [ - "Advanced Features", - "Custom Integrations" + "Core Concepts", + "Optimizers" ] }, { - "objectID": "docs/custom_integrations.html#spectrum", - "href": "docs/custom_integrations.html#spectrum", - "title": "Custom Integrations", - "section": "Spectrum", - "text": "Spectrum\nby Eric Hartford, Lucas Atkins, Fernando Fernandes, David Golchinfar\nThis plugin contains code to freeze the bottom fraction of modules in a model, based on the Signal-to-Noise Ratio (SNR).\nSee https://github.com/cognitivecomputations/spectrum\n\nOverview\nSpectrum is a tool for scanning and evaluating the Signal-to-Noise Ratio (SNR) of layers in large language models.\nBy identifying the top n% of layers with the highest SNR, you can optimize training efficiency.\n\n\nUsage\nplugins:\n - axolotl.integrations.spectrum.SpectrumPlugin\n\nspectrum_top_fraction: 0.5\nspectrum_model_name: meta-llama/Meta-Llama-3.1-8B\n\n\nCitation\n@misc{hartford2024spectrumtargetedtrainingsignal,\n title={Spectrum: Targeted Training on Signal to Noise Ratio},\n author={Eric Hartford and Lucas Atkins and Fernando Fernandes Neto and David Golchinfar},\n year={2024},\n eprint={2406.06623},\n archivePrefix={arXiv},\n primaryClass={cs.LG},\n url={https://arxiv.org/abs/2406.06623},\n}\nPlease see reference here", + "objectID": "docs/optimizers.html#custom-optimizers", + "href": "docs/optimizers.html#custom-optimizers", + "title": "Optimizers", + "section": "Custom Optimizers", + "text": "Custom Optimizers\nEnable custom optimizers by passing a string to the optimizer argument. Each optimizer will receive beta and epsilon args, however, some may accept additional args which are detailed below.\n\noptimi_adamw\noptimizer: optimi_adamw\n\n\nao_adamw_4bit\nDeprecated: Please use adamw_torch_4bit.\n\n\nao_adamw_8bit\nDeprecated: Please use adamw_torch_8bit.\n\n\nao_adamw_fp8\noptimizer: ao_adamw_fp8\n\n\nadopt_adamw\nGitHub: https://github.com/iShohei220/adopt\nPaper: https://arxiv.org/abs/2411.02853\noptimizer: adopt_adamw\n\n\ncame_pytorch\nGitHub: https://github.com/yangluo7/CAME/tree/master\nPaper: https://arxiv.org/abs/2307.02047\noptimizer: came_pytorch\n\n# optional args (defaults below)\nadam_beta1: 0.9\nadam_beta2: 0.999\nadam_beta3: 0.9999\nadam_epsilon: 1e-30\nadam_epsilon2: 1e-16\n\n\nmuon\nBlog: https://kellerjordan.github.io/posts/muon/\nPaper: https://arxiv.org/abs/2502.16982v1\noptimizer: muon\n\n\ndion\nMicrosoft’s Dion (DIstributed OrthoNormalization) optimizer is a scalable and communication-efficient\northonormalizing optimizer that uses low-rank approximations to reduce gradient communication.\nGitHub: https://github.com/microsoft/dion\nPaper: https://arxiv.org/pdf/2504.05295\nNote: Implementation written for PyTorch 2.7+ for DTensor\noptimizer: dion\ndion_lr: 0.01\ndion_momentum: 0.95\nlr: 0.00001 # learning rate for embeddings and parameters that fallback to AdamW", "crumbs": [ - "Advanced Features", - "Custom Integrations" + "Core Concepts", + "Optimizers" ] }, { - "objectID": "docs/custom_integrations.html#swanlab-integration-for-axolotl", - "href": "docs/custom_integrations.html#swanlab-integration-for-axolotl", - "title": "Custom Integrations", - "section": "SwanLab Integration for Axolotl", - "text": "SwanLab Integration for Axolotl\nSwanLab is an open-source, lightweight AI experiment tracking and visualization tool that provides a platform for tracking, recording, comparing, and collaborating on experiments.\nThis integration enables seamless experiment tracking and visualization of Axolotl training runs using SwanLab.\n\nFeatures\n\n📊 Automatic Metrics Logging: Training loss, learning rate, and other metrics are automatically logged\n🎯 Hyperparameter Tracking: Model configuration and training parameters are tracked\n📈 Real-time Visualization: Monitor training progress in real-time through SwanLab dashboard\n☁️ Cloud & Local Support: Works in both cloud-synced and offline modes\n🔄 Experiment Comparison: Compare multiple training runs easily\n🤝 Team Collaboration: Share experiments with team members\n🎭 RLHF Completion Logging: Automatically log model outputs during DPO/KTO/ORPO/GRPO training for qualitative analysis\n⚡ Performance Profiling: Built-in profiling decorators to measure and optimize training performance\n🔔 Lark Notifications: Send real-time training updates to team chat (Feishu/Lark integration)\n\n\n\nInstallation\npip install swanlab\n\n\nQuick Start\n\n\n1. Register for SwanLab (Optional for cloud mode)\nIf you want to use cloud sync features, register at https://swanlab.cn to get your API key.\n\n\n2. Configure Axolotl Config File\nAdd SwanLab configuration to your Axolotl YAML config:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: my-llm-project\nswanlab_experiment_name: qwen-finetune-v1\nswanlab_mode: cloud # Options: cloud, local, offline, disabled\nswanlab_workspace: my-team # Optional: organization name\nswanlab_api_key: YOUR_API_KEY # Optional: can also use env var SWANLAB_API_KEY\n\n\n3. Run Training\nexport SWANLAB_API_KEY=your-api-key-here\n\nswanlab login\n\naccelerate launch -m axolotl.cli.train your-config.yaml\n\n\nConfiguration Options\n\n\nBasic Configuration\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nuse_swanlab\nbool\nfalse\nEnable SwanLab tracking\n\n\nswanlab_project\nstr\nNone\nProject name (required)\n\n\nswanlab_experiment_name\nstr\nNone\nExperiment name\n\n\nswanlab_description\nstr\nNone\nExperiment description\n\n\nswanlab_mode\nstr\ncloud\nSync mode: cloud, local, offline, disabled\n\n\n\n\n\nAdvanced Configuration\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nswanlab_workspace\nstr\nNone\nWorkspace/organization name\n\n\nswanlab_api_key\nstr\nNone\nAPI key (prefer env var)\n\n\nswanlab_web_host\nstr\nNone\nPrivate deployment web host\n\n\nswanlab_api_host\nstr\nNone\nPrivate deployment API host\n\n\nswanlab_log_model\nbool\nfalse\nLog model checkpoints (coming soon)\n\n\nswanlab_lark_webhook_url\nstr\nNone\nLark (Feishu) webhook URL for team notifications\n\n\nswanlab_lark_secret\nstr\nNone\nLark webhook HMAC secret for authentication\n\n\nswanlab_log_completions\nbool\ntrue\nEnable RLHF completion table logging (DPO/KTO/ORPO/GRPO)\n\n\nswanlab_completion_log_interval\nint\n100\nSteps between completion logging\n\n\nswanlab_completion_max_buffer\nint\n128\nMax completions to buffer (memory bound)\n\n\n\n\n\nConfiguration Examples\n\n\nExample 1: Basic Cloud Sync\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: llama-finetune\nswanlab_experiment_name: llama-3-8b-instruct-v1\nswanlab_mode: cloud\n\n\nExample 2: Offline/Local Mode\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: local-experiments\nswanlab_experiment_name: test-run-1\nswanlab_mode: local # or 'offline'\n\n\nExample 3: Team Workspace\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: research-project\nswanlab_experiment_name: experiment-42\nswanlab_workspace: my-research-team\nswanlab_mode: cloud\n\n\nExample 4: Private Deployment\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: internal-project\nswanlab_experiment_name: secure-training\nswanlab_mode: cloud\nswanlab_web_host: https://swanlab.yourcompany.com\nswanlab_api_host: https://api.swanlab.yourcompany.com\n\n\nTeam Notifications with Lark (Feishu)\nSwanLab supports sending real-time training notifications to your team chat via Lark (Feishu), ByteDance’s enterprise collaboration platform. This is especially useful for:\n- Production training monitoring: Get alerts when training starts, completes, or encounters errors\n- Team collaboration: Keep your ML team informed about long-running experiments\n- Multi-timezone teams: Team members can check training progress without being online\n\n\nPrerequisites\n\nLark Bot Setup: Create a custom bot in your Lark group chat\nWebhook URL: Get the webhook URL from your Lark bot settings\nHMAC Secret (recommended): Enable signature verification in your Lark bot for security\n\nFor detailed Lark bot setup instructions, see Lark Custom Bot Documentation.\n\n\nExample 5: Basic Lark Notifications\nSend training notifications to a Lark group chat:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: production-training\nswanlab_experiment_name: llama-3-finetune-v2\nswanlab_mode: cloud\n\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxx\nNote: This configuration will work, but you’ll see a security warning recommending HMAC secret configuration.\n\n\nExample 6: Lark Notifications with HMAC Security (Recommended)\nFor production use, enable HMAC signature verification:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: production-training\nswanlab_experiment_name: llama-3-finetune-v2\nswanlab_mode: cloud\n\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxx\nswanlab_lark_secret: your-webhook-secret-key\nWhy HMAC secret matters:\n- Prevents unauthorized parties from sending fake notifications to your Lark group\n- Ensures notifications genuinely come from your training jobs\n- Required for production deployments with sensitive training data\n\n\nExample 7: Team Workspace + Lark Notifications\nCombine team workspace collaboration with Lark notifications:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: research-project\nswanlab_experiment_name: multimodal-experiment-42\nswanlab_workspace: ml-research-team\nswanlab_mode: cloud\n\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxx\nswanlab_lark_secret: your-webhook-secret-key\n\n\nWhat Notifications Are Sent?\nSwanLab’s Lark integration sends notifications for key training events:\n- Training Start: When your experiment begins\n- Training Complete: When training finishes successfully\n- Training Errors: If training crashes or encounters critical errors\n- Metric Milestones: Configurable alerts for metric thresholds (if configured in SwanLab)\nEach notification includes:\n- Experiment name and project\n- Training status\n- Key metrics (loss, learning rate)\n- Direct link to SwanLab dashboard\n\n\nLark Configuration Validation\nThe plugin validates your Lark configuration at startup:\n\n✅ Valid Configurations\nuse_swanlab: true\nswanlab_project: my-project\n\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxx\nswanlab_lark_secret: your-secret\n\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxx\n\n\n\nSecurity Best Practices\n\nAlways use HMAC secret in production:\nswanlab_lark_webhook_url: https://open.feishu.cn/...\nswanlab_lark_secret: your-secret-key # ✅ Add this!\nStore secrets in environment variables (even better):\n# In your training script/environment\nexport SWANLAB_LARK_WEBHOOK_URL=\"https://open.feishu.cn/...\"\nexport SWANLAB_LARK_SECRET=\"your-secret-key\"\nThen in config:\n# SwanLab plugin will auto-detect environment variables\nuse_swanlab: true\nswanlab_project: my-project\n# Lark URL and secret read from env vars\nRotate webhook secrets periodically: Update your Lark bot’s secret every 90 days\nUse separate webhooks for dev/prod: Don’t mix development and production notifications\n\n\n\nDistributed Training\nLark notifications are automatically deduplicated in distributed training:\n- Only rank 0 sends notifications\n- Other GPU ranks skip Lark registration\n- Prevents duplicate messages in multi-GPU training\ntorchrun --nproc_per_node=4 -m axolotl.cli.train config.yml\n\n\nRLHF Completion Table Logging\nFor RLHF (Reinforcement Learning from Human Feedback) training methods like DPO, KTO, ORPO, and GRPO, SwanLab can log model completions (prompts, chosen/rejected responses, rewards) to a visual table for qualitative analysis. This helps you:\n\nInspect model behavior: See actual model outputs during training\nDebug preference learning: Compare chosen vs rejected responses\nTrack reward patterns: Monitor how rewards evolve over training\nShare examples with team: Visual tables in SwanLab dashboard\n\n\n\nFeatures\n\n✅ Automatic detection: Works with DPO, KTO, ORPO, GRPO trainers\n✅ Memory-safe buffering: Bounded buffer prevents memory leaks in long training runs\n✅ Periodic logging: Configurable logging interval to reduce overhead\n✅ Rich visualization: SwanLab tables show prompts, responses, and metrics side-by-side\n\n\n\nConfiguration\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nswanlab_log_completions\nbool\ntrue\nEnable completion logging for RLHF trainers\n\n\nswanlab_completion_log_interval\nint\n100\nLog completions to SwanLab every N training steps\n\n\nswanlab_completion_max_buffer\nint\n128\nMaximum completions to buffer (memory bound)\n\n\n\n\n\nExample: DPO Training with Completion Logging\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: dpo-training\nswanlab_experiment_name: llama-3-dpo-v1\nswanlab_mode: cloud\n\nswanlab_log_completions: true\nswanlab_completion_log_interval: 100 # Log every 100 steps\nswanlab_completion_max_buffer: 128 # Keep last 128 completions\n\nrl: dpo\ndatasets:\n - path: /path/to/preference_dataset\n type: chatml.intel\n\n\nExample: Disable Completion Logging\nIf you’re doing a quick test run or don’t need completion tables:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: dpo-training\n\nswanlab_log_completions: false\n\n\nSupported RLHF Trainers\nThe completion logging callback automatically activates for these trainer types:\n\nDPO (Direct Preference Optimization): Logs prompts, chosen, rejected, reward_diff\nKTO (Kahneman-Tversky Optimization): Logs prompts, completions, labels, rewards\nORPO (Odds Ratio Preference Optimization): Logs prompts, chosen, rejected, log_odds_ratio\nGRPO (Group Relative Policy Optimization): Logs prompts, completions, rewards, advantages\nCPO (Constrained Policy Optimization): Logs prompts, chosen, rejected\n\nFor non-RLHF trainers (standard supervised fine-tuning), the completion callback is automatically skipped.\n\n\nHow It Works\n\nAuto-detection: Plugin detects trainer type at initialization\nBuffering: Completions are buffered in memory (up to swanlab_completion_max_buffer)\nPeriodic logging: Every swanlab_completion_log_interval steps, buffer is logged to SwanLab\nMemory safety: Old completions are automatically dropped when buffer is full (uses collections.deque)\nFinal flush: Remaining completions are logged when training completes\n\n\n\nViewing Completion Tables\nAfter training starts, you can view completion tables in your SwanLab dashboard:\n\nNavigate to your experiment in SwanLab\nLook for the “rlhf_completions” table in the metrics panel\nThe table shows:\n\nstep: Training step when completion was generated\nprompt: Input prompt\nchosen: Preferred response (DPO/ORPO)\nrejected: Non-preferred response (DPO/ORPO)\ncompletion: Model output (KTO/GRPO)\nreward_diff/reward: Reward metrics\nTrainer-specific metrics (e.g., log_odds_ratio for ORPO)\n\n\n\n\nMemory Management\nThe completion buffer is memory-bounded to prevent memory leaks:\nfrom collections import deque\n\nbuffer = deque(maxlen=128) # Old completions automatically dropped\nMemory usage estimate:\n- Average completion: ~500 characters (prompt + responses)\n- Buffer size 128: ~64 KB (negligible)\n- Buffer size 1024: ~512 KB (still small)\nRecommendation: Default buffer size (128) works well for most cases. Increase to 512-1024 only if you need to review more historical completions.\n\n\nPerformance Impact\nCompletion logging has minimal overhead:\n\nBuffering: O(1) append operation, negligible CPU/memory\nLogging: Only happens every N steps (default: 100)\nNetwork: SwanLab batches table uploads efficiently\n\nExpected overhead: < 0.5% per training step\n\n\nTroubleshooting\n\nCompletions not appearing in SwanLab\nCause: Trainer may not be logging completion data in the expected format.\nDiagnostic steps:\n1. Check trainer type detection in logs:\ntext INFO: SwanLab RLHF completion logging enabled for DPOTrainer (type: dpo)\n2. Verify your trainer is an RLHF trainer (DPO/KTO/ORPO/GRPO)\n3. Check if trainer logs completion data (this depends on TRL version)\nNote: The current implementation expects trainers to log completion data in the logs dict during on_log() callback. Some TRL trainers may not expose this data by default. You may need to patch the trainer to expose completions.\n\n\nBuffer fills up too quickly\nCause: High logging frequency with small buffer size.\nSolution: Increase buffer size or logging interval:\nswanlab_completion_log_interval: 200 # Log less frequently\nswanlab_completion_max_buffer: 512 # Larger buffer\n\n\nMemory usage growing over time\nCause: Buffer should be bounded, so this indicates a bug.\nSolution:\n1. Verify swanlab_completion_max_buffer is set\n2. Check SwanLab version is up to date\n3. Report issue with memory profiling data\n\n\n\nPerformance Profiling\nSwanLab integration includes profiling utilities to measure and log execution time of trainer methods. This helps you:\n\nIdentify bottlenecks: Find slow operations in your training loop\nOptimize performance: Track improvements after optimization changes\nMonitor distributed training: See per-rank timing differences\nDebug hangs: Detect methods that take unexpectedly long\n\n\n\nFeatures\n\n✅ Zero-config profiling: Automatic timing of key trainer methods\n✅ Decorator-based: Easy to add profiling to custom methods with @swanlab_profile\n✅ Context manager: Fine-grained profiling with swanlab_profiling_context()\n✅ Advanced filtering: ProfilingConfig for throttling and minimum duration thresholds\n✅ Exception-safe: Logs duration even if function raises an exception\n\n\n\nBasic Usage: Decorator\nAdd profiling to any trainer method with the @swanlab_profile decorator:\nfrom axolotl.integrations.swanlab.profiling import swanlab_profile\n\nclass MyCustomTrainer(AxolotlTrainer):\n @swanlab_profile\n def training_step(self, model, inputs):\n # Your training step logic\n return super().training_step(model, inputs)\n\n @swanlab_profile\n def prediction_step(self, model, inputs, prediction_loss_only):\n # Your prediction logic\n return super().prediction_step(model, inputs, prediction_loss_only)\nThe decorator automatically:\n1. Measures execution time with high-precision timer\n2. Logs to SwanLab as profiling/Time taken: ClassName.method_name\n3. Only logs if SwanLab is enabled (use_swanlab: true)\n4. Gracefully handles exceptions (logs duration, then re-raises)\n\n\nAdvanced Usage: Context Manager\nFor fine-grained profiling within a method:\nfrom axolotl.integrations.swanlab.profiling import swanlab_profiling_context\n\nclass MyTrainer(AxolotlTrainer):\n def complex_training_step(self, model, inputs):\n # Profile just the forward pass\n with swanlab_profiling_context(self, \"forward_pass\"):\n outputs = model(**inputs)\n\n # Profile just the backward pass\n with swanlab_profiling_context(self, \"backward_pass\"):\n loss = outputs.loss\n loss.backward()\n\n return outputs\n\n\nAdvanced Usage: ProfilingConfig\nFilter and throttle profiling logs with ProfilingConfig:\nfrom axolotl.integrations.swanlab.profiling import (\n swanlab_profiling_context_advanced,\n ProfilingConfig,\n)\n\nprofiling_config = ProfilingConfig(\n enabled=True,\n min_duration_ms=1.0, # Only log if duration > 1ms\n log_interval=10, # Log every 10th call\n)\n\nclass MyTrainer(AxolotlTrainer):\n def frequently_called_method(self, data):\n with swanlab_profiling_context_advanced(\n self,\n \"frequent_op\",\n config=profiling_config\n ):\n # This only logs every 10th call, and only if it takes > 1ms\n result = expensive_computation(data)\n return result\nProfilingConfig Parameters:\n- enabled: Enable/disable profiling globally (default: True)\n- min_duration_ms: Minimum duration to log in milliseconds (default: 0.1)\n- log_interval: Log every Nth function call (default: 1 = log all)\nUse cases:\n- High-frequency methods: Use log_interval=100 to reduce logging overhead\n- Filter noise: Use min_duration_ms=1.0 to skip very fast operations\n- Debugging: Use log_interval=1, min_duration_ms=0.0 to log everything\n\n\nViewing Profiling Metrics\nIn your SwanLab dashboard, profiling metrics appear under the “profiling” namespace:\nprofiling/Time taken: AxolotlTrainer.training_step\nprofiling/Time taken: AxolotlTrainer.prediction_step\nprofiling/Time taken: MyTrainer.forward_pass\nprofiling/Time taken: MyTrainer.backward_pass\nYou can:\n- Track over time: See if methods get faster/slower during training\n- Compare runs: Compare profiling metrics across experiments\n- Identify regressions: Detect if a code change slowed down training\n\n\nConfiguration in Axolotl Config\nProfiling is automatically enabled when SwanLab is enabled. No additional config needed:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: my-project\nTo disable profiling while keeping SwanLab enabled:\nfrom axolotl.integrations.swanlab.profiling import DEFAULT_PROFILING_CONFIG\n\nDEFAULT_PROFILING_CONFIG.enabled = False\n\n\nPerformance Impact\n\nDecorator overhead: ~2-5 microseconds per call (negligible)\nContext manager overhead: ~1-3 microseconds (negligible)\nLogging overhead: Only when SwanLab is enabled and method duration exceeds threshold\nNetwork overhead: SwanLab batches metrics efficiently\n\nExpected overhead: < 0.1% per training step (effectively zero)\n\n\nBest Practices\n\nProfile bottlenecks first: Start by profiling suspected slow operations\nUse min_duration_ms: Filter out fast operations (< 1ms) to reduce noise\nThrottle high-frequency calls: Use log_interval for methods called > 100 times/step\nProfile across runs: Compare profiling metrics before/after optimization\nMonitor distributed training: Check for rank-specific slowdowns\n\n\n\nExample: Complete Profiling Setup\nfrom axolotl.integrations.swanlab.profiling import (\n swanlab_profile,\n swanlab_profiling_context,\n ProfilingConfig,\n)\n\nclass OptimizedTrainer(AxolotlTrainer):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n\n # Custom profiling config for high-frequency operations\n self.fast_op_config = ProfilingConfig(\n enabled=True,\n min_duration_ms=0.5,\n log_interval=50,\n )\n\n @swanlab_profile\n def training_step(self, model, inputs):\n \"\"\"Main training step - always profile.\"\"\"\n return super().training_step(model, inputs)\n\n @swanlab_profile\n def compute_loss(self, model, inputs, return_outputs=False):\n \"\"\"Loss computation - always profile.\"\"\"\n return super().compute_loss(model, inputs, return_outputs)\n\n def _prepare_inputs(self, inputs):\n \"\"\"High-frequency operation - throttled profiling.\"\"\"\n with swanlab_profiling_context_advanced(\n self,\n \"prepare_inputs\",\n config=self.fast_op_config,\n ):\n return super()._prepare_inputs(inputs)\n\n\nTroubleshooting\n\nProfiling metrics not appearing in SwanLab\nCause: SwanLab is not enabled or not initialized.\nSolution:\nuse_swanlab: true\nswanlab_project: my-project\nCheck logs for:\nINFO: SwanLab initialized for project: my-project\n\n\nToo many profiling metrics cluttering dashboard\nCause: Profiling every function call for high-frequency operations.\nSolution: Use ProfilingConfig with throttling:\nconfig = ProfilingConfig(\n min_duration_ms=1.0, # Skip fast ops\n log_interval=100, # Log every 100th call\n)\n\n\nProfiling overhead impacting training speed\nCause: Profiling itself should have negligible overhead (< 0.1%). If you see > 1% slowdown, this indicates a bug.\nSolution:\n1. Disable profiling temporarily to confirm:\npython DEFAULT_PROFILING_CONFIG.enabled = False\n2. Report issue with profiling data and trainer details\n\n\nProfiling shows inconsistent timing\nCause: Normal variation due to GPU warmup, data loading, or system load.\nSolution:\n- Ignore first few steps (warmup period)\n- Look at average/median timing over many steps\n- Use log_interval to reduce noise from individual outliers\n\n\n\nComplete Config Example\nHere’s a complete example integrating SwanLab with your RVQ-Alpha training:\nbase_model: /path/to/your/model\nmodel_type: Qwen2ForCausalLM\n\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin\n\nuse_swanlab: true\nswanlab_project: RVQ-Alpha-Training\nswanlab_experiment_name: Qwen2.5-7B-MetaQA-Perturb-P020\nswanlab_description: \"Training on MetaQA and Perturbation datasets with NEW-RVQ encoding\"\nswanlab_mode: cloud\nswanlab_workspace: single-cell-genomics\n\nsequence_len: 32768\nmicro_batch_size: 1\ngradient_accumulation_steps: 1\nnum_epochs: 2\nlearning_rate: 2e-5\noptimizer: adamw_torch_fused\n\ndatasets:\n - path: /path/to/dataset\n type: chat_template\n\noutput_dir: ./outputs\n\n\nModes Explained\n\n\ncloud Mode (Default)\n\nSyncs experiments to SwanLab cloud in real-time\nRequires API key and internet connection\nBest for: Team collaboration, remote monitoring\n\n\n\nlocal Mode\n\nSaves experiments locally only\nNo cloud sync\nBest for: Local development, air-gapped environments\n\n\n\noffline Mode\n\nSaves metadata locally\nCan sync to cloud later using swanlab sync\nBest for: Unstable internet, sync later\n\n\n\ndisabled Mode\n\nTurns off SwanLab completely\nNo logging or tracking\nBest for: Debugging, testing\n\n\n\nConfiguration Validation & Conflict Detection\nSwanLab integration includes comprehensive validation and conflict detection to help you catch configuration errors early and avoid performance issues.\n\n\nRequired Fields Validation\nThe plugin validates your configuration at startup and provides clear error messages with solutions:\n\nMissing Project Name\nuse_swanlab: true\nSolution:\nuse_swanlab: true\nswanlab_project: my-project\n\n\nInvalid Mode\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_mode: invalid-mode\nSolution:\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_mode: cloud # or: local, offline, disabled\n\n\nEmpty Project Name\nuse_swanlab: true\nswanlab_project: \"\"\nSolution:\nuse_swanlab: true\nswanlab_project: my-project\n\n\n\nCloud Mode API Key Warning\nWhen using cloud mode without an API key, you’ll receive a warning with multiple solutions:\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_mode: cloud\nSolutions:\n1. Set environment variable: export SWANLAB_API_KEY=your-api-key\n2. Add to config (less secure): swanlab_api_key: your-api-key\n3. Run swanlab login before training\n4. Use swanlab_mode: local for offline tracking\n\n\nMulti-Logger Performance Warnings\nUsing multiple logging tools simultaneously (SwanLab + WandB + MLflow + Comet) can impact training performance:\n\nTwo Loggers - Warning\nuse_swanlab: true\nswanlab_project: my-project\n\nuse_wandb: true\nwandb_project: my-project\nImpact:\n- Performance overhead: ~1-2% per logger (cumulative)\n- Increased memory usage\n- Longer training time per step\n- Potential config/callback conflicts\nRecommendations:\n- Choose ONE primary logging tool for production training\n- Use multiple loggers only for:\n- Migration period (transitioning between tools)\n- Short comparison runs\n- Debugging specific tool issues\n- Monitor system resources (CPU, memory) during training\n\n\nThree+ Loggers - Error-Level Warning\nuse_swanlab: true\nswanlab_project: my-project\n\nuse_wandb: true\nwandb_project: my-project\n\nuse_mlflow: true\nmlflow_tracking_uri: http://localhost:5000\nWhy This Matters:\n- With 3 loggers: ~4-5% overhead per step → significant slowdown over long training\n- Example: 10,000 steps at 2s/step → ~400-500 seconds extra (6-8 minutes)\n- Memory overhead scales with number of loggers\n- Rare edge cases with callback ordering conflicts\n\n\n\nAuto-Enable Logic\nFor convenience, SwanLab will auto-enable if you specify a project without setting use_swanlab:\nswanlab_project: my-project\n\nuse_swanlab: true\nswanlab_project: my-project\n\n\nDistributed Training Detection\nIn distributed training scenarios (multi-GPU), the plugin automatically detects and reports:\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_mode: cloud\nWhy Only Rank 0:\n- Avoids duplicate experiment runs\n- Reduces network/cloud API overhead on worker ranks\n- Prevents race conditions in metric logging\n\n\nAuthentication\n\n\nMethod 1: Environment Variable (Recommended)\nexport SWANLAB_API_KEY=your-api-key-here\n\n\nMethod 2: Login Command\nswanlab login\n\n\nMethod 3: Config File\nswanlab_api_key: your-api-key-here\n\n\nWhat Gets Logged?\n\n\nAutomatically Logged Metrics\n\nTraining loss\nLearning rate\nGradient norm\nTraining steps\nEpoch progress\n\n\n\nAutomatically Logged Config\n\nModel configuration (base_model, model_type)\nTraining hyperparameters (learning_rate, batch_size, etc.)\nOptimizer settings\nParallelization settings (FSDP, DeepSpeed, Context Parallel)\nAxolotl configuration file\nDeepSpeed configuration (if used)\n\n\n\nViewing Your Experiments\n\n\nCloud Mode\nVisit https://swanlab.cn and navigate to your project to view:\n- Real-time training metrics\n- Hyperparameter comparison\n- System resource usage\n- Configuration files\n\n\nLocal Mode\nswanlab watch ./swanlog\n\n\nIntegration with Existing Tools\nSwanLab can work alongside other tracking tools:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: my-project\n\nuse_wandb: true\nwandb_project: my-project\n\n\nTroubleshooting\n\n\nConfiguration Errors\n\nError: “SwanLab enabled but ‘swanlab_project’ is not set”\nCause: You enabled SwanLab (use_swanlab: true) but forgot to specify a project name.\nSolution:\nuse_swanlab: true\nswanlab_project: my-project # Add this line\n\n\nError: “Invalid swanlab_mode: ‘xxx’”\nCause: You provided an invalid mode value.\nSolution: Use one of the valid modes:\nswanlab_mode: cloud # or: local, offline, disabled\n\n\nError: “swanlab_project cannot be an empty string”\nCause: You set swanlab_project: \"\" (empty string).\nSolution: Either provide a valid name or remove the field:\nswanlab_project: my-project\n\n\n\nImport Errors\n\nError: “SwanLab is not installed”\nCause: SwanLab package is not installed in your environment.\nSolution:\npip install swanlab\npip install swanlab>=0.3.0\n\n\n\nPerformance Issues\n\nWarning: “Multiple logging tools enabled”\nCause: You have multiple experiment tracking tools enabled (e.g., SwanLab + WandB + MLflow).\nImpact: ~1-2% performance overhead per logger, cumulative.\nSolution: For production training, disable all but one logger:\nuse_swanlab: true\nswanlab_project: my-project\nuse_wandb: false # Disable others\nuse_mlflow: false\n\nuse_swanlab: false\nuse_wandb: true\nwandb_project: my-project\nException: Multiple loggers are acceptable for:\n- Short comparison runs (< 100 steps)\n- Migration testing between logging tools\n- Debugging logger-specific issues\n\n\n\nDistributed Training Issues\n\nSwanLab creates duplicate runs in multi-GPU training\nCause: All ranks are initializing SwanLab instead of just rank 0.\nExpected Behavior: The plugin automatically ensures only rank 0 initializes SwanLab. You should see:\nInfo: Distributed training detected (world_size=4)\nInfo: Only rank 0 will initialize SwanLab\nInfo: Other ranks will skip SwanLab to avoid conflicts\nIf you see duplicates:\n1. Check your plugin is loaded correctly\n2. Verify you’re using the latest SwanLab integration code\n3. Check logs for initialization messages on all ranks\n\n\n\nSwanLab not logging metrics\nSolution: Ensure SwanLab is initialized before training starts. The plugin automatically handles this in pre_model_load.\n\n\nAPI Key errors\nSolution:\necho $SWANLAB_API_KEY\n\nswanlab login\n\n\nCloud sync issues\nSolution: Use offline mode and sync later:\nswanlab_mode: offline\nThen sync when ready:\nswanlab sync ./swanlog\n\n\nPlugin not loaded\nSolution: Verify plugin path in config:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin # Correct path\n\n\nLark Notification Issues\n\nError: “Failed to import SwanLab Lark plugin”\nCause: Your SwanLab version doesn’t include the Lark plugin (requires SwanLab >= 0.3.0).\nSolution:\npip install --upgrade swanlab\n\npip install 'swanlab>=0.3.0'\n\n\nWarning: “Lark webhook has no secret configured”\nCause: You provided swanlab_lark_webhook_url but no swanlab_lark_secret.\nImpact: Lark notifications will work, but without HMAC authentication (security risk).\nSolution: Add HMAC secret for production use:\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxx\nswanlab_lark_secret: your-webhook-secret # Add this line\nWhen it’s OK to skip secret:\n- Local development and testing\n- Internal networks with restricted access\n- Non-sensitive training experiments\nWhen secret is required:\n- Production training jobs\n- Training with proprietary data\n- Multi-team shared Lark groups\n\n\nError: “Failed to register Lark callback”\nCause: Invalid webhook URL or network connectivity issues.\nDiagnostic steps:\ncurl -X POST \"YOUR_WEBHOOK_URL\" \\\n -H 'Content-Type: application/json' \\\n -d '{\"msg_type\":\"text\",\"content\":{\"text\":\"Test from Axolotl\"}}'\n\npip show swanlab\nSolution:\n1. Verify webhook URL is correct (copy from Lark bot settings)\n2. Check network connectivity to Lark API\n3. Ensure webhook is not expired (Lark webhooks can expire)\n4. Regenerate webhook URL in Lark bot settings if needed\n\n\nLark notifications not received\nCause: Multiple possible causes.\nDiagnostic checklist:\n\nCheck training logs for Lark registration confirmation:\n# Expected log message (rank 0 only):\nINFO: Registered Lark notification callback with HMAC authentication\nVerify webhook in Lark: Test webhook manually (see above)\nCheck distributed training: Only rank 0 sends notifications\n# If running multi-GPU, check rank 0 logs specifically\ngrep \"Registered Lark\" logs/rank_0.log\nVerify SwanLab is initialized: Lark callback needs SwanLab to be running\nuse_swanlab: true # Must be enabled\nswanlab_project: my-project # Must be set\nCheck Lark bot permissions: Ensure bot is added to the target group chat\n\n\n\nDuplicate Lark notifications in multi-GPU training\nExpected Behavior: Should NOT happen - only rank 0 sends notifications.\nIf you see duplicates:\n1. Check that all GPUs are using the same config file\n2. Verify plugin is loaded correctly on all ranks\n3. Check logs for unexpected Lark initialization on non-zero ranks\n4. Ensure RANK or LOCAL_RANK environment variables are set correctly\nSolution: This is a bug if it occurs. Report with:\n- Full training command\n- Logs from all ranks\n- Config file\n\n\n\nComparison: SwanLab vs WandB\n\n\n\nFeature\nSwanLab\nWandB\n\n\n\n\nOpen Source\n✅ Yes\n❌ No\n\n\nSelf-Hosting\n✅ Easy\n⚠️ Complex\n\n\nFree Tier\n✅ Generous\n⚠️ Limited\n\n\nChinese Support\n✅ Native\n⚠️ Limited\n\n\nOffline Mode\n✅ Full support\n✅ Supported\n\n\nIntegration\n🆕 New\n✅ Mature\n\n\n\n\n\nAdvanced Usage\n\n\nCustom Logging\nYou can add custom metrics in your callbacks:\nimport swanlab\n\nswanlab.log({\n \"custom_metric\": value,\n \"epoch\": epoch_num\n})\n\n\nExperiment Comparison\nswanlab compare run1 run2 run3\n\n\nSupport\n\nDocumentation: https://docs.swanlab.cn\nGitHub: https://github.com/SwanHubX/SwanLab\nIssues: Report bugs at GitHub Issues\n\n\n\nLicense\nThis integration follows the Axolotl Community License Agreement.\n\n\nAcknowledgements\nThis integration is built on top of:\n- SwanLab - Experiment tracking tool\n- Transformers - SwanLabCallback\n- Axolotl - Training framework\nPlease see reference here", + "objectID": "docs/ebft.html", + "href": "docs/ebft.html", + "title": "EBFT Training", + "section": "", + "text": "Energy-Based Fine-Tuning (EBFT) is a training method that optimizes language models by matching the internal feature representations of generated text to those of ground-truth completions. Instead of relying on external reward models or hand-crafted reward functions, EBFT extracts hidden states from intermediate layers of a frozen copy of the model and uses cosine similarity between generated and reference features as the reward signal.\nPaper: “Matching Features, Not Tokens: Energy-Based Fine-Tuning of Language Models” (Jelassi et al., 2026)\n\n\n\n\n\n\n\n\n\n\n\nMethod\nReward Signal\nRequires\nBest For\n\n\n\n\nGRPO\nExternal reward function(s)\nCustom reward code or reward model\nTasks with verifiable answers (math, code)\n\n\nDPO\nPreference pairs (chosen vs rejected)\nPaired preference data\nAlignment with human preferences\n\n\nEBFT\nFeature similarity to ground truth\nGround-truth completions\nAny task with reference outputs\n\n\n\nEBFT’s key advantage is that it needs only ground-truth completions – no reward engineering, no preference annotation, and no reward model training. The model’s own internal representations serve as the reward signal. This makes it particularly effective for:\n\nCode generation (match features of known-good solutions)\nInstruction following with reference outputs\nContinual pretraining on unstructured text (strided mode)\nMulti-turn dialogue with reference conversations\n\n\n\n\nThe EBFT reward for each generated completion is:\nreward = alignment_coef * cosine_similarity(gen_features, gt_features)\n - diversity_coef * mean_pairwise_similarity(gen_features)\n\nAlignment: How closely the generated output’s internal representations match the ground truth. Higher is better.\nDiversity: Penalizes generated samples that are too similar to each other (prevents mode collapse). Lower is better.\nCFM loss (Cross-Feature Matching): Tracks ||mean(gen_features) - gt_features||^2 as a diagnostic. This is the quantity that EBFT ultimately minimizes.", "crumbs": [ - "Advanced Features", - "Custom Integrations" + "How To Guides", + "EBFT Training" ] }, { - "objectID": "docs/custom_integrations.html#adding-a-new-integration", - "href": "docs/custom_integrations.html#adding-a-new-integration", - "title": "Custom Integrations", - "section": "Adding a new integration", - "text": "Adding a new integration\nPlugins can be used to customize the behavior of the training pipeline through hooks. See axolotl.integrations.BasePlugin for the possible hooks.\nTo add a new integration, please follow these steps:\n\nCreate a new folder in the src/axolotl/integrations directory.\nAdd any relevant files (LICENSE, README.md, ACKNOWLEDGEMENTS.md, etc.) to the new folder.\nAdd __init__.py and args.py files to the new folder.\n\n\n__init__.py should import the integration and hook into the appropriate functions.\nargs.py should define the arguments for the integration.\n\n\n(If applicable) Add CPU tests under tests/integrations or GPU tests under tests/e2e/integrations.\n\n\n\n\n\n\n\nTip\n\n\n\nSee src/axolotl/integrations/cut_cross_entropy for a minimal integration example.\n\n\n\n\n\n\n\n\nWarning\n\n\n\nIf you could not load your integration, please ensure you are pip installing in editable mode.\npip install -e .\nand correctly spelled the integration name in the config file.\nplugins:\n - axolotl.integrations.your_integration_name.YourIntegrationPlugin\n\n\n\n\n\n\n\n\nNote\n\n\n\nIt is not necessary to place your integration in the integrations folder. It can be in any location, so long as it’s installed in a package in your python env.\nSee this repo for an example: https://github.com/axolotl-ai-cloud/diff-transformer", + "objectID": "docs/ebft.html#overview", + "href": "docs/ebft.html#overview", + "title": "EBFT Training", + "section": "", + "text": "Energy-Based Fine-Tuning (EBFT) is a training method that optimizes language models by matching the internal feature representations of generated text to those of ground-truth completions. Instead of relying on external reward models or hand-crafted reward functions, EBFT extracts hidden states from intermediate layers of a frozen copy of the model and uses cosine similarity between generated and reference features as the reward signal.\nPaper: “Matching Features, Not Tokens: Energy-Based Fine-Tuning of Language Models” (Jelassi et al., 2026)\n\n\n\n\n\n\n\n\n\n\n\nMethod\nReward Signal\nRequires\nBest For\n\n\n\n\nGRPO\nExternal reward function(s)\nCustom reward code or reward model\nTasks with verifiable answers (math, code)\n\n\nDPO\nPreference pairs (chosen vs rejected)\nPaired preference data\nAlignment with human preferences\n\n\nEBFT\nFeature similarity to ground truth\nGround-truth completions\nAny task with reference outputs\n\n\n\nEBFT’s key advantage is that it needs only ground-truth completions – no reward engineering, no preference annotation, and no reward model training. The model’s own internal representations serve as the reward signal. This makes it particularly effective for:\n\nCode generation (match features of known-good solutions)\nInstruction following with reference outputs\nContinual pretraining on unstructured text (strided mode)\nMulti-turn dialogue with reference conversations\n\n\n\n\nThe EBFT reward for each generated completion is:\nreward = alignment_coef * cosine_similarity(gen_features, gt_features)\n - diversity_coef * mean_pairwise_similarity(gen_features)\n\nAlignment: How closely the generated output’s internal representations match the ground truth. Higher is better.\nDiversity: Penalizes generated samples that are too similar to each other (prevents mode collapse). Lower is better.\nCFM loss (Cross-Feature Matching): Tracks ||mean(gen_features) - gt_features||^2 as a diagnostic. This is the quantity that EBFT ultimately minimizes.", "crumbs": [ - "Advanced Features", - "Custom Integrations" + "How To Guides", + "EBFT Training" + ] + }, + { + "objectID": "docs/ebft.html#modes", + "href": "docs/ebft.html#modes", + "title": "EBFT Training", + "section": "Modes", + "text": "Modes\nEBFT supports three operational modes, each suited to different use cases.\n\nStructured Mode (Sync)\nUses vLLM on a separate GPU for generation, with sequential generate-score-train steps. This is the simplest mode and recommended for getting started.\nGPU 0: vLLM Server (generates completions, receives weight syncs)\nGPU 1: Trainer (feature extraction, reward computation, GRPO training)\nWhen to use: Standard instruction-following or QA datasets where you have prompt/completion pairs. Requires 2 GPUs.\n\n\nStructured Mode (Async)\nSame architecture as sync, but overlaps generation of the next batch with training on the current batch. Faster throughput at the cost of slightly stale weights during generation.\nWhen to use: Same data as sync mode, but when you want faster training and can tolerate weight staleness (controlled by vllm_sync_interval).\n\n\nStrided Mode\nRuns entirely on a single GPU with no vLLM dependency. Places anchor points throughout a document and generates short rollouts at each anchor using block-parallel attention patterns.\nSingle GPU: Base model + LoRA adapter\n - Strided block-parallel generation (flex_attention)\n - Feature extraction via disable_adapter()\n - No vLLM needed\nWhen to use: Unstructured text data (raw code, prose, documents) where there is no natural prompt/completion split. Also works with structured data that includes prompt boundaries. Requires only 1 GPU.", + "crumbs": [ + "How To Guides", + "EBFT Training" + ] + }, + { + "objectID": "docs/ebft.html#quick-start", + "href": "docs/ebft.html#quick-start", + "title": "EBFT Training", + "section": "Quick Start", + "text": "Quick Start\n\nStructured Mode\nThis minimal example fine-tunes Qwen2-0.5B on code data using EBFT with vLLM generation.\nStep 1: Create a config file ebft_quickstart.yaml:\nbase_model: Qwen/Qwen2-0.5B-Instruct\n\nrl: ebft\n\nebft:\n feature_layers: [0.25, 0.5, 0.75]\n embed_method: last_token\n alignment_coef: 1.0\n diversity_coef: 1.0\n\ntrl:\n num_generations: 4\n max_completion_length: 256\n temperature: 0.7\n use_vllm: true\n vllm_server_host: 0.0.0.0\n vllm_server_port: 8000\n vllm_lora_sync: true\n vllm_sync_interval: 3\n use_data_producer: true\n async_prefetch: false\n scale_rewards: true\n loss_type: grpo\n\nvllm:\n gpu_memory_utilization: 0.5\n max_model_len: 1024\n\ndatasets:\n - path: nvidia/OpenCodeInstruct\n type: ebft_opencode.transform\n split: train[:500]\n\n# Standard training settings (see getting-started.qmd for details)\nadapter: lora\nlora_r: 16\nlora_alpha: 32\nlora_target_linear: true\nsequence_len: 1024\nmicro_batch_size: 2\ngradient_accumulation_steps: 4\nmax_steps: 20\nlearning_rate: 5.0e-6\nbf16: auto\nflash_attention: true\ngradient_checkpointing: true\noutput_dir: ./outputs/ebft-quickstart\nStep 2: Start vLLM on GPU 0:\nCUDA_VISIBLE_DEVICES=0 axolotl vllm-serve ebft_quickstart.yaml\nStep 3: Wait approximately 30 seconds for vLLM to initialize, then start training on GPU 1:\nCUDA_VISIBLE_DEVICES=1 axolotl train ebft_quickstart.yaml\n\n\n\n\n\n\nImportant\n\n\n\nThe micro_batch_size must be divisible by num_generations. For example, with num_generations: 4, valid values are 4, 8, 12, etc.\n\n\n\n\nDataset Format\nStructured mode datasets must produce two fields after the transform:\n\nprompt: Either a string or a list of chat messages ([{\"role\": \"user\", \"content\": \"...\"}])\nground_truth: A string containing the reference completion\n\nExample raw dataset row:\n{\n \"input\": \"Write a function to compute fibonacci numbers.\",\n \"output\": \"def fibonacci(n):\\n if n <= 1:\\n return n\\n return fibonacci(n-1) + fibonacci(n-2)\"\n}\nThe ebft_opencode.transform converts this to the required {prompt, ground_truth} format automatically.", + "crumbs": [ + "How To Guides", + "EBFT Training" + ] + }, + { + "objectID": "docs/ebft.html#feature-extraction", + "href": "docs/ebft.html#feature-extraction", + "title": "EBFT Training", + "section": "Feature Extraction", + "text": "Feature Extraction\nEBFT extracts hidden states from intermediate transformer layers and pools them into per-sequence embeddings. These embeddings are compared between generated and ground-truth completions to compute rewards.\n\nFeature Layers\nThe feature_layers parameter specifies which layers to extract, as fractions of total model depth:\nebft:\n feature_layers: [0.25, 0.5, 0.75] # Quarter, middle, three-quarter depth\nFor a 32-layer model, this extracts layers 8, 16, and 24. The hidden states from all selected layers are concatenated along the feature dimension, producing embeddings of size num_layers * hidden_dim.\n\n\n\n\n\n\nTip\n\n\n\nUsing multiple layers captures both low-level syntactic features (early layers) and high-level semantic features (later layers). The default [0.25, 0.5, 0.75] works well across model sizes.\n\n\n\n\nEmbed Methods\nThe embed_method controls how per-token hidden states are pooled into a single vector per sequence:\n\n\n\n\n\n\n\n\n\nMethod\nDescription\nOutput Shape\nNotes\n\n\n\n\nlast_token\nHidden state at the last non-padding token\n(B, D)\nDefault. Good for autoregressive models where the last token summarizes the sequence.\n\n\nmean_pooling\nMean of all non-padding token states\n(B, D)\nConsiders the entire sequence equally.\n\n\ncompletion_mean\nMean over completion tokens only (excludes prompt)\n(B, D)\nFocuses reward signal on generated content. Requires prompt length information.\n\n\nconcat\nConcatenation of states at 25%, 50%, 75% positions\n(B, 3*D)\nCaptures positional structure. Higher dimensional.\n\n\n\nebft:\n embed_method: completion_mean # Focus on completion features\n\n\nSVD Whitening\nWhitening decorrelates the feature dimensions so that no single direction dominates the feature-matching loss. This is computed via SVD on the generated embeddings, with the same transform applied to the ground-truth embeddings.\nebft:\n use_whitening: true\nWhen whitening is enabled, the reward computation applies a whitening matrix W = U @ diag(1/S) @ U^T derived from the SVD of generated embeddings. This ensures all feature dimensions contribute equally to the alignment reward.\n\n\n\n\n\n\nNote\n\n\n\nSingular values scale with sqrt(batch_size), so reward magnitudes are batch-size dependent. This is acceptable because the number of samples per prompt (n_samples_per_prompt or num_generations) is fixed during training.\n\n\n\n\nAlignment and Diversity Coefficients\nThe two reward components are weighted by coefficients:\nebft:\n alignment_coef: 1.0 # Weight for cosine similarity with ground truth\n diversity_coef: 1.0 # Weight for pairwise similarity penalty\nBoth values are scaled by 2 internally (per paper equation 7). The final reward per sample is:\nreward_j = 2 * alignment_coef * cos(gen_j, gt)\n - 2 * diversity_coef * (1/(n-1)) * sum_{j' != j} dot(gen_j, gen_j')\nSetting diversity_coef: 0.0 disables the diversity penalty entirely, which may be appropriate when num_generations is small (e.g., 2).", + "crumbs": [ + "How To Guides", + "EBFT Training" + ] + }, + { + "objectID": "docs/ebft.html#strided-mode-1", + "href": "docs/ebft.html#strided-mode-1", + "title": "EBFT Training", + "section": "Strided Mode", + "text": "Strided Mode\nStrided mode is designed for training on unstructured text data where there is no natural prompt/completion boundary. Instead of generating full completions with vLLM, it places anchor points at regular intervals throughout each document and generates short rollouts at each anchor using block-parallel attention.\n\nHow Block-Parallel Generation Works\nGiven a document of length S tokens:\n\nAnchor placement: Starting at position anchor_offset, place anchors every stride tokens. Each anchor defines a block.\nContext window: Each block sees context_length tokens of preceding context from the original document.\nGeneration: At each anchor, generate generate_max_len tokens autoregressively, conditioned only on the context window.\nParallelism: All blocks are processed in a single forward pass using a specialized attention mask that prevents information leakage between blocks.\n\nDocument: [tok0, tok1, ..., tok_S]\n | | |\n anchor_0 anchor_1 anchor_2\n | | |\n [ctx][gen] [ctx][gen] [ctx][gen]\nThe attention mask ensures:\n\nPrompt tokens use standard causal attention\nEach generated block attends to its own context window and its own preceding generated tokens\nBlocks do not attend to each other’s generated tokens\n\nWhen flex_attention is available (PyTorch >= 2.5), the mask is compiled into efficient fused kernels. Otherwise, a dense 4D attention mask is used as a fallback.\n\n\nStrided Mode Configuration\nbase_model: meta-llama/Llama-3.2-1B\nrl: ebft\n\nebft:\n mode: strided\n stride: 8 # Tokens between anchor points\n context_length: 8 # Context window per block\n generate_max_len: 8 # Tokens to generate per block\n n_samples_per_prompt: 4 # Independent rollouts per document\n temperature: 0.6\n feature_layers: [0.25, 0.5, 0.75]\n embed_method: last_token\n use_whitening: true\n alignment_coef: 1.0\n diversity_coef: 1.0\n rl_coef: 1.0 # RL policy gradient loss weight\n ce_coef: 0.03 # Cross-entropy loss on GT tokens\n advantage_estimator: rloo # rloo, group_norm, or reinforce\n min_completion_prefix: 8 # Skip anchors in prompt region\n\ndatasets:\n - path: nvidia/OpenCodeInstruct\n type: ebft_strided_structured.transform\n split: train[:1%]\n\nsequence_len: 2048\nmicro_batch_size: 1\ngradient_accumulation_steps: 2\n\nadapter: lora\nlora_r: 16\nlora_alpha: 32\nlora_target_linear: true\n\nbf16: auto\nflex_attention: true\ngradient_checkpointing: true\ngradient_checkpointing_kwargs:\n use_reentrant: true # Required with flex_attention\nRun with a single command (no vLLM needed):\nCUDA_VISIBLE_DEVICES=0 axolotl train config.yaml\n\n\nAdvantage Estimators\nStrided mode supports three advantage estimation methods:\n\n\n\n\n\n\n\n\nEstimator\nFormula\nRequirements\n\n\n\n\nrloo\nLeave-one-out baseline: reward_j - mean(rewards_{-j})\nn_samples_per_prompt >= 2\n\n\ngroup_norm\nGroup normalization: (reward_j - mean) / std\nn_samples_per_prompt >= 2\n\n\nreinforce\nRaw reward as advantage (no baseline)\nWorks with n_samples_per_prompt = 1\n\n\n\n\n\n\n\n\n\nWarning\n\n\n\nWhen n_samples_per_prompt: 1, the trainer automatically falls back to reinforce and disables the diversity penalty (which requires multiple samples).\n\n\n\n\nStrided Mode Constraints\n\nflex_attention: true is strongly recommended. Without it, dense 4D masks consume significantly more memory.\ntorch_compile: true must NOT be set. flex_attention compiles its own kernels internally; adding torch_compile causes conflicts and OOM.\nGradient checkpointing must use use_reentrant: true. Non-reentrant checkpointing causes CheckpointError with flex_attention block masks.\nactivation_offloading is incompatible with flex_attention.\n\n\n\nCross-Entropy Loss\nStrided mode supports an optional cross-entropy loss term on ground-truth tokens. This acts as a regularizer to prevent the model from drifting too far from the original distribution:\nebft:\n ce_coef: 0.03 # Small CE coefficient\n rl_coef: 1.0 # RL loss coefficient\nThe total loss is rl_coef * rl_loss + ce_coef * ce_loss. For structured mode, ce_coef is typically 0.0 since vLLM generation provides sufficient learning signal.", + "crumbs": [ + "How To Guides", + "EBFT Training" + ] + }, + { + "objectID": "docs/ebft.html#dataset-formats", + "href": "docs/ebft.html#dataset-formats", + "title": "EBFT Training", + "section": "Dataset Formats", + "text": "Dataset Formats\nEBFT provides several built-in dataset transforms in src/axolotl/prompt_strategies/ebft/.\n\nBuilt-In Transforms\n\n\n\n\n\n\n\n\n\nTransform\nInput Format\nOutput Fields\nUse Case\n\n\n\n\nebft_opencode.transform\n{input, output}\n{prompt, ground_truth}\nOpenCodeInstruct, structured QA\n\n\nebft_strided_structured.transform\n{input, output}\n{input_ids, labels, prompt_length}\nStrided mode with structured data\n\n\nebft_strided_chat.transform\n{messages: [...]}\n{input_ids, labels, prompt_length}\nStrided mode with chat data\n\n\nebft_chat_multiturn.transform\n{messages: [...]}\n{prompt, ground_truth, remaining_turns}\nMulti-turn: first-turn target\n\n\nebft_chat_multiturn.transform_last_turn\n{messages: [...]}\n{prompt, ground_truth}\nMulti-turn: last-turn target\n\n\nebft_chat_multiturn.transform_all_turns\n{messages: [...]}\n{prompt[], ground_truth[]}\nMulti-turn: one example per turn\n\n\nebft_reasoning.transform\n{messages: [...]} (with <think>)\n{prompt, ground_truth}\nReasoning/thinking datasets\n\n\n\n\n\nStructured Mode Datasets\nFor structured (sync/async) mode, the transform must produce prompt and ground_truth fields:\ndatasets:\n - path: nvidia/OpenCodeInstruct\n type: ebft_opencode.transform\n split: train[:500]\n\n\nMulti-Turn Datasets\nMulti-turn transforms extract conversation data for sequential rollout. The transform variant targets the first assistant turn, while transform_last_turn targets the final turn:\ndatasets:\n - path: your/multiturn-dataset\n type: ebft_chat_multiturn.transform\nWhen remaining_turns is present in the dataset output, the trainer performs sequential rollouts: it generates the first assistant turn with vLLM, then continues generating subsequent turns by building up the conversation history.\n\n\nStrided Mode Datasets\nStrided transforms tokenize the full document and produce input_ids, labels, and prompt_length:\ndatasets:\n - path: nvidia/OpenCodeInstruct\n type: ebft_strided_structured.transform\n split: train[:1%]\n\n\nCustom Transforms\nTo use your own dataset format, write a transform function:\ndef transform(cfg, **kwargs):\n def transform_fn(example, tokenizer=None):\n return {\n \"prompt\": [{\"role\": \"user\", \"content\": example[\"question\"]}],\n \"ground_truth\": example[\"answer\"],\n }\n return transform_fn, {\"remove_columns\": \"__all__\"}\nThe \"__all__\" sentinel removes all original dataset columns after the mapping step. Reference this transform in your config:\ndatasets:\n - path: your/dataset\n type: your_module.transform", + "crumbs": [ + "How To Guides", + "EBFT Training" + ] + }, + { + "objectID": "docs/ebft.html#configuration-reference", + "href": "docs/ebft.html#configuration-reference", + "title": "EBFT Training", + "section": "Configuration Reference", + "text": "Configuration Reference\n\nCommon Parameters (All Modes)\nThese parameters are set under the ebft: key in the YAML config.\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nmode\n\"structured\" or \"strided\"\n\"structured\"\nEBFT operating mode\n\n\nfeature_layers\nlist[float]\n[0.25, 0.5, 0.75]\nFractional layer depths for feature extraction\n\n\nembed_method\nstring\n\"last_token\"\nPooling method: last_token, mean_pooling, completion_mean, or concat\n\n\nuse_whitening\nbool\nfalse\nApply SVD whitening to feature embeddings before reward computation\n\n\nalignment_coef\nfloat\n1.0\nWeight for alignment reward (cosine similarity with ground truth)\n\n\ndiversity_coef\nfloat\n1.0\nWeight for diversity penalty (pairwise dot product between samples)\n\n\nce_coef\nfloat\n0.0\nCross-entropy loss coefficient on ground-truth tokens\n\n\nadaptive_max_tokens\nbool\ntrue\nDynamically set vLLM max_tokens based on ground-truth length (structured mode)\n\n\ngt_length_multiplier\nfloat\n1.5\nMultiplier for ground-truth token count when computing adaptive max tokens (min 0.1)\n\n\n\n\n\nStrided Mode Parameters\nThese additional parameters apply only when mode: strided.\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nstride\nint\n8\nNumber of tokens between anchor points (must be >= 1)\n\n\ncontext_length\nint\n8\nContext window size for each generated block (must be >= 1)\n\n\ngenerate_max_len\nint\n8\nNumber of tokens to generate per block (must be >= 1)\n\n\nn_samples_per_prompt\nint\n4\nNumber of independent rollouts per document (must be >= 1)\n\n\ntemperature\nfloat\n0.6\nSampling temperature for strided generation\n\n\ntop_p\nfloat\n1.0\nTop-p nucleus sampling threshold\n\n\nrl_coef\nfloat\n1.0\nRL policy gradient loss coefficient\n\n\nadvantage_estimator\nstring\n\"rloo\"\nAdvantage estimation method: rloo, group_norm, or reinforce\n\n\nmin_completion_prefix\nint\n0\nMinimum tokens into the completion span before placing anchors\n\n\n\n\n\nStructured Mode TRL Parameters\nThese are set under the trl: key and control the GRPO training loop.\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nnum_generations\nint\n–\nNumber of completions generated per prompt\n\n\nmax_completion_length\nint\n–\nMaximum tokens per generated completion\n\n\ntemperature\nfloat\n0.7\nSampling temperature for vLLM generation\n\n\nuse_vllm\nbool\n–\nEnable vLLM generation backend\n\n\nvllm_lora_sync\nbool\nfalse\nSync LoRA adapters via filesystem (recommended)\n\n\nvllm_sync_interval\nint\n1\nSteps between weight syncs to vLLM\n\n\nuse_data_producer\nbool\n–\nRequired for sync mode with LoRA sync\n\n\nasync_prefetch\nbool\nfalse\nEnable async generation (overlaps with training)\n\n\nstreaming_partial_batch\nbool\nfalse\nScore groups incrementally (async mode)\n\n\nskip_zero_advantage_batches\nbool\nfalse\nSkip micro-batches where all advantages are zero\n\n\nscale_rewards\nbool\n–\nNormalize rewards within each prompt group\n\n\nloss_type\nstring\n\"grpo\"\nLoss type for policy optimization\n\n\nepsilon\nfloat\n0.2\nClipping parameter for importance sampling\n\n\n\n\n\nStop Tokens\nvLLM needs explicit stop token IDs for generation. Common configurations:\ntrl:\n generation_kwargs:\n stop_token_ids: [151645, 151643] # Qwen: <|im_end|>, <|endoftext|>\n\n\nMulti-Turn Chat Settings\nFor multi-turn conversations with Qwen3.5, disable thinking mode to prevent <think> tags in completions:\ntrl:\n chat_template_kwargs:\n enable_thinking: false", + "crumbs": [ + "How To Guides", + "EBFT Training" + ] + }, + { + "objectID": "docs/ebft.html#monitoring", + "href": "docs/ebft.html#monitoring", + "title": "EBFT Training", + "section": "Monitoring", + "text": "Monitoring\n\nKey Metrics\nEBFT logs several custom metrics to wandb and the training console. Here is what to watch for:\n\n\n\n\n\n\n\n\nMetric\nHealthy Range\nInterpretation\n\n\n\n\nebft/alignment\n0.3 – 0.9, trending upward\nCosine similarity between generated and ground-truth features. Higher means the model is learning to produce representations that match the reference.\n\n\nebft/diversity\n0.01 – 0.1\nMean pairwise similarity between different generations for the same prompt. Values above 1.0 indicate mode collapse.\n\n\nebft/cfm_loss\nBelow 10, trending downward\nCross-Feature Matching loss. This is the core quantity being minimized. Consistently above 100 indicates instability.\n\n\nebft/reward\nTrending upward (may start negative)\nCombined reward signal. If stuck at -1.0, the diversity penalty is dominating alignment.\n\n\ngrad_norm\n0.1 – 3.0\nGradient magnitude. Values of 0.0 indicate zero-advantage skip (normal). Values above 10 suggest instability.\n\n\nentropy\n0.05 – 0.5\nPolicy entropy. Values below 0.01 suggest mode collapse.\n\n\nIS ratio min\nAbove 0.1\nImportance sampling ratio minimum. Near-zero values mean the policy is too far off-policy; increase vllm_sync_interval.\n\n\n\n\n\nConsole Log Example\nDuring training, you will see periodic EBFT reward logs:\nebft reward | align +0.412 ^ | divers +0.023 v | cfm 4.231 v | reward +0.389 ^\nThe arrows indicate the desired direction: alignment and reward should trend upward, while diversity and CFM loss should trend downward.\n\n\nTroubleshooting\n\n\n\n\n\n\n\n\nSymptom\nLikely Cause\nFix\n\n\n\n\nalignment stays below 0.1\nFeature layers not capturing useful information\nTry different feature_layers or embed_method\n\n\ndiversity exceeds 1.0\nMode collapse – generations are too similar\nIncrease diversity_coef or temperature\n\n\nreward stuck at -1.0\nDiversity penalty dominates alignment\nReduce diversity_coef or increase alignment_coef\n\n\ngrad_norm consistently 0.0\nAll micro-batches have zero advantage\nIncrease num_generations or check data quality\n\n\nCheckpointError in strided mode\nIncompatible gradient checkpointing settings\nSet use_reentrant: true in gradient_checkpointing_kwargs\n\n\nOOM during training\nLogits tensor too large\nReduce sequence_len or micro_batch_size; strided mode uses chunked lm_head to mitigate this\n\n\nvLLM 500 errors\ntruncate_prompt_tokens not supported\nEnsure you are using axolotl vllm-serve (not trl vllm-serve)\n\n\n\n\n\nFeature Network Memory\nIn PEFT (LoRA) mode, the feature network shares base weights with the actor model by using the disable_adapter() context manager. This saves an entire model copy in VRAM (approximately 1–16 GB depending on model size). For non-PEFT training, a separate frozen deepcopy is created.\n\n\n\n\n\n\nNote\n\n\n\nThe disable_adapter() approach relies on an invariant: merge_adapter() is never called on the base weights. All weight sync paths (LoRA sync, HTTP, NCCL) compute merged weights as new tensors or save the adapter to the filesystem, leaving base weights unmodified.", + "crumbs": [ + "How To Guides", + "EBFT Training" + ] + }, + { + "objectID": "docs/ebft.html#examples", + "href": "docs/ebft.html#examples", + "title": "EBFT Training", + "section": "Examples", + "text": "Examples\nComplete example configurations are available in examples/ebft/:\n\n\n\n\n\n\n\n\n\nConfig\nModel\nMode\nDescription\n\n\n\n\nllama-1b-ebft-strided-structured.yaml\nLlama 3.2 1B\nStrided\nSingle-GPU strided training on code data\n\n\nqwen3-4b-ebft-structured.yaml\nQwen3 4B\nStructured (sync)\nTwo-GPU structured training\n\n\nqwen3-4b-ebft-structured-async.yaml\nQwen3 4B\nStructured (async)\nTwo-GPU async training with prefetch\n\n\nqwen3-8b-ebft-structured.yaml\nQwen3 8B\nStructured (sync)\nTwo-GPU structured training for larger model\n\n\nqwen35-4b-ebft-structured.yaml\nQwen3.5 4B\nStructured (sync)\nTwo-GPU with Qwen3.5\n\n\nqwen35-4b-ebft-structured-async.yaml\nQwen3.5 4B\nStructured (async)\nTwo-GPU async with Qwen3.5\n\n\nqwen35-9b-ebft-structured.yaml\nQwen3.5 9B\nStructured (sync)\nTwo-GPU structured for 9B model", + "crumbs": [ + "How To Guides", + "EBFT Training" ] }, { @@ -2930,7 +3151,7 @@ "href": "docs/getting-started.html#sec-next-steps", "title": "Quickstart", "section": "5 Next Steps", - "text": "5 Next Steps\nNow that you have the basics, you might want to:\n\nTry different model architectures\nExperiment with hyperparameters\nUse more advanced training methods\nScale up to larger models\n\nCheck our other guides for details on these topics:\n\nConfiguration Guide - Full configuration options\nDataset Loading - Loading datasets from various sources\nDataset Formats - Working with different data formats\nMulti-GPU Training\nMulti-Node Training", + "text": "5 Next Steps\nNow that you have the basics, explore these guides based on what you want to do:\nChoose your path:\n\nChoosing a Fine-Tuning Method — SFT vs LoRA vs QLoRA vs GRPO vs DPO, with hardware recommendations\n\nCore guides:\n\nDataset Loading — Loading datasets from various sources\nDataset Formats — Working with different data formats\nOptimizations — Flash attention, gradient checkpointing, sample packing\nTraining Stability & Debugging — Monitoring metrics, fixing NaN, OOM debugging\n\nAdvanced training methods:\n\nRLHF / Preference Learning — DPO, KTO, GRPO, EBFT\nGRPO Training — RL with custom rewards and vLLM generation\nvLLM Serving — Setting up vLLM for GRPO\n\nScaling up:\n\nMulti-GPU Training — DeepSpeed, FSDP, DDP\nMulti-Node Training — Distributed training across machines", "crumbs": [ "Getting Started", "Quickstart" @@ -3482,36 +3703,212 @@ ] }, { - "objectID": "docs/optimizers.html", - "href": "docs/optimizers.html", - "title": "Optimizers", + "objectID": "docs/custom_integrations.html", + "href": "docs/custom_integrations.html", + "title": "Custom Integrations", "section": "", - "text": "Axolotl supports all optimizers supported by transformers OptimizerNames\nHere is a list of optimizers supported by transformers as of v4.54.0:\n\nadamw_torch\nadamw_torch_fused\nadamw_torch_xla\nadamw_torch_npu_fused\nadamw_apex_fused\nadafactor\nadamw_anyprecision\nadamw_torch_4bit\nadamw_torch_8bit\nademamix\nsgd\nadagrad\nadamw_bnb_8bit\nadamw_8bit # alias for adamw_bnb_8bit\nademamix_8bit\nlion_8bit\nlion_32bit\npaged_adamw_32bit\npaged_adamw_8bit\npaged_ademamix_32bit\npaged_ademamix_8bit\npaged_lion_32bit\npaged_lion_8bit\nrmsprop\nrmsprop_bnb\nrmsprop_bnb_8bit\nrmsprop_bnb_32bit\ngalore_adamw\ngalore_adamw_8bit\ngalore_adafactor\ngalore_adamw_layerwise\ngalore_adamw_8bit_layerwise\ngalore_adafactor_layerwise\nlomo\nadalomo\ngrokadamw\nschedule_free_radam\nschedule_free_adamw\nschedule_free_sgd\napollo_adamw\napollo_adamw_layerwise\nstable_adamw", + "text": "Axolotl adds custom features through integrations. They are located within the src/axolotl/integrations directory.\nTo enable them, please check the respective documentations.", "crumbs": [ - "Core Concepts", - "Optimizers" + "Advanced Features", + "Custom Integrations" ] }, { - "objectID": "docs/optimizers.html#overview", - "href": "docs/optimizers.html#overview", - "title": "Optimizers", - "section": "", - "text": "Axolotl supports all optimizers supported by transformers OptimizerNames\nHere is a list of optimizers supported by transformers as of v4.54.0:\n\nadamw_torch\nadamw_torch_fused\nadamw_torch_xla\nadamw_torch_npu_fused\nadamw_apex_fused\nadafactor\nadamw_anyprecision\nadamw_torch_4bit\nadamw_torch_8bit\nademamix\nsgd\nadagrad\nadamw_bnb_8bit\nadamw_8bit # alias for adamw_bnb_8bit\nademamix_8bit\nlion_8bit\nlion_32bit\npaged_adamw_32bit\npaged_adamw_8bit\npaged_ademamix_32bit\npaged_ademamix_8bit\npaged_lion_32bit\npaged_lion_8bit\nrmsprop\nrmsprop_bnb\nrmsprop_bnb_8bit\nrmsprop_bnb_32bit\ngalore_adamw\ngalore_adamw_8bit\ngalore_adafactor\ngalore_adamw_layerwise\ngalore_adamw_8bit_layerwise\ngalore_adafactor_layerwise\nlomo\nadalomo\ngrokadamw\nschedule_free_radam\nschedule_free_adamw\nschedule_free_sgd\napollo_adamw\napollo_adamw_layerwise\nstable_adamw", + "objectID": "docs/custom_integrations.html#cut-cross-entropy", + "href": "docs/custom_integrations.html#cut-cross-entropy", + "title": "Custom Integrations", + "section": "Cut Cross Entropy", + "text": "Cut Cross Entropy\nCut Cross Entropy (CCE) reduces VRAM usage through optimization on the cross-entropy operation during loss calculation.\nSee https://github.com/apple/ml-cross-entropy\n\nRequirements\n\nPyTorch 2.4.0 or higher\n\n\n\nInstallation\nRun the following command to install cut_cross_entropy[transformers] if you don’t have it already.\n\nIf you are in dev environment\n\npython scripts/cutcrossentropy_install.py | sh\n\nIf you are installing from pip\n\npip3 uninstall -y cut-cross-entropy && pip3 install \"cut-cross-entropy[transformers] @ git+https://github.com/axolotl-ai-cloud/ml-cross-entropy.git@63b15e6\"\n\n\nUsage\nplugins:\n - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin\n\n\nSupported Models\n\nafmoe\napertus\narcee\ncohere\ncohere2\ndeepseek_v3\nexaone4\ngemma\ngemma2\ngemma3\ngemma3_text\ngemma3n\ngemma3n_text\nglm\nglm4\nglm4_moe\nglm4_moe_lite\nglm46v\nglm4v\nglm4v_moe\nglm_image\nglm_moe_dsa\ngpt_oss\ngranite\ngranitemoe\ngranitemoehybrid\ngranitemoeshared\nhunyuan_v1_dense\nhunyuan_v1_moe\ninternvl\nkimi_linear\nlfm2\nlfm2_moe\nlfm2_vl\nllama\nllama4\nllama4_text\nllava\nministral\nministral3\nmistral\nmistral3\nmistral4\nmixtral\nmllama\nnemotron_h\nolmo\nolmo2\nolmo3\nolmoe\nphi\nphi3\nphi4_multimodal\nqwen2\nqwen2_5_vl\nqwen2_moe\nqwen2_vl\nqwen3\nqwen3_5\nqwen3_5_text\nqwen3_5_moe\nqwen3_5_moe_text\nqwen3_moe\nqwen3_next\nqwen3_vl\nqwen3_vl_moe\nseed_oss\nsmollm3\nstep3p5\nvoxtral\n\n\n\nCitation\n@article{wijmans2024cut,\n author = {Erik Wijmans and\n Brody Huval and\n Alexander Hertzberg and\n Vladlen Koltun and\n Philipp Kr\\\"ahenb\\\"uhl},\n title = {Cut Your Losses in Large-Vocabulary Language Models},\n journal = {arXiv},\n year = {2024},\n url = {https://arxiv.org/abs/2411.09009},\n}\nPlease see reference here", "crumbs": [ - "Core Concepts", - "Optimizers" + "Advanced Features", + "Custom Integrations" ] }, { - "objectID": "docs/optimizers.html#custom-optimizers", - "href": "docs/optimizers.html#custom-optimizers", - "title": "Optimizers", - "section": "Custom Optimizers", - "text": "Custom Optimizers\nEnable custom optimizers by passing a string to the optimizer argument. Each optimizer will receive beta and epsilon args, however, some may accept additional args which are detailed below.\n\noptimi_adamw\noptimizer: optimi_adamw\n\n\nao_adamw_4bit\nDeprecated: Please use adamw_torch_4bit.\n\n\nao_adamw_8bit\nDeprecated: Please use adamw_torch_8bit.\n\n\nao_adamw_fp8\noptimizer: ao_adamw_fp8\n\n\nadopt_adamw\nGitHub: https://github.com/iShohei220/adopt\nPaper: https://arxiv.org/abs/2411.02853\noptimizer: adopt_adamw\n\n\ncame_pytorch\nGitHub: https://github.com/yangluo7/CAME/tree/master\nPaper: https://arxiv.org/abs/2307.02047\noptimizer: came_pytorch\n\n# optional args (defaults below)\nadam_beta1: 0.9\nadam_beta2: 0.999\nadam_beta3: 0.9999\nadam_epsilon: 1e-30\nadam_epsilon2: 1e-16\n\n\nmuon\nBlog: https://kellerjordan.github.io/posts/muon/\nPaper: https://arxiv.org/abs/2502.16982v1\noptimizer: muon\n\n\ndion\nMicrosoft’s Dion (DIstributed OrthoNormalization) optimizer is a scalable and communication-efficient\northonormalizing optimizer that uses low-rank approximations to reduce gradient communication.\nGitHub: https://github.com/microsoft/dion\nPaper: https://arxiv.org/pdf/2504.05295\nNote: Implementation written for PyTorch 2.7+ for DTensor\noptimizer: dion\ndion_lr: 0.01\ndion_momentum: 0.95\nlr: 0.00001 # learning rate for embeddings and parameters that fallback to AdamW", + "objectID": "docs/custom_integrations.html#densemixer", + "href": "docs/custom_integrations.html#densemixer", + "title": "Custom Integrations", + "section": "DenseMixer", + "text": "DenseMixer\nSee DenseMixer\nSimply add the following to your axolotl YAML config:\nplugins:\n - axolotl.integrations.densemixer.DenseMixerPlugin\nPlease see reference here", "crumbs": [ - "Core Concepts", - "Optimizers" + "Advanced Features", + "Custom Integrations" + ] + }, + { + "objectID": "docs/custom_integrations.html#diffusion-lm-training-plugin-for-axolotl", + "href": "docs/custom_integrations.html#diffusion-lm-training-plugin-for-axolotl", + "title": "Custom Integrations", + "section": "Diffusion LM Training Plugin for Axolotl", + "text": "Diffusion LM Training Plugin for Axolotl\nThis plugin enables diffusion language model training using an approach inspired by\nLLaDA (Large Language Diffusion Models) within Axolotl.\n\nOverview\nLLaDA is a diffusion-based approach to language model training that uses:\n- Random token masking during training instead of next-token prediction\n- Bidirectional attention to allow the model to attend to the full context\n- Importance weighting based on masking probabilities for stable training\nThis approach can lead to more robust language models with better understanding of\nbidirectional context.\n\n\nInstallation\nThe plugin is included with Axolotl. See our\ninstallation docs.\n\n\nQuickstart\nTrain with an example config (Llama‑3.2 1B):\n- Pretrain: axolotl train examples/llama-3/diffusion-3.2-1b-pretrain.yaml\n- SFT: axolotl train examples/llama-3/diffusion-3.2-1b-sft.yaml\n\n\nBasic Configuration\nYou can also modify your existing configs to enable / customize diffusion training.\nAdd the following to your Axolotl config:\nplugins:\n - axolotl.integrations.diffusion.DiffusionPlugin\nAnd, configure the nested diffusion block (defaults shown):\ndiffusion:\n noise_schedule: linear # or \"cosine\"\n min_mask_ratio: 0.1\n max_mask_ratio: 0.9\n num_diffusion_steps: 128\n eps: 1e-3\n importance_weighting: true\n\n # Mask token (training auto-adds if missing, avoid pad/eos)\n mask_token_str: \"<|diffusion_mask|>\"\n # Or use an existing special token id (e.g., 128002 for Llama-3.x)\n # mask_token_id: 128002\n\n # Sample generation during training (optional)\n generate_samples: true\n generation_interval: 100\n num_generation_samples: 3\n generation_steps: 128\n generation_temperature: 0.0\n generation_max_length: 100\n\n\nSupported Models\nAny models that support 4D attention masks should work out of the box. If not, please\ncreate an issue or open a\nPR!\n\n\nHow It Works\n\n\nRandom Masking\nDuring training, tokens are randomly masked:\n- Sample timestep t uniformly from [0, 1]\n- Calculate masking probability: p = (1 - eps) * t + eps\n- Randomly mask tokens with probability p\n\n\nDiffusion Loss\nLoss is computed only on masked tokens with (optional) importance weighting:\nloss = sum(cross_entropy(pred, target) / p_mask) / total_tokens\n\n\nSample Generation\nWhen diffusion.generate_samples: true, the plugin generates samples during training:\nSample 1:\n Original (45 tokens): The quick brown fox jumps over the lazy dog...\n Masked (18/45 tokens, 40.0%): The [MASK] [MASK] fox [MASK] over [MASK] lazy [MASK]...\n Generated: The quick brown fox jumps over the lazy dog...\nSamples are logged to console and wandb (if enabled).\n\n\nInference\nDiffusion inference is integrated into the standard Axolotl CLI. Use the same config\nyou trained with and run:\naxolotl inference path/to/your-config.yaml\nOptionally, pass --gradio to use a simple web interface.\nInteractive controls (prefix the prompt with commands):\n- :complete N → completion mode with N new masked tokens appended (default 64)\n- :mask R → random masking mode with target mask ratio R in [0.0, 1.0]\nExample session:\n================================================================================\nCommands:\n:complete N -> completion mode with N tokens (default 64)\n:mask R -> random masking with ratio R (0.0–1.0)\n================================================================================\nGive me an instruction (Ctrl + D to submit):\n\n:mask 0.4 The quick brown fox jumps over the lazy dog\n\nMasked (40.0%):\nThe [MASK] brown [MASK] jumps over the [MASK] dog\n\nGenerated:\nThe quick brown fox jumps over the loud dog\n\n\nMetrics and Monitoring\nThe plugin adds (or modifies) several metrics to track diffusion training:\n\ntrain/loss: Weighted diffusion loss\ntrain/accuracy: Accuracy on masked tokens\ntrain/mask_ratio: Average fraction of tokens masked\ntrain/num_masked_tokens: Number of tokens masked\ntrain/avg_p_mask: Average masking probability\ntrain/ce_loss: Unweighted cross-entropy loss\ntrain/importance_weight_avg: Average importance weight\n\n\n\nLimitations\n\nNo flash attention support\nNo RL training support\n\n\n\nReferences\n\nLLaDA Paper\nAxolotl Documentation\nAPI reference for plugin\n\nPlease see reference here", + "crumbs": [ + "Advanced Features", + "Custom Integrations" + ] + }, + { + "objectID": "docs/custom_integrations.html#grokfast", + "href": "docs/custom_integrations.html#grokfast", + "title": "Custom Integrations", + "section": "Grokfast", + "text": "Grokfast\nSee https://github.com/ironjr/grokfast\n\nUsage\nplugins:\n - axolotl.integrations.grokfast.GrokfastPlugin\n\ngrokfast_alpha: 2.0\ngrokfast_lamb: 0.98\n\n\nCitation\n@article{lee2024grokfast,\n title={{Grokfast}: Accelerated Grokking by Amplifying Slow Gradients},\n author={Lee, Jaerin and Kang, Bong Gyun and Kim, Kihoon and Lee, Kyoung Mu},\n journal={arXiv preprint arXiv:2405.20233},\n year={2024}\n}\nPlease see reference here", + "crumbs": [ + "Advanced Features", + "Custom Integrations" + ] + }, + { + "objectID": "docs/custom_integrations.html#kernels-integration", + "href": "docs/custom_integrations.html#kernels-integration", + "title": "Custom Integrations", + "section": "Kernels Integration", + "text": "Kernels Integration\nMoE (Mixture of Experts) kernels speed up training for MoE layers and reduce VRAM costs. In transformers v5, batched_mm and grouped_mm were integrated as built-in options via the experts_implementation config kwarg:\nclass ExpertsInterface(GeneralInterface):\n _global_mapping = {\n \"batched_mm\": batched_mm_experts_forward,\n \"grouped_mm\": grouped_mm_experts_forward,\n }\nIn our custom integration, we add support for ScatterMoE and SonicMoE, which are more efficient and faster than grouped_mm.\n\nUsage\nAdd the following to your axolotl YAML config:\nplugins:\n - axolotl.integrations.kernels.KernelsPlugin\n\nuse_kernels: true\n\nuse_scattermoe: true\nuse_sonicmoe: true\nImportant: Setting experts_implementation is incompatible with custom kernel options.\n\n\nSonicMoE installation\nPrerequisites:\n- NVIDIA Hopper (H100, H200) or Blackwell (B200, GB200) GPU\n- CUDA 12.9+ (13.0+ for B300)\n- PyTorch 2.7+ (2.9.1 recommended)\n- For B300: Triton 3.6.0\npip install --ignore-requires-python --no-deps \"sonic-moe @ git+https://github.com/Dao-AILab/sonic-moe.git@116e2df0a41874f77fa0ad269ce7df3f0cfcb956\" && pip install nvidia-cutlass-dsl==4.4.0 quack-kernels==0.2.5\nSee the SonicMoE installation guide for the latest prerequisite details.\nNote: Blackwell support is in upstream beta. On Blackwell GPUs, Axolotl automatically sets USE_QUACK_GEMM=1 to enable the Blackwell kernels.\n\n\nHow It Works\nThe KernelsPlugin runs before model loading and:\n\n\nScatterMoE\n\nRegisters the ScatterMoE kernel from the local libs/scattermoe_lora package (includes fused LoRA support via Triton kernels).\nPatches the model’s SparseMoeBlock forward method with the optimized ScatterMoE implementation.\n\n\n\nSonicMoE\n\nResolves the model’s MoE block class(es) from constants.py.\nPatches the forward method with SonicMoE’s optimized kernels and registers a weight converter for the interleaved gate/up projection format.\nSupports both softmax->topk and sigmoid->topk routing strategies.\n\nBoth paths use the shared resolve_moe_block_classes utility in constants.py for model-type-to-class resolution.\n\nSupported Models\nSee constants.py for the full list of supported model types (Qwen2-MoE, Qwen3-MoE, OLMoE, Mixtral, DeepSeek-V3, GLM-MoE, MiniMax, etc.).\n\n\n\nLimitations\nScatterMoE uses a softmax -> topk routing, so results may be different for some model architectures as baseline (GPT-OSS, etc). Incompatible with GLM_MOE_DSA (GLM 5) and GLM4_MOE_LITE (GLM 4.7 Flash) at the moment.\nSonicMoE supports both softmax->topk and sigmoid->topk routing, covering a wider range of architectures.\nScatterMoE does not work for GLM4.7 Flash (glm4_moe_lite) atm.\n\n\nNote on MegaBlocks\nWe tested MegaBlocks but were unable to ensure numerical accuracy, so we did not integrate it. It was also incompatible with many newer model architectures in transformers.\nPlease see reference here", + "crumbs": [ + "Advanced Features", + "Custom Integrations" + ] + }, + { + "objectID": "docs/custom_integrations.html#knowledge-distillation-kd", + "href": "docs/custom_integrations.html#knowledge-distillation-kd", + "title": "Custom Integrations", + "section": "Knowledge Distillation (KD)", + "text": "Knowledge Distillation (KD)\n\nUsage\nplugins:\n - \"axolotl.integrations.kd.KDPlugin\"\n\nkd_trainer: True\nkd_ce_alpha: 0.1\nkd_alpha: 0.9\nkd_temperature: 1.0\n\ntorch_compile: True # torch>=2.6.0, recommended to reduce vram\n\ndatasets:\n - path: ...\n type: \"axolotl.integrations.kd.chat_template\"\n field_messages: \"messages_combined\"\n logprobs_field: \"llm_text_generation_vllm_logprobs\" # for kd only, field of logprobs\nAn example dataset can be found at axolotl-ai-co/evolkit-logprobs-pipeline-75k-v2-sample\nPlease see reference here", + "crumbs": [ + "Advanced Features", + "Custom Integrations" + ] + }, + { + "objectID": "docs/custom_integrations.html#llmcompressor", + "href": "docs/custom_integrations.html#llmcompressor", + "title": "Custom Integrations", + "section": "LLMCompressor", + "text": "LLMCompressor\nFine-tune sparsified models in Axolotl using Neural Magic’s LLMCompressor.\nThis integration enables fine-tuning of models sparsified using LLMCompressor within the Axolotl training framework. By combining LLMCompressor’s model compression capabilities with Axolotl’s distributed training pipelines, users can efficiently fine-tune sparse models at scale.\nIt uses Axolotl’s plugin system to hook into the fine-tuning flows while maintaining sparsity throughout training.\n\n\nRequirements\n\nAxolotl with llmcompressor extras:\npip install \"axolotl[llmcompressor]\"\nRequires llmcompressor >= 0.5.1\n\nThis will install all necessary dependencies to fine-tune sparsified models using the integration.\n\n\n\nUsage\nTo enable sparse fine-tuning with this integration, include the plugin in your Axolotl config:\nplugins:\n - axolotl.integrations.llm_compressor.LLMCompressorPlugin\n\nllmcompressor:\n recipe:\n finetuning_stage:\n finetuning_modifiers:\n ConstantPruningModifier:\n targets: [\n 're:.*q_proj.weight',\n 're:.*k_proj.weight',\n 're:.*v_proj.weight',\n 're:.*o_proj.weight',\n 're:.*gate_proj.weight',\n 're:.*up_proj.weight',\n 're:.*down_proj.weight',\n ]\n start: 0\n save_compressed: true\nThis plugin does not apply pruning or sparsification itself — it is intended for fine-tuning models that have already been sparsified.\nPre-sparsified checkpoints can be:\n- Generated using LLMCompressor\n- Downloaded from Neural Magic’s Hugging Face page\n- Any custom LLM with compatible sparsity patterns that you’ve created yourself\nTo learn more about writing and customizing LLMCompressor recipes, refer to the official documentation:\nhttps://github.com/vllm-project/llm-compressor/blob/main/README.md\n\n\nStorage Optimization with save_compressed\nSetting save_compressed: true in your configuration enables saving models in a compressed format, which:\n- Reduces disk space usage by approximately 40%\n- Maintains compatibility with vLLM for accelerated inference\n- Maintains compatibility with llmcompressor for further optimization (example: quantization)\nThis option is highly recommended when working with sparse models to maximize the benefits of model compression.\n\n\nExample Config\nSee examples/llama-3/sparse-finetuning.yaml for a complete example.\n\n\n\nInference with vLLM\nAfter fine-tuning your sparse model, you can leverage vLLM for efficient inference.\nYou can also use LLMCompressor to apply additional quantization to your fine-tuned\nsparse model before inference for even greater performance benefits.:\nfrom vllm import LLM, SamplingParams\n\nprompts = [\n \"Hello, my name is\",\n \"The president of the United States is\",\n \"The capital of France is\",\n \"The future of AI is\",\n]\nsampling_params = SamplingParams(temperature=0.8, top_p=0.95)\nllm = LLM(\"path/to/your/sparse/model\")\noutputs = llm.generate(prompts, sampling_params)\n\nfor output in outputs:\n prompt = output.prompt\n generated_text = output.outputs[0].text\n print(f\"Prompt: {prompt!r}, Generated text: {generated_text!r}\")\nFor more details on vLLM’s capabilities and advanced configuration options, see the official vLLM documentation.\n\n\nLearn More\nFor details on available sparsity and quantization schemes, fine-tuning recipes, and usage examples, visit the official LLMCompressor repository:\nhttps://github.com/vllm-project/llm-compressor\nPlease see reference here", + "crumbs": [ + "Advanced Features", + "Custom Integrations" + ] + }, + { + "objectID": "docs/custom_integrations.html#language-model-evaluation-harness-lm-eval", + "href": "docs/custom_integrations.html#language-model-evaluation-harness-lm-eval", + "title": "Custom Integrations", + "section": "Language Model Evaluation Harness (LM Eval)", + "text": "Language Model Evaluation Harness (LM Eval)\nRun evaluation on model using the popular lm-evaluation-harness library.\nSee https://github.com/EleutherAI/lm-evaluation-harness\n\nUsage\nThere are two ways to use the LM Eval integration:\n\n\n1. Post-Training Evaluation\nWhen training with the plugin enabled, evaluation runs automatically after training completes:\nplugins:\n - axolotl.integrations.lm_eval.LMEvalPlugin\n\nlm_eval_tasks:\n - gsm8k\n - hellaswag\n - arc_easy\n\nlm_eval_batch_size: # Batch size for evaluation\n\noutput_dir:\nRun training as usual:\naxolotl train config.yml\n\n\n2. Standalone CLI Evaluation\nEvaluate any model directly without training:\nlm_eval_model: meta-llama/Llama-2-7b-hf\n\nplugins:\n - axolotl.integrations.lm_eval.LMEvalPlugin\n\nlm_eval_tasks:\n - gsm8k\n - hellaswag\n - arc_easy\n\nlm_eval_batch_size: 8\noutput_dir: ./outputs\nRun evaluation:\naxolotl lm-eval config.yml\n\n\nModel Selection Priority\nThe model to evaluate is selected in the following priority order:\n\nlm_eval_model - Explicit model path or HuggingFace repo (highest priority)\nhub_model_id - Trained model pushed to HuggingFace Hub\noutput_dir - Local checkpoint directory containing trained model weights\n\n\n\nCitation\n@misc{eval-harness,\n author = {Gao, Leo and Tow, Jonathan and Abbasi, Baber and Biderman, Stella and Black, Sid and DiPofi, Anthony and Foster, Charles and Golding, Laurence and Hsu, Jeffrey and Le Noac'h, Alain and Li, Haonan and McDonell, Kyle and Muennighoff, Niklas and Ociepa, Chris and Phang, Jason and Reynolds, Laria and Schoelkopf, Hailey and Skowron, Aviya and Sutawika, Lintang and Tang, Eric and Thite, Anish and Wang, Ben and Wang, Kevin and Zou, Andy},\n title = {A framework for few-shot language model evaluation},\n month = 07,\n year = 2024,\n publisher = {Zenodo},\n version = {v0.4.3},\n doi = {10.5281/zenodo.12608602},\n url = {https://zenodo.org/records/12608602}\n}\nPlease see reference here", + "crumbs": [ + "Advanced Features", + "Custom Integrations" + ] + }, + { + "objectID": "docs/custom_integrations.html#liger-kernels", + "href": "docs/custom_integrations.html#liger-kernels", + "title": "Custom Integrations", + "section": "Liger Kernels", + "text": "Liger Kernels\nLiger Kernel provides efficient Triton kernels for LLM training, offering:\n\n20% increase in multi-GPU training throughput\n60% reduction in memory usage\nCompatibility with both FSDP and DeepSpeed\n\nSee https://github.com/linkedin/Liger-Kernel\n\nUsage\nplugins:\n - axolotl.integrations.liger.LigerPlugin\nliger_rope: true\nliger_rms_norm: true\nliger_glu_activation: true\nliger_layer_norm: true\nliger_fused_linear_cross_entropy: true\n\nliger_use_token_scaling: true\n\n\nSupported Models\n\ndeepseek_v2\ngemma\ngemma2\ngemma3\ngranite\njamba\nllama\nmistral\nmixtral\nmllama\nmllama_text_model\nolmo2\npaligemma\nphi3\nqwen2\nqwen2_5_vl\nqwen2_vl\n\n\n\nCitation\n@article{hsu2024ligerkernelefficienttriton,\n title={Liger Kernel: Efficient Triton Kernels for LLM Training},\n author={Pin-Lun Hsu and Yun Dai and Vignesh Kothapalli and Qingquan Song and Shao Tang and Siyu Zhu and Steven Shimizu and Shivam Sahni and Haowen Ning and Yanning Chen},\n year={2024},\n eprint={2410.10989},\n archivePrefix={arXiv},\n primaryClass={cs.LG},\n url={https://arxiv.org/abs/2410.10989},\n journal={arXiv preprint arXiv:2410.10989},\n}\nPlease see reference here", + "crumbs": [ + "Advanced Features", + "Custom Integrations" + ] + }, + { + "objectID": "docs/custom_integrations.html#nemo-gym-integration-for-axolotl", + "href": "docs/custom_integrations.html#nemo-gym-integration-for-axolotl", + "title": "Custom Integrations", + "section": "NeMo Gym Integration for Axolotl", + "text": "NeMo Gym Integration for Axolotl\nTrain LLMs with reinforcement learning using NVIDIA NeMo Gym environments as reward sources. NeMo Gym provides 50+ verified RL environments spanning math, coding, tool-use, reasoning, and safety — each with deterministic reward signals.\n\nValidated Training Paths\n\n\n\n\n\n\n\n\n\nPath\nSpeed\nMulti-turn\nArchitecture\n\n\n\n\nAsync GRPO + Data Producer\nFastest (3x)\nYes\nNemoGymDataProducer replaces vLLM generation\n\n\nStandard GRPO + Data Producer\nBaseline\nYes\nSame producer, no async prefetch\n\n\nStandard GRPO + /verify\nSimplest\nNo\nReward function calls /verify directly\n\n\nFSDP2 + /verify (2 GPU)\nDistributed\nNo\nfsdp_version: 2\n\n\n\nMulti-turn uses nemo_gym_multi_turn: true which auto-enables the async trainer’s\ndata producer protocol. The plugin’s NemoGymDataProducer calls NeMo Gym agent /run\nendpoints and returns RolloutDataset with proper IS correction, env_mask, and rewards.\nAll paths tested end-to-end with Qwen3-0.6B + LoRA, logged to wandb project nemo-gym-rl.\n\n\nQuick Start\n\n\nPrerequisites\n\nuv package manager (for NeMo Gym’s venv)\nTwo GPUs recommended (one for vLLM server, one for training)\n\n\n\n1. Set Up NeMo Gym\ngit clone https://github.com/NVIDIA-NeMo/Gym.git ~/Gym\ncd ~/Gym\nuv venv --python 3.12 && source .venv/bin/activate && uv sync\n\nCFLAGS=\"\" uv pip install pycosat --python .venv/bin/python --no-build-isolation\n\nfor dir in resources_servers/reasoning_gym resources_servers/example_single_tool_call responses_api_models/vllm_model responses_api_agents/simple_agent; do\n uv venv --seed --allow-existing --python 3.12 $dir/.venv\n CFLAGS=\"\" uv pip install --python $dir/.venv/bin/python pycosat --no-build-isolation 2>/dev/null\n uv pip install --python $dir/.venv/bin/python -e . \"ray[default]==2.52.1\"\ndone\n\nuv pip install --python resources_servers/reasoning_gym/.venv/bin/python \\\n reasoning-gym matplotlib pillow cycler contourpy kiwisolver\n\n\n2. Multi-Turn with Async GRPO (Recommended — Fastest Path)\nThis is the fully validated, highest-performance path. NeMo Gym’s agent server handles\nmulti-turn tool execution while axolotl’s async GRPO prefetches data in background threads.\nStep 1: Create the NeMo Gym agent config\nCreate ~/Gym/configs/axolotl_tool_calling.yaml:\nexample_single_tool_call:\n resources_servers:\n example_single_tool_call:\n entrypoint: app.py\n domain: agent\n verified: false\n\npolicy_model:\n responses_api_models:\n vllm_model:\n entrypoint: app.py\n base_url: http://localhost:8000/v1\n api_key: dummy_key\n model: Qwen/Qwen3-0.6B # Must match your training model\n return_token_id_information: true\n uses_reasoning_parser: false\n\nexample_single_tool_call_simple_agent:\n responses_api_agents:\n simple_agent:\n entrypoint: app.py\n resources_server:\n type: resources_servers\n name: example_single_tool_call\n model_server:\n type: responses_api_models\n name: policy_model\n datasets:\n - name: weather\n type: example\n jsonl_fpath: resources_servers/example_single_tool_call/data/weather_tool_calling.jsonl\nStep 2: Start three services\nCUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \\\n --model Qwen/Qwen3-0.6B --max-model-len 2048 --gpu-memory-utilization 0.85\n\ncd ~/Gym && .venv/bin/ng_run \\\n \"+config_paths=[configs/axolotl_tool_calling.yaml]\" \"+skip_venv_if_present=true\"\n\ncd experiments && CUDA_VISIBLE_DEVICES=1 CUDA_HOME=$HOME/env-claude-cu130/cuda_shim \\\n axolotl train nemo_gym_async_agent.yaml\nStep 3: Training config (nemo_gym_async_agent.yaml):\nbase_model: Qwen/Qwen3-0.6B\nadapter: lora\nlora_r: 16\nlora_alpha: 32\nlora_target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]\nsequence_len: 2048\n\nrl: grpo\nchat_template: tokenizer_default\n\ntrl:\n use_vllm: true\n vllm_mode: server\n vllm_server_host: localhost\n vllm_server_port: 8000\n vllm_lora_sync: true\n vllm_sync_interval: 5\n # Async GRPO — 3x faster than standard\n use_data_producer: true\n async_prefetch: true\n num_generations: 4\n max_completion_length: 512\n temperature: 0.8\n reward_funcs:\n - axolotl.integrations.nemo_gym.rewards.reward_env\n\nplugins:\n - axolotl.integrations.nemo_gym.NemoGymPlugin\n\nnemo_gym_enabled: true\nnemo_gym_auto_start: false\nnemo_gym_head_port: 11000\nnemo_gym_multi_turn: true\nnemo_gym_verify_timeout: 120\nnemo_gym_datasets:\n - path: ~/Gym/resources_servers/example_single_tool_call/data/weather_tool_calling.jsonl\n server_name: example_single_tool_call\n\ndatasets:\n - path: ~/Gym/resources_servers/example_single_tool_call/data/weather_tool_calling.jsonl\n type: chat_template\n field_messages: responses_create_params.input\n message_field_content: content\n message_field_role: role\n\nvllm:\n gpu_memory_utilization: 0.85\n max_model_len: 2048\n tensor_parallel_size: 1\n\nlearning_rate: 5e-6\nmicro_batch_size: 1\ngradient_accumulation_steps: 4\nmax_steps: 30\ngradient_checkpointing: true\nbf16: true\noutput_dir: ./outputs/nemo_gym_async\n\nuse_wandb: true\nwandb_project: nemo-gym-rl\n\n\n3. Single-Turn Training (Simplest — No Agent Server Needed)\nFor environments that only need single-turn verify (math, coding challenges), you don’t need\nan agent server. The plugin’s reward function calls /verify directly.\nbase_model: Qwen/Qwen2.5-0.5B-Instruct\nrl: grpo\nchat_template: tokenizer_default\n\ntrl:\n use_vllm: true\n vllm_mode: colocate\n vllm_enable_sleep_mode: false\n num_generations: 8\n max_completion_length: 128\n temperature: 0.9\n reward_funcs:\n - axolotl.integrations.nemo_gym.rewards.reward_nemo_gym_verify\n\nplugins:\n - axolotl.integrations.nemo_gym.NemoGymPlugin\n\nnemo_gym_enabled: true\nnemo_gym_auto_start: false\nnemo_gym_head_port: 11000\nnemo_gym_datasets:\n - path: ~/Gym/resources_servers/reasoning_gym/data/train_basic_arithmetic.jsonl\n server_name: reasoning_gym\n\ndatasets:\n - path: ~/Gym/resources_servers/reasoning_gym/data/train_basic_arithmetic.jsonl\n type: chat_template\n field_messages: responses_create_params.input\n message_field_content: content\n message_field_role: role\n\nvllm:\n gpu_memory_utilization: 0.3\n max_model_len: 512\n tensor_parallel_size: 1\n\nlearning_rate: 1e-5\nmicro_batch_size: 4\ngradient_accumulation_steps: 2\nmax_steps: 50\noutput_dir: ./outputs/nemo_gym_arithmetic\nOnly needs ng_run with resource servers (no agent config):\ncd ~/Gym && ng_run \"+config_paths=[resources_servers/reasoning_gym/configs/resources_only.yaml]\" \"+skip_venv_if_present=true\"\n\n\nHow It Works\n\n\nSingle-Turn\naxolotl train → GRPO Trainer generates completions\n → NeMo Gym plugin reward_fn calls POST /verify on resource server\n → reward flows back to GRPO for advantage computation\n\n\nMulti-Turn (Agent /run)\n┌─────────────┐ ┌──────────────┐ ┌──────────────────┐\n│ axolotl │ │ NeMo Gym │────▶│ vLLM OpenAI │\n│ train │────▶│ Agent /run │◀────│ Server (GPU 0) │\n│ (GPU 1) │ │ │ │ /v1/completions │\n└─────────────┘ └──────┬───────┘ └──────────────────┘\n │\n ▼\n ┌──────────────┐\n │ Resource │\n │ Server │\n │ (tools + │\n │ verify) │\n └─────────────┘\nThe agent server orchestrates the entire multi-turn loop:\n1. Calls our vLLM server for model generation\n2. Parses tool calls from model output\n3. Executes tools against resource servers\n4. Feeds tool results back to the model\n5. Repeats until done, then calls /verify for reward\n6. Returns token IDs + logprobs + reward to our rollout_func\n\n\nData Producer Architecture (Multi-Turn)\nWhen nemo_gym_multi_turn: true, the plugin automatically forces use_data_producer: true\nwhich selects the AxolotlAsyncGRPOTrainer. The plugin then swaps the trainer’s data\nproducer with NemoGymDataProducer, which:\n\nGets a prompt batch from the dataset iterator\nExpands by num_generations (one agent call per rollout)\nCalls NeMo Gym agents via async HTTP (aiohttp.gather)\nParses responses into padded tensors (RolloutDataset)\nReturns with _pending_policy_logps=True for deferred scoring\n\nThe main thread then runs _compute_deferred_scores() which:\n- Computes policy logprobs on the training model (GPU forward pass)\n- Computes IS correction using agent’s sampling logprobs vs training model logprobs\n- Computes advantages with group-level normalization\n- All downstream features work: replay buffer, re-roll, streaming, zero-adv skip\nWith async_prefetch: true, the data producer runs in a background thread — giving ~3x\nspeedup as generation and training overlap. With async_prefetch: false, it runs\nsynchronously on the main thread (still uses the data producer protocol).\n\n\nWeight Sync (LoRA Mode)\nWith vllm_lora_sync: true, the plugin (or async trainer) replaces NCCL-based weight\nsync with filesystem + HTTP:\n\naccelerator.get_state_dict() gathers LoRA weights from all ranks\nRank 0 saves adapter to /tmp/lora_sync_*/vN/\nRank 0 POSTs to /set_lora_adapter/ on vLLM server\nvLLM loads adapter natively via Punica kernels\nOnly ~40MB transferred (vs multiple GBs for full model weights)\n\n\n\nMulti-Environment Support\nDatasets support per-row environment routing via agent_ref:\n{\"agent_ref\": {\"name\": \"reasoning_gym\"}, \"responses_create_params\": {...}}\n{\"agent_ref\": {\"name\": \"instruction_following\"}, \"responses_create_params\": {...}}\nOr use the simpler per-dataset routing:\nnemo_gym_datasets:\n - path: reasoning_data.jsonl\n server_name: reasoning_gym\n - path: tool_data.jsonl\n server_name: example_single_tool_call\n\n\nConfiguration Reference\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nnemo_gym_enabled\nbool\nnull\nEnable the NeMo Gym integration\n\n\nnemo_gym_dir\nstr\n~/Gym\nPath to NeMo Gym repo\n\n\nnemo_gym_auto_clone\nbool\ntrue\nAuto-clone NeMo Gym repo if missing\n\n\nnemo_gym_auto_start\nbool\ntrue\nAuto-start resource servers\n\n\nnemo_gym_config_paths\nlist[str]\n—\nServer config YAMLs (relative to gym_dir)\n\n\nnemo_gym_datasets\nlist[dict]\nrequired\nDataset configs with path and optional server_name\n\n\nnemo_gym_head_port\nint\n11000\nHead server port\n\n\nnemo_gym_server_timeout\nint\n360\nServer startup timeout (seconds)\n\n\nnemo_gym_verify_timeout\nint\n30\nPer-request timeout (seconds)\n\n\nnemo_gym_multi_turn\nbool\nfalse\nEnable multi-turn via agent /run\n\n\n\n\n\nDataset JSONL Format\nEach line must have responses_create_params with input messages:\n{\n \"responses_create_params\": {\n \"input\": [{\"role\": \"user\", \"content\": \"What's the weather in SF?\"}],\n \"tools\": [{\"name\": \"get_weather\", \"type\": \"function\", \"strict\": true, \"parameters\": {...}}]\n }\n}\nFor multi-turn agent routing, include agent_ref:\n{\"agent_ref\": {\"name\": \"my_agent\"}, \"responses_create_params\": {...}}\nNote: Tool definitions MUST include \"strict\": true and \"additionalProperties\": false for NeMo Gym agent compatibility.\n\n\nReward Functions\nThe plugin provides two built-in reward functions — no user code needed:\ntrl:\n reward_funcs:\n # Multi-turn (nemo_gym_multi_turn: true):\n # Passthrough — agent /run already computed the reward\n - axolotl.integrations.nemo_gym.rewards.reward_env\n\n # Single-turn (nemo_gym_multi_turn: false):\n # Calls /verify endpoints on NeMo Gym resource servers\n - axolotl.integrations.nemo_gym.rewards.reward_nemo_gym_verify\nBoth are also importable from Python:\nfrom axolotl.integrations.nemo_gym import reward_env, reward_nemo_gym_verify\n\n\nKnown Issues / Troubleshooting\n\n\nNeMo Gym Server Setup\n\npycosat build failure: CFLAGS=\"\" uv pip install pycosat --no-build-isolation\nRay version mismatch: Pin ray[default]==2.52.1 in all server venvs\nPre-build venvs: ng_run creates per-server venvs via Ray. Pre-build them and use +skip_venv_if_present=true\nTool strict field required: Agent server validates tool definitions require strict: true\n\n\n\nvLLM / Weight Sync\n\nStart vLLM with LoRA + tool calling + runtime loading:\nVLLM_ALLOW_RUNTIME_LORA_UPDATING=1 \\\nCUDA_VISIBLE_DEVICES=0 python -m vllm.entrypoints.openai.api_server \\\n --model Qwen/Qwen3-4B-Instruct-2507 \\\n --max-model-len 4096 \\\n --gpu-memory-utilization 0.7 \\\n --enable-lora --max-lora-rank 64 \\\n --enable-auto-tool-choice --tool-call-parser hermes\nVLLM_ALLOW_RUNTIME_LORA_UPDATING=1: Required for vllm_lora_sync: true. Without it, vLLM won’t expose the /v1/load_lora_adapter endpoint and weight sync will fail silently. The plugin warns if this endpoint is missing.\n--enable-lora: Enables LoRA adapter support in vLLM\n--enable-auto-tool-choice --tool-call-parser hermes: Required for Qwen3 tool calling\nmax_model_len must be > max_completion_length: Leave room for prompt tokens (~200). If equal, the NeMo Gym model proxy gets a 400 error and returns empty completions.\nCUDA_HOME required: DeepSpeed import needs it for the nvcc shim\nNCCL weight sync broken with vLLM 0.17: Use vllm_lora_sync: true (filesystem + HTTP via /v1/load_lora_adapter)\n\n\n\nMulti-Turn\n\nAgent server required: Multi-turn delegates to NeMo Gym’s agent server /run endpoint. Without an agent, the plugin falls back to single-turn /verify\nModel server proxy: NeMo Gym needs a responses_api_models server that proxies to your vLLM. See the agent config example above\n\n\n\nFSDP2\n\nValidated on 2 GPUs with single-turn + LoRA\nAsync field filtering: The builder automatically filters async-only config fields when using the standard GRPO trainer\n\n\n\nComparison with Other Integrations\n\n\n\n\n\n\n\n\n\nFeature\nAxolotl + NeMo Gym\nUnsloth + NeMo Gym\nNeMo RL (native)\n\n\n\n\nServer management\nAutomatic\nManual (notebook)\nBuilt-in\n\n\nMulti-environment\nPer-row routing\nManual code\nYAML config\n\n\nMulti-turn / tool use\nAgent /run delegation\nNo\nAgent /run (Ray)\n\n\nAsync GRPO (3x speedup)\nYes\nNo\nYes\n\n\nLoRA sync\nFilesystem + HTTP\nN/A\nNCCL\n\n\nMulti-GPU (FSDP2)\nYes\nNo\nYes (Ray)\n\n\nConfig-driven\nYes\nNo (code)\nYes\n\n\n\nPlease see reference here", + "crumbs": [ + "Advanced Features", + "Custom Integrations" + ] + }, + { + "objectID": "docs/custom_integrations.html#spectrum", + "href": "docs/custom_integrations.html#spectrum", + "title": "Custom Integrations", + "section": "Spectrum", + "text": "Spectrum\nby Eric Hartford, Lucas Atkins, Fernando Fernandes, David Golchinfar\nThis plugin contains code to freeze the bottom fraction of modules in a model, based on the Signal-to-Noise Ratio (SNR).\nSee https://github.com/cognitivecomputations/spectrum\n\nOverview\nSpectrum is a tool for scanning and evaluating the Signal-to-Noise Ratio (SNR) of layers in large language models.\nBy identifying the top n% of layers with the highest SNR, you can optimize training efficiency.\n\n\nUsage\nplugins:\n - axolotl.integrations.spectrum.SpectrumPlugin\n\nspectrum_top_fraction: 0.5\nspectrum_model_name: meta-llama/Meta-Llama-3.1-8B\n\n\nCitation\n@misc{hartford2024spectrumtargetedtrainingsignal,\n title={Spectrum: Targeted Training on Signal to Noise Ratio},\n author={Eric Hartford and Lucas Atkins and Fernando Fernandes Neto and David Golchinfar},\n year={2024},\n eprint={2406.06623},\n archivePrefix={arXiv},\n primaryClass={cs.LG},\n url={https://arxiv.org/abs/2406.06623},\n}\nPlease see reference here", + "crumbs": [ + "Advanced Features", + "Custom Integrations" + ] + }, + { + "objectID": "docs/custom_integrations.html#swanlab-integration-for-axolotl", + "href": "docs/custom_integrations.html#swanlab-integration-for-axolotl", + "title": "Custom Integrations", + "section": "SwanLab Integration for Axolotl", + "text": "SwanLab Integration for Axolotl\nSwanLab is an open-source, lightweight AI experiment tracking and visualization tool that provides a platform for tracking, recording, comparing, and collaborating on experiments.\nThis integration enables seamless experiment tracking and visualization of Axolotl training runs using SwanLab.\n\nFeatures\n\n📊 Automatic Metrics Logging: Training loss, learning rate, and other metrics are automatically logged\n🎯 Hyperparameter Tracking: Model configuration and training parameters are tracked\n📈 Real-time Visualization: Monitor training progress in real-time through SwanLab dashboard\n☁️ Cloud & Local Support: Works in both cloud-synced and offline modes\n🔄 Experiment Comparison: Compare multiple training runs easily\n🤝 Team Collaboration: Share experiments with team members\n🎭 RLHF Completion Logging: Automatically log model outputs during DPO/KTO/ORPO/GRPO training for qualitative analysis\n⚡ Performance Profiling: Built-in profiling decorators to measure and optimize training performance\n🔔 Lark Notifications: Send real-time training updates to team chat (Feishu/Lark integration)\n\n\n\nInstallation\npip install swanlab\n\n\nQuick Start\n\n\n1. Register for SwanLab (Optional for cloud mode)\nIf you want to use cloud sync features, register at https://swanlab.cn to get your API key.\n\n\n2. Configure Axolotl Config File\nAdd SwanLab configuration to your Axolotl YAML config:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: my-llm-project\nswanlab_experiment_name: qwen-finetune-v1\nswanlab_mode: cloud # Options: cloud, local, offline, disabled\nswanlab_workspace: my-team # Optional: organization name\nswanlab_api_key: YOUR_API_KEY # Optional: can also use env var SWANLAB_API_KEY\n\n\n3. Run Training\nexport SWANLAB_API_KEY=your-api-key-here\n\nswanlab login\n\naccelerate launch -m axolotl.cli.train your-config.yaml\n\n\nConfiguration Options\n\n\nBasic Configuration\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nuse_swanlab\nbool\nfalse\nEnable SwanLab tracking\n\n\nswanlab_project\nstr\nNone\nProject name (required)\n\n\nswanlab_experiment_name\nstr\nNone\nExperiment name\n\n\nswanlab_description\nstr\nNone\nExperiment description\n\n\nswanlab_mode\nstr\ncloud\nSync mode: cloud, local, offline, disabled\n\n\n\n\n\nAdvanced Configuration\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nswanlab_workspace\nstr\nNone\nWorkspace/organization name\n\n\nswanlab_api_key\nstr\nNone\nAPI key (prefer env var)\n\n\nswanlab_web_host\nstr\nNone\nPrivate deployment web host\n\n\nswanlab_api_host\nstr\nNone\nPrivate deployment API host\n\n\nswanlab_log_model\nbool\nfalse\nLog model checkpoints (coming soon)\n\n\nswanlab_lark_webhook_url\nstr\nNone\nLark (Feishu) webhook URL for team notifications\n\n\nswanlab_lark_secret\nstr\nNone\nLark webhook HMAC secret for authentication\n\n\nswanlab_log_completions\nbool\ntrue\nEnable RLHF completion table logging (DPO/KTO/ORPO/GRPO)\n\n\nswanlab_completion_log_interval\nint\n100\nSteps between completion logging\n\n\nswanlab_completion_max_buffer\nint\n128\nMax completions to buffer (memory bound)\n\n\n\n\n\nConfiguration Examples\n\n\nExample 1: Basic Cloud Sync\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: llama-finetune\nswanlab_experiment_name: llama-3-8b-instruct-v1\nswanlab_mode: cloud\n\n\nExample 2: Offline/Local Mode\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: local-experiments\nswanlab_experiment_name: test-run-1\nswanlab_mode: local # or 'offline'\n\n\nExample 3: Team Workspace\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: research-project\nswanlab_experiment_name: experiment-42\nswanlab_workspace: my-research-team\nswanlab_mode: cloud\n\n\nExample 4: Private Deployment\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: internal-project\nswanlab_experiment_name: secure-training\nswanlab_mode: cloud\nswanlab_web_host: https://swanlab.yourcompany.com\nswanlab_api_host: https://api.swanlab.yourcompany.com\n\n\nTeam Notifications with Lark (Feishu)\nSwanLab supports sending real-time training notifications to your team chat via Lark (Feishu), ByteDance’s enterprise collaboration platform. This is especially useful for:\n- Production training monitoring: Get alerts when training starts, completes, or encounters errors\n- Team collaboration: Keep your ML team informed about long-running experiments\n- Multi-timezone teams: Team members can check training progress without being online\n\n\nPrerequisites\n\nLark Bot Setup: Create a custom bot in your Lark group chat\nWebhook URL: Get the webhook URL from your Lark bot settings\nHMAC Secret (recommended): Enable signature verification in your Lark bot for security\n\nFor detailed Lark bot setup instructions, see Lark Custom Bot Documentation.\n\n\nExample 5: Basic Lark Notifications\nSend training notifications to a Lark group chat:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: production-training\nswanlab_experiment_name: llama-3-finetune-v2\nswanlab_mode: cloud\n\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxx\nNote: This configuration will work, but you’ll see a security warning recommending HMAC secret configuration.\n\n\nExample 6: Lark Notifications with HMAC Security (Recommended)\nFor production use, enable HMAC signature verification:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: production-training\nswanlab_experiment_name: llama-3-finetune-v2\nswanlab_mode: cloud\n\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxx\nswanlab_lark_secret: your-webhook-secret-key\nWhy HMAC secret matters:\n- Prevents unauthorized parties from sending fake notifications to your Lark group\n- Ensures notifications genuinely come from your training jobs\n- Required for production deployments with sensitive training data\n\n\nExample 7: Team Workspace + Lark Notifications\nCombine team workspace collaboration with Lark notifications:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: research-project\nswanlab_experiment_name: multimodal-experiment-42\nswanlab_workspace: ml-research-team\nswanlab_mode: cloud\n\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxx\nswanlab_lark_secret: your-webhook-secret-key\n\n\nWhat Notifications Are Sent?\nSwanLab’s Lark integration sends notifications for key training events:\n- Training Start: When your experiment begins\n- Training Complete: When training finishes successfully\n- Training Errors: If training crashes or encounters critical errors\n- Metric Milestones: Configurable alerts for metric thresholds (if configured in SwanLab)\nEach notification includes:\n- Experiment name and project\n- Training status\n- Key metrics (loss, learning rate)\n- Direct link to SwanLab dashboard\n\n\nLark Configuration Validation\nThe plugin validates your Lark configuration at startup:\n\n✅ Valid Configurations\nuse_swanlab: true\nswanlab_project: my-project\n\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxx\nswanlab_lark_secret: your-secret\n\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxx\n\n\n\nSecurity Best Practices\n\nAlways use HMAC secret in production:\nswanlab_lark_webhook_url: https://open.feishu.cn/...\nswanlab_lark_secret: your-secret-key # ✅ Add this!\nStore secrets in environment variables (even better):\n# In your training script/environment\nexport SWANLAB_LARK_WEBHOOK_URL=\"https://open.feishu.cn/...\"\nexport SWANLAB_LARK_SECRET=\"your-secret-key\"\nThen in config:\n# SwanLab plugin will auto-detect environment variables\nuse_swanlab: true\nswanlab_project: my-project\n# Lark URL and secret read from env vars\nRotate webhook secrets periodically: Update your Lark bot’s secret every 90 days\nUse separate webhooks for dev/prod: Don’t mix development and production notifications\n\n\n\nDistributed Training\nLark notifications are automatically deduplicated in distributed training:\n- Only rank 0 sends notifications\n- Other GPU ranks skip Lark registration\n- Prevents duplicate messages in multi-GPU training\ntorchrun --nproc_per_node=4 -m axolotl.cli.train config.yml\n\n\nRLHF Completion Table Logging\nFor RLHF (Reinforcement Learning from Human Feedback) training methods like DPO, KTO, ORPO, and GRPO, SwanLab can log model completions (prompts, chosen/rejected responses, rewards) to a visual table for qualitative analysis. This helps you:\n\nInspect model behavior: See actual model outputs during training\nDebug preference learning: Compare chosen vs rejected responses\nTrack reward patterns: Monitor how rewards evolve over training\nShare examples with team: Visual tables in SwanLab dashboard\n\n\n\nFeatures\n\n✅ Automatic detection: Works with DPO, KTO, ORPO, GRPO trainers\n✅ Memory-safe buffering: Bounded buffer prevents memory leaks in long training runs\n✅ Periodic logging: Configurable logging interval to reduce overhead\n✅ Rich visualization: SwanLab tables show prompts, responses, and metrics side-by-side\n\n\n\nConfiguration\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nswanlab_log_completions\nbool\ntrue\nEnable completion logging for RLHF trainers\n\n\nswanlab_completion_log_interval\nint\n100\nLog completions to SwanLab every N training steps\n\n\nswanlab_completion_max_buffer\nint\n128\nMaximum completions to buffer (memory bound)\n\n\n\n\n\nExample: DPO Training with Completion Logging\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: dpo-training\nswanlab_experiment_name: llama-3-dpo-v1\nswanlab_mode: cloud\n\nswanlab_log_completions: true\nswanlab_completion_log_interval: 100 # Log every 100 steps\nswanlab_completion_max_buffer: 128 # Keep last 128 completions\n\nrl: dpo\ndatasets:\n - path: /path/to/preference_dataset\n type: chatml.intel\n\n\nExample: Disable Completion Logging\nIf you’re doing a quick test run or don’t need completion tables:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: dpo-training\n\nswanlab_log_completions: false\n\n\nSupported RLHF Trainers\nThe completion logging callback automatically activates for these trainer types:\n\nDPO (Direct Preference Optimization): Logs prompts, chosen, rejected, reward_diff\nKTO (Kahneman-Tversky Optimization): Logs prompts, completions, labels, rewards\nORPO (Odds Ratio Preference Optimization): Logs prompts, chosen, rejected, log_odds_ratio\nGRPO (Group Relative Policy Optimization): Logs prompts, completions, rewards, advantages\nCPO (Constrained Policy Optimization): Logs prompts, chosen, rejected\n\nFor non-RLHF trainers (standard supervised fine-tuning), the completion callback is automatically skipped.\n\n\nHow It Works\n\nAuto-detection: Plugin detects trainer type at initialization\nBuffering: Completions are buffered in memory (up to swanlab_completion_max_buffer)\nPeriodic logging: Every swanlab_completion_log_interval steps, buffer is logged to SwanLab\nMemory safety: Old completions are automatically dropped when buffer is full (uses collections.deque)\nFinal flush: Remaining completions are logged when training completes\n\n\n\nViewing Completion Tables\nAfter training starts, you can view completion tables in your SwanLab dashboard:\n\nNavigate to your experiment in SwanLab\nLook for the “rlhf_completions” table in the metrics panel\nThe table shows:\n\nstep: Training step when completion was generated\nprompt: Input prompt\nchosen: Preferred response (DPO/ORPO)\nrejected: Non-preferred response (DPO/ORPO)\ncompletion: Model output (KTO/GRPO)\nreward_diff/reward: Reward metrics\nTrainer-specific metrics (e.g., log_odds_ratio for ORPO)\n\n\n\n\nMemory Management\nThe completion buffer is memory-bounded to prevent memory leaks:\nfrom collections import deque\n\nbuffer = deque(maxlen=128) # Old completions automatically dropped\nMemory usage estimate:\n- Average completion: ~500 characters (prompt + responses)\n- Buffer size 128: ~64 KB (negligible)\n- Buffer size 1024: ~512 KB (still small)\nRecommendation: Default buffer size (128) works well for most cases. Increase to 512-1024 only if you need to review more historical completions.\n\n\nPerformance Impact\nCompletion logging has minimal overhead:\n\nBuffering: O(1) append operation, negligible CPU/memory\nLogging: Only happens every N steps (default: 100)\nNetwork: SwanLab batches table uploads efficiently\n\nExpected overhead: < 0.5% per training step\n\n\nTroubleshooting\n\nCompletions not appearing in SwanLab\nCause: Trainer may not be logging completion data in the expected format.\nDiagnostic steps:\n1. Check trainer type detection in logs:\ntext INFO: SwanLab RLHF completion logging enabled for DPOTrainer (type: dpo)\n2. Verify your trainer is an RLHF trainer (DPO/KTO/ORPO/GRPO)\n3. Check if trainer logs completion data (this depends on TRL version)\nNote: The current implementation expects trainers to log completion data in the logs dict during on_log() callback. Some TRL trainers may not expose this data by default. You may need to patch the trainer to expose completions.\n\n\nBuffer fills up too quickly\nCause: High logging frequency with small buffer size.\nSolution: Increase buffer size or logging interval:\nswanlab_completion_log_interval: 200 # Log less frequently\nswanlab_completion_max_buffer: 512 # Larger buffer\n\n\nMemory usage growing over time\nCause: Buffer should be bounded, so this indicates a bug.\nSolution:\n1. Verify swanlab_completion_max_buffer is set\n2. Check SwanLab version is up to date\n3. Report issue with memory profiling data\n\n\n\nPerformance Profiling\nSwanLab integration includes profiling utilities to measure and log execution time of trainer methods. This helps you:\n\nIdentify bottlenecks: Find slow operations in your training loop\nOptimize performance: Track improvements after optimization changes\nMonitor distributed training: See per-rank timing differences\nDebug hangs: Detect methods that take unexpectedly long\n\n\n\nFeatures\n\n✅ Zero-config profiling: Automatic timing of key trainer methods\n✅ Decorator-based: Easy to add profiling to custom methods with @swanlab_profile\n✅ Context manager: Fine-grained profiling with swanlab_profiling_context()\n✅ Advanced filtering: ProfilingConfig for throttling and minimum duration thresholds\n✅ Exception-safe: Logs duration even if function raises an exception\n\n\n\nBasic Usage: Decorator\nAdd profiling to any trainer method with the @swanlab_profile decorator:\nfrom axolotl.integrations.swanlab.profiling import swanlab_profile\n\nclass MyCustomTrainer(AxolotlTrainer):\n @swanlab_profile\n def training_step(self, model, inputs):\n # Your training step logic\n return super().training_step(model, inputs)\n\n @swanlab_profile\n def prediction_step(self, model, inputs, prediction_loss_only):\n # Your prediction logic\n return super().prediction_step(model, inputs, prediction_loss_only)\nThe decorator automatically:\n1. Measures execution time with high-precision timer\n2. Logs to SwanLab as profiling/Time taken: ClassName.method_name\n3. Only logs if SwanLab is enabled (use_swanlab: true)\n4. Gracefully handles exceptions (logs duration, then re-raises)\n\n\nAdvanced Usage: Context Manager\nFor fine-grained profiling within a method:\nfrom axolotl.integrations.swanlab.profiling import swanlab_profiling_context\n\nclass MyTrainer(AxolotlTrainer):\n def complex_training_step(self, model, inputs):\n # Profile just the forward pass\n with swanlab_profiling_context(self, \"forward_pass\"):\n outputs = model(**inputs)\n\n # Profile just the backward pass\n with swanlab_profiling_context(self, \"backward_pass\"):\n loss = outputs.loss\n loss.backward()\n\n return outputs\n\n\nAdvanced Usage: ProfilingConfig\nFilter and throttle profiling logs with ProfilingConfig:\nfrom axolotl.integrations.swanlab.profiling import (\n swanlab_profiling_context_advanced,\n ProfilingConfig,\n)\n\nprofiling_config = ProfilingConfig(\n enabled=True,\n min_duration_ms=1.0, # Only log if duration > 1ms\n log_interval=10, # Log every 10th call\n)\n\nclass MyTrainer(AxolotlTrainer):\n def frequently_called_method(self, data):\n with swanlab_profiling_context_advanced(\n self,\n \"frequent_op\",\n config=profiling_config\n ):\n # This only logs every 10th call, and only if it takes > 1ms\n result = expensive_computation(data)\n return result\nProfilingConfig Parameters:\n- enabled: Enable/disable profiling globally (default: True)\n- min_duration_ms: Minimum duration to log in milliseconds (default: 0.1)\n- log_interval: Log every Nth function call (default: 1 = log all)\nUse cases:\n- High-frequency methods: Use log_interval=100 to reduce logging overhead\n- Filter noise: Use min_duration_ms=1.0 to skip very fast operations\n- Debugging: Use log_interval=1, min_duration_ms=0.0 to log everything\n\n\nViewing Profiling Metrics\nIn your SwanLab dashboard, profiling metrics appear under the “profiling” namespace:\nprofiling/Time taken: AxolotlTrainer.training_step\nprofiling/Time taken: AxolotlTrainer.prediction_step\nprofiling/Time taken: MyTrainer.forward_pass\nprofiling/Time taken: MyTrainer.backward_pass\nYou can:\n- Track over time: See if methods get faster/slower during training\n- Compare runs: Compare profiling metrics across experiments\n- Identify regressions: Detect if a code change slowed down training\n\n\nConfiguration in Axolotl Config\nProfiling is automatically enabled when SwanLab is enabled. No additional config needed:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: my-project\nTo disable profiling while keeping SwanLab enabled:\nfrom axolotl.integrations.swanlab.profiling import DEFAULT_PROFILING_CONFIG\n\nDEFAULT_PROFILING_CONFIG.enabled = False\n\n\nPerformance Impact\n\nDecorator overhead: ~2-5 microseconds per call (negligible)\nContext manager overhead: ~1-3 microseconds (negligible)\nLogging overhead: Only when SwanLab is enabled and method duration exceeds threshold\nNetwork overhead: SwanLab batches metrics efficiently\n\nExpected overhead: < 0.1% per training step (effectively zero)\n\n\nBest Practices\n\nProfile bottlenecks first: Start by profiling suspected slow operations\nUse min_duration_ms: Filter out fast operations (< 1ms) to reduce noise\nThrottle high-frequency calls: Use log_interval for methods called > 100 times/step\nProfile across runs: Compare profiling metrics before/after optimization\nMonitor distributed training: Check for rank-specific slowdowns\n\n\n\nExample: Complete Profiling Setup\nfrom axolotl.integrations.swanlab.profiling import (\n swanlab_profile,\n swanlab_profiling_context,\n ProfilingConfig,\n)\n\nclass OptimizedTrainer(AxolotlTrainer):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n\n # Custom profiling config for high-frequency operations\n self.fast_op_config = ProfilingConfig(\n enabled=True,\n min_duration_ms=0.5,\n log_interval=50,\n )\n\n @swanlab_profile\n def training_step(self, model, inputs):\n \"\"\"Main training step - always profile.\"\"\"\n return super().training_step(model, inputs)\n\n @swanlab_profile\n def compute_loss(self, model, inputs, return_outputs=False):\n \"\"\"Loss computation - always profile.\"\"\"\n return super().compute_loss(model, inputs, return_outputs)\n\n def _prepare_inputs(self, inputs):\n \"\"\"High-frequency operation - throttled profiling.\"\"\"\n with swanlab_profiling_context_advanced(\n self,\n \"prepare_inputs\",\n config=self.fast_op_config,\n ):\n return super()._prepare_inputs(inputs)\n\n\nTroubleshooting\n\nProfiling metrics not appearing in SwanLab\nCause: SwanLab is not enabled or not initialized.\nSolution:\nuse_swanlab: true\nswanlab_project: my-project\nCheck logs for:\nINFO: SwanLab initialized for project: my-project\n\n\nToo many profiling metrics cluttering dashboard\nCause: Profiling every function call for high-frequency operations.\nSolution: Use ProfilingConfig with throttling:\nconfig = ProfilingConfig(\n min_duration_ms=1.0, # Skip fast ops\n log_interval=100, # Log every 100th call\n)\n\n\nProfiling overhead impacting training speed\nCause: Profiling itself should have negligible overhead (< 0.1%). If you see > 1% slowdown, this indicates a bug.\nSolution:\n1. Disable profiling temporarily to confirm:\npython DEFAULT_PROFILING_CONFIG.enabled = False\n2. Report issue with profiling data and trainer details\n\n\nProfiling shows inconsistent timing\nCause: Normal variation due to GPU warmup, data loading, or system load.\nSolution:\n- Ignore first few steps (warmup period)\n- Look at average/median timing over many steps\n- Use log_interval to reduce noise from individual outliers\n\n\n\nComplete Config Example\nHere’s a complete example integrating SwanLab with your RVQ-Alpha training:\nbase_model: /path/to/your/model\nmodel_type: Qwen2ForCausalLM\n\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin\n\nuse_swanlab: true\nswanlab_project: RVQ-Alpha-Training\nswanlab_experiment_name: Qwen2.5-7B-MetaQA-Perturb-P020\nswanlab_description: \"Training on MetaQA and Perturbation datasets with NEW-RVQ encoding\"\nswanlab_mode: cloud\nswanlab_workspace: single-cell-genomics\n\nsequence_len: 32768\nmicro_batch_size: 1\ngradient_accumulation_steps: 1\nnum_epochs: 2\nlearning_rate: 2e-5\noptimizer: adamw_torch_fused\n\ndatasets:\n - path: /path/to/dataset\n type: chat_template\n\noutput_dir: ./outputs\n\n\nModes Explained\n\n\ncloud Mode (Default)\n\nSyncs experiments to SwanLab cloud in real-time\nRequires API key and internet connection\nBest for: Team collaboration, remote monitoring\n\n\n\nlocal Mode\n\nSaves experiments locally only\nNo cloud sync\nBest for: Local development, air-gapped environments\n\n\n\noffline Mode\n\nSaves metadata locally\nCan sync to cloud later using swanlab sync\nBest for: Unstable internet, sync later\n\n\n\ndisabled Mode\n\nTurns off SwanLab completely\nNo logging or tracking\nBest for: Debugging, testing\n\n\n\nConfiguration Validation & Conflict Detection\nSwanLab integration includes comprehensive validation and conflict detection to help you catch configuration errors early and avoid performance issues.\n\n\nRequired Fields Validation\nThe plugin validates your configuration at startup and provides clear error messages with solutions:\n\nMissing Project Name\nuse_swanlab: true\nSolution:\nuse_swanlab: true\nswanlab_project: my-project\n\n\nInvalid Mode\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_mode: invalid-mode\nSolution:\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_mode: cloud # or: local, offline, disabled\n\n\nEmpty Project Name\nuse_swanlab: true\nswanlab_project: \"\"\nSolution:\nuse_swanlab: true\nswanlab_project: my-project\n\n\n\nCloud Mode API Key Warning\nWhen using cloud mode without an API key, you’ll receive a warning with multiple solutions:\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_mode: cloud\nSolutions:\n1. Set environment variable: export SWANLAB_API_KEY=your-api-key\n2. Add to config (less secure): swanlab_api_key: your-api-key\n3. Run swanlab login before training\n4. Use swanlab_mode: local for offline tracking\n\n\nMulti-Logger Performance Warnings\nUsing multiple logging tools simultaneously (SwanLab + WandB + MLflow + Comet) can impact training performance:\n\nTwo Loggers - Warning\nuse_swanlab: true\nswanlab_project: my-project\n\nuse_wandb: true\nwandb_project: my-project\nImpact:\n- Performance overhead: ~1-2% per logger (cumulative)\n- Increased memory usage\n- Longer training time per step\n- Potential config/callback conflicts\nRecommendations:\n- Choose ONE primary logging tool for production training\n- Use multiple loggers only for:\n- Migration period (transitioning between tools)\n- Short comparison runs\n- Debugging specific tool issues\n- Monitor system resources (CPU, memory) during training\n\n\nThree+ Loggers - Error-Level Warning\nuse_swanlab: true\nswanlab_project: my-project\n\nuse_wandb: true\nwandb_project: my-project\n\nuse_mlflow: true\nmlflow_tracking_uri: http://localhost:5000\nWhy This Matters:\n- With 3 loggers: ~4-5% overhead per step → significant slowdown over long training\n- Example: 10,000 steps at 2s/step → ~400-500 seconds extra (6-8 minutes)\n- Memory overhead scales with number of loggers\n- Rare edge cases with callback ordering conflicts\n\n\n\nAuto-Enable Logic\nFor convenience, SwanLab will auto-enable if you specify a project without setting use_swanlab:\nswanlab_project: my-project\n\nuse_swanlab: true\nswanlab_project: my-project\n\n\nDistributed Training Detection\nIn distributed training scenarios (multi-GPU), the plugin automatically detects and reports:\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_mode: cloud\nWhy Only Rank 0:\n- Avoids duplicate experiment runs\n- Reduces network/cloud API overhead on worker ranks\n- Prevents race conditions in metric logging\n\n\nAuthentication\n\n\nMethod 1: Environment Variable (Recommended)\nexport SWANLAB_API_KEY=your-api-key-here\n\n\nMethod 2: Login Command\nswanlab login\n\n\nMethod 3: Config File\nswanlab_api_key: your-api-key-here\n\n\nWhat Gets Logged?\n\n\nAutomatically Logged Metrics\n\nTraining loss\nLearning rate\nGradient norm\nTraining steps\nEpoch progress\n\n\n\nAutomatically Logged Config\n\nModel configuration (base_model, model_type)\nTraining hyperparameters (learning_rate, batch_size, etc.)\nOptimizer settings\nParallelization settings (FSDP, DeepSpeed, Context Parallel)\nAxolotl configuration file\nDeepSpeed configuration (if used)\n\n\n\nViewing Your Experiments\n\n\nCloud Mode\nVisit https://swanlab.cn and navigate to your project to view:\n- Real-time training metrics\n- Hyperparameter comparison\n- System resource usage\n- Configuration files\n\n\nLocal Mode\nswanlab watch ./swanlog\n\n\nIntegration with Existing Tools\nSwanLab can work alongside other tracking tools:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: my-project\n\nuse_wandb: true\nwandb_project: my-project\n\n\nTroubleshooting\n\n\nConfiguration Errors\n\nError: “SwanLab enabled but ‘swanlab_project’ is not set”\nCause: You enabled SwanLab (use_swanlab: true) but forgot to specify a project name.\nSolution:\nuse_swanlab: true\nswanlab_project: my-project # Add this line\n\n\nError: “Invalid swanlab_mode: ‘xxx’”\nCause: You provided an invalid mode value.\nSolution: Use one of the valid modes:\nswanlab_mode: cloud # or: local, offline, disabled\n\n\nError: “swanlab_project cannot be an empty string”\nCause: You set swanlab_project: \"\" (empty string).\nSolution: Either provide a valid name or remove the field:\nswanlab_project: my-project\n\n\n\nImport Errors\n\nError: “SwanLab is not installed”\nCause: SwanLab package is not installed in your environment.\nSolution:\npip install swanlab\npip install swanlab>=0.3.0\n\n\n\nPerformance Issues\n\nWarning: “Multiple logging tools enabled”\nCause: You have multiple experiment tracking tools enabled (e.g., SwanLab + WandB + MLflow).\nImpact: ~1-2% performance overhead per logger, cumulative.\nSolution: For production training, disable all but one logger:\nuse_swanlab: true\nswanlab_project: my-project\nuse_wandb: false # Disable others\nuse_mlflow: false\n\nuse_swanlab: false\nuse_wandb: true\nwandb_project: my-project\nException: Multiple loggers are acceptable for:\n- Short comparison runs (< 100 steps)\n- Migration testing between logging tools\n- Debugging logger-specific issues\n\n\n\nDistributed Training Issues\n\nSwanLab creates duplicate runs in multi-GPU training\nCause: All ranks are initializing SwanLab instead of just rank 0.\nExpected Behavior: The plugin automatically ensures only rank 0 initializes SwanLab. You should see:\nInfo: Distributed training detected (world_size=4)\nInfo: Only rank 0 will initialize SwanLab\nInfo: Other ranks will skip SwanLab to avoid conflicts\nIf you see duplicates:\n1. Check your plugin is loaded correctly\n2. Verify you’re using the latest SwanLab integration code\n3. Check logs for initialization messages on all ranks\n\n\n\nSwanLab not logging metrics\nSolution: Ensure SwanLab is initialized before training starts. The plugin automatically handles this in pre_model_load.\n\n\nAPI Key errors\nSolution:\necho $SWANLAB_API_KEY\n\nswanlab login\n\n\nCloud sync issues\nSolution: Use offline mode and sync later:\nswanlab_mode: offline\nThen sync when ready:\nswanlab sync ./swanlog\n\n\nPlugin not loaded\nSolution: Verify plugin path in config:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin # Correct path\n\n\nLark Notification Issues\n\nError: “Failed to import SwanLab Lark plugin”\nCause: Your SwanLab version doesn’t include the Lark plugin (requires SwanLab >= 0.3.0).\nSolution:\npip install --upgrade swanlab\n\npip install 'swanlab>=0.3.0'\n\n\nWarning: “Lark webhook has no secret configured”\nCause: You provided swanlab_lark_webhook_url but no swanlab_lark_secret.\nImpact: Lark notifications will work, but without HMAC authentication (security risk).\nSolution: Add HMAC secret for production use:\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxx\nswanlab_lark_secret: your-webhook-secret # Add this line\nWhen it’s OK to skip secret:\n- Local development and testing\n- Internal networks with restricted access\n- Non-sensitive training experiments\nWhen secret is required:\n- Production training jobs\n- Training with proprietary data\n- Multi-team shared Lark groups\n\n\nError: “Failed to register Lark callback”\nCause: Invalid webhook URL or network connectivity issues.\nDiagnostic steps:\ncurl -X POST \"YOUR_WEBHOOK_URL\" \\\n -H 'Content-Type: application/json' \\\n -d '{\"msg_type\":\"text\",\"content\":{\"text\":\"Test from Axolotl\"}}'\n\npip show swanlab\nSolution:\n1. Verify webhook URL is correct (copy from Lark bot settings)\n2. Check network connectivity to Lark API\n3. Ensure webhook is not expired (Lark webhooks can expire)\n4. Regenerate webhook URL in Lark bot settings if needed\n\n\nLark notifications not received\nCause: Multiple possible causes.\nDiagnostic checklist:\n\nCheck training logs for Lark registration confirmation:\n# Expected log message (rank 0 only):\nINFO: Registered Lark notification callback with HMAC authentication\nVerify webhook in Lark: Test webhook manually (see above)\nCheck distributed training: Only rank 0 sends notifications\n# If running multi-GPU, check rank 0 logs specifically\ngrep \"Registered Lark\" logs/rank_0.log\nVerify SwanLab is initialized: Lark callback needs SwanLab to be running\nuse_swanlab: true # Must be enabled\nswanlab_project: my-project # Must be set\nCheck Lark bot permissions: Ensure bot is added to the target group chat\n\n\n\nDuplicate Lark notifications in multi-GPU training\nExpected Behavior: Should NOT happen - only rank 0 sends notifications.\nIf you see duplicates:\n1. Check that all GPUs are using the same config file\n2. Verify plugin is loaded correctly on all ranks\n3. Check logs for unexpected Lark initialization on non-zero ranks\n4. Ensure RANK or LOCAL_RANK environment variables are set correctly\nSolution: This is a bug if it occurs. Report with:\n- Full training command\n- Logs from all ranks\n- Config file\n\n\n\nComparison: SwanLab vs WandB\n\n\n\nFeature\nSwanLab\nWandB\n\n\n\n\nOpen Source\n✅ Yes\n❌ No\n\n\nSelf-Hosting\n✅ Easy\n⚠️ Complex\n\n\nFree Tier\n✅ Generous\n⚠️ Limited\n\n\nChinese Support\n✅ Native\n⚠️ Limited\n\n\nOffline Mode\n✅ Full support\n✅ Supported\n\n\nIntegration\n🆕 New\n✅ Mature\n\n\n\n\n\nAdvanced Usage\n\n\nCustom Logging\nYou can add custom metrics in your callbacks:\nimport swanlab\n\nswanlab.log({\n \"custom_metric\": value,\n \"epoch\": epoch_num\n})\n\n\nExperiment Comparison\nswanlab compare run1 run2 run3\n\n\nSupport\n\nDocumentation: https://docs.swanlab.cn\nGitHub: https://github.com/SwanHubX/SwanLab\nIssues: Report bugs at GitHub Issues\n\n\n\nLicense\nThis integration follows the Axolotl Community License Agreement.\n\n\nAcknowledgements\nThis integration is built on top of:\n- SwanLab - Experiment tracking tool\n- Transformers - SwanLabCallback\n- Axolotl - Training framework\nPlease see reference here", + "crumbs": [ + "Advanced Features", + "Custom Integrations" + ] + }, + { + "objectID": "docs/custom_integrations.html#adding-a-new-integration", + "href": "docs/custom_integrations.html#adding-a-new-integration", + "title": "Custom Integrations", + "section": "Adding a new integration", + "text": "Adding a new integration\nPlugins can be used to customize the behavior of the training pipeline through hooks. See axolotl.integrations.BasePlugin for the possible hooks.\nTo add a new integration, please follow these steps:\n\nCreate a new folder in the src/axolotl/integrations directory.\nAdd any relevant files (LICENSE, README.md, ACKNOWLEDGEMENTS.md, etc.) to the new folder.\nAdd __init__.py and args.py files to the new folder.\n\n\n__init__.py should import the integration and hook into the appropriate functions.\nargs.py should define the arguments for the integration.\n\n\n(If applicable) Add CPU tests under tests/integrations or GPU tests under tests/e2e/integrations.\n\n\n\n\n\n\n\nTip\n\n\n\nSee src/axolotl/integrations/cut_cross_entropy for a minimal integration example.\n\n\n\n\n\n\n\n\nWarning\n\n\n\nIf you could not load your integration, please ensure you are pip installing in editable mode.\npip install -e .\nand correctly spelled the integration name in the config file.\nplugins:\n - axolotl.integrations.your_integration_name.YourIntegrationPlugin\n\n\n\n\n\n\n\n\nNote\n\n\n\nIt is not necessary to place your integration in the integrations folder. It can be in any location, so long as it’s installed in a package in your python env.\nSee this repo for an example: https://github.com/axolotl-ai-cloud/diff-transformer", + "crumbs": [ + "Advanced Features", + "Custom Integrations" + ] + }, + { + "objectID": "docs/ray-integration.html", + "href": "docs/ray-integration.html", + "title": "Ray Train", + "section": "", + "text": "Axolotl supports using Ray as an alternative to accelerate for orchestrating training. This is especially useful for multi-node training since you only have to setup code and dependencies in a single node and launch training as if you were using a single node.\nWith the --use-ray CLI flag, Axolotl will use Ray Train’s TorchTrainer to run training.", + "crumbs": [ + "Deployments", + "Ray Train" + ] + }, + { + "objectID": "docs/ray-integration.html#ray-cluster-setup", + "href": "docs/ray-integration.html#ray-cluster-setup", + "title": "Ray Train", + "section": "Ray cluster setup", + "text": "Ray cluster setup\nA prerequisite using the Ray Train integration is to setup a Ray cluster on your desired node(s). For a detailed guide on how you can get started with ray clusters, check the official Ray docs here.\nEvery Ray cluster has one head node and a set of worker nodes. The head node is just like any other worker node, but it also runs certain special processes related to scheduling and orchestration. Ray-enabled scripts are run on the head node and depending on the resources (number of CPUs, GPUs, etc) they request, will be scheduled to run certain tasks on the worker nodes. For more on key concepts behind a Ray cluster, you can refer this doc.", + "crumbs": [ + "Deployments", + "Ray Train" + ] + }, + { + "objectID": "docs/ray-integration.html#sanity-check", + "href": "docs/ray-integration.html#sanity-check", + "title": "Ray Train", + "section": "Sanity check", + "text": "Sanity check\nTo run a sanity check on whether your ray cluster is setup properly, execute the following on the head node:\nray status\nThe output should have a summary of your Ray cluster - list of all the nodes in your cluster, the number of CPUs and GPUs in your cluster, etc. For example, if you have a cluster with 1 CPU-only head node and 2 4xL40S worker nodes, the output can look like this:\nNode status\n---------------------------------------------------------------\nActive:\n 1 head\nIdle:\n 2 4xL40S:48CPU-384GB\nPending:\n (no pending nodes)\nRecent failures:\n (no failures)\n\nResources\n---------------------------------------------------------------\nUsage:\n 0.0/96.0 CPU\n 0.0/8.0 GPU\n 0B/800.00GiB memory\n 0B/229.57GiB object_store_memory\n\nDemands:\n (no resource demands)\nYou should also be able to see the same on the Ray dashboard.", + "crumbs": [ + "Deployments", + "Ray Train" + ] + }, + { + "objectID": "docs/ray-integration.html#configuring-training-with-ray-train", + "href": "docs/ray-integration.html#configuring-training-with-ray-train", + "title": "Ray Train", + "section": "Configuring training with Ray Train", + "text": "Configuring training with Ray Train\nYou can find an example configuration at configs/llama-3/lora-1b-ray.yaml.\nThe key parameters to note here are:\nuse_ray: true\nray_num_workers: 4\n# optional\nresources_per_worker:\n GPU: 1\n\nuse_ray: This is the flag that enables the Ray Train integration. You can either use the corresponding --use-ray flag in the CLI or set use_ray in the config file.\nray_num_workers: This is the number of workers/GPUs to use for training.\nresources_per_worker: This is the Ray resource request for each worker. This can be used to request a specific GPU type or a custom resource for each worker. For example, if your ray cluster has GPUs of different types, and you only want to use NVIDIA L40S GPUs, you can do\n\nresources_per_worker:\n accelerator_type:L40S: 0.001", + "crumbs": [ + "Deployments", + "Ray Train" + ] + }, + { + "objectID": "docs/ray-integration.html#launching-training", + "href": "docs/ray-integration.html#launching-training", + "title": "Ray Train", + "section": "Launching training", + "text": "Launching training\nYou can simply run the following command on the head node:\naxolotl train examples/llama-3/lora-1b-ray.yml --use-ray\nThis will launch training on the head node and workers will be scheduled automatically by Ray Train to run on the appropriate head or worker nodes.\nYou can also monitor training progress on the Ray dashboard.\nComing back to the example on a Ray cluster with 1 head node and 2 4xL40S worker nodes, let’s say you want to make use of all 8 GPUs. You would be able to just set ray_num_workers: 8 and run the previous command. The Cluster tab will show the following:\n\n\n\nRay dashboard", + "crumbs": [ + "Deployments", + "Ray Train" ] }, { @@ -3537,36 +3934,190 @@ ] }, { - "objectID": "docs/multipack.html", - "href": "docs/multipack.html", - "title": "Multipack (Sample Packing)", + "objectID": "docs/grpo.html", + "href": "docs/grpo.html", + "title": "GRPO Training", "section": "", - "text": "Because Flash Attention simply drops the attention mask, we do not need to\nconstruct a 4d attention mask. We only need to concatenate the sequences into\na single batch and let flash attention know where each new sequence begins.\n4k context, bsz =4,\neach character represents 256 tokens\nX represents a padding token\n 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n[[ A A A A A A A A A A A ]\n B B B B B B ]\n C C C C C C C ]\n D D D D ]]\n\n[[ E E E E E E E E ]\n [ F F F F ]\n [ G G G ]\n [ H H H H ]]\n\n[[ I I I ]\n [ J J J ]\n [ K K K K K]\n [ L L L ]]\nafter padding to longest input in each step\n 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n[[ A A A A A A A A A A A ]\n B B B B B B X X X X X X ]\n C C C C C C C X X X X ]\n D D D D X X X X X X X ]]\n\n[[ E E E E E E E E ]\n [ F F F F X X X X ]\n [ G G G X X X X X ]\n [ H H H H X X X X ]]\n\n[[ I I I X X ]\n [ J J J X X ]\n [ K K K K K ]\n [ L L L X X ]]\nw packing ( note it’s the same effective number of tokens per step, but a true bsz of 1)\n 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n[[ A A A A A A A A A A A B B B B B\n B C C C C C C C D D D D E E E E\n E E E E F F F F F G G G H H H H\n I I I J J J J K K K K K L L L X ]]\ncu_seqlens:\n[[ 0, 11, 17, 24, 28, 36, 41 44, 48, 51, 55, 60, 64]]", + "text": "Group Relative Policy Optimization (GRPO) is a reinforcement learning method that improves language models by generating multiple completions per prompt, scoring them with reward functions, and using the relative ranking within each group to compute advantage estimates. Unlike DPO, which requires pre-collected preference pairs, GRPO generates its own training data online and can work with any programmatic reward signal (math correctness, format compliance, code execution results, etc.).\nUse GRPO when you have a task with a verifiable reward signal and want the model to discover solution strategies on its own. Use DPO when you already have human preference data. Use SFT when you have gold-standard completions to imitate directly.\nAxolotl’s GRPO implementation builds on TRL and adds async generation, streaming scoring, importance sampling correction, replay buffers, and multi-GPU scaling via FSDP and DeepSpeed.", "crumbs": [ - "Core Concepts", - "Multipack (Sample Packing)" + "How To Guides", + "GRPO Training" ] }, { - "objectID": "docs/multipack.html#visualization-of-multipack-with-flash-attention", - "href": "docs/multipack.html#visualization-of-multipack-with-flash-attention", - "title": "Multipack (Sample Packing)", + "objectID": "docs/grpo.html#overview", + "href": "docs/grpo.html#overview", + "title": "GRPO Training", "section": "", - "text": "Because Flash Attention simply drops the attention mask, we do not need to\nconstruct a 4d attention mask. We only need to concatenate the sequences into\na single batch and let flash attention know where each new sequence begins.\n4k context, bsz =4,\neach character represents 256 tokens\nX represents a padding token\n 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n[[ A A A A A A A A A A A ]\n B B B B B B ]\n C C C C C C C ]\n D D D D ]]\n\n[[ E E E E E E E E ]\n [ F F F F ]\n [ G G G ]\n [ H H H H ]]\n\n[[ I I I ]\n [ J J J ]\n [ K K K K K]\n [ L L L ]]\nafter padding to longest input in each step\n 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n[[ A A A A A A A A A A A ]\n B B B B B B X X X X X X ]\n C C C C C C C X X X X ]\n D D D D X X X X X X X ]]\n\n[[ E E E E E E E E ]\n [ F F F F X X X X ]\n [ G G G X X X X X ]\n [ H H H H X X X X ]]\n\n[[ I I I X X ]\n [ J J J X X ]\n [ K K K K K ]\n [ L L L X X ]]\nw packing ( note it’s the same effective number of tokens per step, but a true bsz of 1)\n 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5\n[[ A A A A A A A A A A A B B B B B\n B C C C C C C C D D D D E E E E\n E E E E F F F F F G G G H H H H\n I I I J J J J K K K K K L L L X ]]\ncu_seqlens:\n[[ 0, 11, 17, 24, 28, 36, 41 44, 48, 51, 55, 60, 64]]", + "text": "Group Relative Policy Optimization (GRPO) is a reinforcement learning method that improves language models by generating multiple completions per prompt, scoring them with reward functions, and using the relative ranking within each group to compute advantage estimates. Unlike DPO, which requires pre-collected preference pairs, GRPO generates its own training data online and can work with any programmatic reward signal (math correctness, format compliance, code execution results, etc.).\nUse GRPO when you have a task with a verifiable reward signal and want the model to discover solution strategies on its own. Use DPO when you already have human preference data. Use SFT when you have gold-standard completions to imitate directly.\nAxolotl’s GRPO implementation builds on TRL and adds async generation, streaming scoring, importance sampling correction, replay buffers, and multi-GPU scaling via FSDP and DeepSpeed.", "crumbs": [ - "Core Concepts", - "Multipack (Sample Packing)" + "How To Guides", + "GRPO Training" ] }, { - "objectID": "docs/multipack.html#multipack-without-flash-attention", - "href": "docs/multipack.html#multipack-without-flash-attention", - "title": "Multipack (Sample Packing)", - "section": "Multipack without Flash Attention", - "text": "Multipack without Flash Attention\nMultipack can still be achieved without Flash attention, but with lower packing\nefficiency as we are not able to join multiple batches into a single batch due to\ncontext length limits without flash attention. We can use either Pytorch’s Scaled\nDot Product Attention implementation or native Pytorch attention implementation\nalong with 4d attention masks\nto pack sequences together and avoid cross attention.", + "objectID": "docs/grpo.html#architecture", + "href": "docs/grpo.html#architecture", + "title": "GRPO Training", + "section": "Architecture", + "text": "Architecture\nGRPO training uses a two-process architecture: a vLLM server for fast generation and a trainer process for scoring and gradient updates.\nTerminal 1 (GPU 0) Terminal 2 (GPU 1)\n┌──────────────────────┐ ┌──────────────────────────────────┐\n│ vLLM Server │ │ Trainer │\n│ │ HTTP │ │\n│ Serves base model │◄────────────►│ Background thread: │\n│ + LoRA adapter │ /generate │ Send prompts to vLLM │\n│ │ /set_lora │ Pad & collate completions │\n│ Punica kernels for │ │ │\n│ LoRA inference │ │ Main thread: │\n│ │ │ Score completions (rewards) │\n└──────────────────────┘ │ Compute policy log-probs │\n │ Calculate advantages │\n │ PPO-clip gradient update │\n │ Sync LoRA weights to vLLM │\n └──────────────────────────────────┘\nData flow for each training step:\n\nThe background thread sends prompts to vLLM, which generates num_generations completions per prompt.\nThe main thread scores completions using your reward functions.\nAdvantages are computed within each prompt group (group-relative normalization).\nPolicy log-probabilities are computed by running a forward pass on the training model.\nThe PPO-clip loss is computed and gradients are applied.\nPeriodically, LoRA adapter weights are synced back to vLLM so future generations reflect the updated policy.\n\nWith async prefetch enabled, step 1 for the next batch runs concurrently with steps 2-6 for the current batch.", "crumbs": [ - "Core Concepts", - "Multipack (Sample Packing)" + "How To Guides", + "GRPO Training" + ] + }, + { + "objectID": "docs/grpo.html#quick-start", + "href": "docs/grpo.html#quick-start", + "title": "GRPO Training", + "section": "Quick Start", + "text": "Quick Start\nA GRPO training run requires three components: a YAML config, a reward module (Python file), and a running vLLM server.\n\n1. Write a reward module\nCreate a file called rewards.py in your working directory:\n# rewards.py\nimport re\n\n\ndef accuracy_reward(completions, answer, **kwargs) -> list[float]:\n \"\"\"Check if the completion contains the correct numerical answer.\"\"\"\n rewards = []\n for completion, correct in zip(completions, answer):\n text = completion[0][\"content\"]\n # Extract the last number from the completion\n numbers = re.findall(r\"-?\\d+(?:\\.\\d+)?\", text)\n predicted = numbers[-1] if numbers else \"\"\n rewards.append(1.0 if predicted == str(correct) else 0.0)\n return rewards\n\n\ndef format_reward(completions, **kwargs) -> list[float]:\n \"\"\"Reward completions that use a structured thinking format.\"\"\"\n rewards = []\n for completion in completions:\n text = completion[0][\"content\"]\n has_think = \"<think>\" in text and \"</think>\" in text\n has_answer = \"<answer>\" in text and \"</answer>\" in text\n rewards.append(1.0 if has_think and has_answer else 0.0)\n return rewards\n\n\ndef prompt_transform(cfg, *args, **kwargs):\n \"\"\"Convert GSM8K dataset rows into chat prompts.\"\"\"\n def transform_fn(example, tokenizer=None):\n label = example[\"answer\"].split(\"####\")[-1].strip().replace(\",\", \"\")\n return {\n \"prompt\": [\n {\"role\": \"system\", \"content\": \"Solve the math problem. Show your reasoning in <think> tags and your final numerical answer in <answer> tags.\"},\n {\"role\": \"user\", \"content\": example[\"question\"]},\n ],\n \"answer\": label,\n }\n return transform_fn, {\"remove_columns\": [\"question\"]}\n\n\n2. Write the config\nCreate config.yaml:\nbase_model: Qwen/Qwen2.5-1.5B-Instruct\n\nrl: grpo\nchat_template: tokenizer_default\n\nvllm:\n host: 0.0.0.0\n port: 8000\n gpu_memory_utilization: 0.85\n dtype: auto\n max_model_len: 2048\n\nadapter: lora\nlora_r: 32\nlora_alpha: 64\nlora_target_linear: true\n\ntrl:\n use_vllm: true\n use_data_producer: true\n vllm_server_host: 0.0.0.0\n vllm_server_port: 8000\n vllm_server_timeout: 300\n vllm_lora_sync: true\n num_generations: 8\n max_completion_length: 512\n temperature: 0.7\n reward_funcs:\n - rewards.accuracy_reward\n - rewards.format_reward\n reward_weights:\n - 1.0\n - 0.5\n\ndatasets:\n - path: openai/gsm8k\n name: main\n type: rewards.prompt_transform\n split: train\n\nskip_prepare_dataset: true\nval_set_size: 0.0\nsequence_len: 512\nmicro_batch_size: 2\ngradient_accumulation_steps: 4\nmax_steps: 200\nlearning_rate: 5.0e-6\noptimizer: adamw_torch_fused\nlr_scheduler: cosine\nwarmup_steps: 10\n\nbf16: true\nflash_attention: true\ngradient_checkpointing: true\n\nspecial_tokens:\n pad_token: \"<|endoftext|>\"\n\noutput_dir: ./grpo-output\nlogging_steps: 1\n\n\n3. Start vLLM and train\n# Terminal 1: Start vLLM server on GPU 0\nCUDA_VISIBLE_DEVICES=0 axolotl vllm-serve config.yaml\n\n# Wait 30-90 seconds for model loading and CUDA graph capture\n\n# Terminal 2: Train on GPU 1\nCUDA_VISIBLE_DEVICES=1 axolotl train config.yaml\n\n\n\n\n\n\nTip\n\n\n\nUse tmux or separate terminal sessions to manage the two processes. The vLLM server must remain running for the entire training duration.", + "crumbs": [ + "How To Guides", + "GRPO Training" + ] + }, + { + "objectID": "docs/grpo.html#custom-reward-functions", + "href": "docs/grpo.html#custom-reward-functions", + "title": "GRPO Training", + "section": "Custom Reward Functions", + "text": "Custom Reward Functions\n\nFunction signature\nTRL calls reward functions with this signature:\ndef my_reward(completions, **kwargs) -> list[float]:\n\ncompletions is a list of single-element lists, where each element is a dict {\"role\": \"assistant\", \"content\": \"...\"}. So completions[i][0][\"content\"] gives you the text of the i-th completion.\n**kwargs contains all dataset columns that were not removed by the dataset transform. This is how you pass ground truth answers, metadata, or any other information to your reward function.\nReturn a list[float] with the same length as completions. You may return None for individual elements to exclude them from aggregation.\n\n\n\nExample: accuracy reward with answer extraction\ndef accuracy_reward(completions, answer, **kwargs) -> list[float]:\n rewards = []\n for completion, correct_answer in zip(completions, answer):\n text = completion[0][\"content\"]\n # Extract answer from <answer>...</answer> tags\n match = re.search(r\"<answer>(.*?)</answer>\", text, re.DOTALL)\n predicted = match.group(1).strip() if match else \"\"\n rewards.append(1.0 if predicted == str(correct_answer) else 0.0)\n return rewards\n\n\nExample: length penalty\ndef length_penalty(completions, **kwargs) -> list[float]:\n \"\"\"Penalize very short or very long completions.\"\"\"\n rewards = []\n for completion in completions:\n length = len(completion[0][\"content\"])\n if length < 50:\n rewards.append(-0.5)\n elif length > 2000:\n rewards.append(-0.2)\n else:\n rewards.append(0.0)\n return rewards\n\n\nMultiple rewards and weighting\nYou can combine multiple reward functions with different weights:\ntrl:\n reward_funcs:\n - rewards.accuracy_reward\n - rewards.format_reward\n - rewards.length_penalty\n reward_weights:\n - 1.0 # accuracy is most important\n - 0.5 # format compliance\n - 0.1 # mild length preference\nRewards are combined by the multi_objective_aggregation strategy:\n\nsum_then_normalize (default): weights and sums all rewards first, then normalizes across the group.\nnormalize_then_sum (GDPO): normalizes each reward independently, then sums. This prevents one reward from dominating and is recommended when using multiple reward functions with different scales.\n\ntrl:\n multi_objective_aggregation: normalize_then_sum\n\n\nDataset transforms\nThe dataset transform converts raw HuggingFace dataset rows into chat-format prompts:\ndef prompt_transform(cfg, *args, **kwargs):\n def map_fn(example, tokenizer=None):\n return {\n \"prompt\": [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": example[\"question\"]},\n ],\n # Keep 'answer' column for the reward function\n \"answer\": example[\"answer\"],\n }\n # Remove columns consumed by the transform; keep columns needed by rewards\n return map_fn, {\"remove_columns\": [\"question\"]}\nThe transform returns a tuple of (map_function, kwargs_dict). The remove_columns in the kwargs dict removes columns that are no longer needed. Columns that your reward functions reference via **kwargs (like answer) must not be removed.\n\n\n\n\n\n\nWarning\n\n\n\nThe reward module must be importable from the directory where you run axolotl train. If your reward file is rewards.py, the import path is rewards.accuracy_reward. If it is inside a package my_rewards/scoring.py, use my_rewards.scoring.accuracy_reward.\n\n\n\n\nReward models (neural network rewards)\nInstead of a Python function, you can pass a HuggingFace model path as a reward function. TRL will load it as a reward model and use its scalar output as the reward:\ntrl:\n reward_funcs:\n - OpenAssistant/reward-model-deberta-v3-large-v2\n - rewards.format_reward\n reward_weights:\n - 1.0\n - 0.3\n\n\nUsing math_verify\nThe math_verify library provides robust mathematical answer verification but uses signal.alarm() internally, which only works in the main thread. If you use math_verify in a reward function, set reward_num_workers to use subprocess workers:\ntrl:\n reward_num_workers: 4\nEach worker runs in its own subprocess with its own main thread, so signal.alarm() works correctly.", + "crumbs": [ + "How To Guides", + "GRPO Training" + ] + }, + { + "objectID": "docs/grpo.html#vllm-setup", + "href": "docs/grpo.html#vllm-setup", + "title": "GRPO Training", + "section": "vLLM Setup", + "text": "vLLM Setup\nGRPO requires a running vLLM server for generation. For a complete guide on server modes, LoRA sync, weight synchronization, and restart procedures, see vLLM Serving.\nThe minimal setup:\nvllm:\n host: 0.0.0.0\n port: 8000\n gpu_memory_utilization: 0.85\n\ntrl:\n use_vllm: true\n vllm_lora_sync: true # Recommended with LoRA — faster sync, no NCCL contention\n vllm_sync_interval: 5 # Sync weights every 5 steps\nCUDA_VISIBLE_DEVICES=0 axolotl vllm-serve config.yaml # GPU 0: vLLM\nCUDA_VISIBLE_DEVICES=1 axolotl train config.yaml # GPU 1: training\n\n\n\n\n\n\nWarning\n\n\n\nvLLM must be restarted between experiments — stale weight syncs corrupt server state. See Restart Requirements.", + "crumbs": [ + "How To Guides", + "GRPO Training" + ] + }, + { + "objectID": "docs/grpo.html#async-training-features", + "href": "docs/grpo.html#async-training-features", + "title": "GRPO Training", + "section": "Async Training Features", + "text": "Async Training Features\nAsync GRPO overlaps generation and training to reduce wall-clock time. While the model trains on the current batch, the next batch is already being generated by vLLM.\n\nEnabling async prefetch\ntrl:\n use_data_producer: true\n async_prefetch: true\n prefetch_depth: 1\n vllm_sync_interval: 2\n\nuse_data_producer: true enables the data producer protocol (required for all async features).\nasync_prefetch: true runs generation in a background thread.\nprefetch_depth controls how many batches to prefetch ahead (1 is usually sufficient).\nvllm_sync_interval controls how often LoRA weights are synced to vLLM (every N optimizer steps). Lower values mean fresher generations but more sync overhead.\n\n\n\n\n\n\n\nTip\n\n\n\nBecause the background thread generates with slightly stale model weights, async mode benefits from importance sampling correction (see next section). Enable vllm_importance_sampling_correction: true when using async_prefetch: true.\n\n\n\n\nStreaming partial batch\nInstead of scoring the entire batch at once, streaming mode scores one prompt group at a time. This reduces peak memory during scoring and enables finer-grained zero-advantage skipping.\ntrl:\n streaming_partial_batch: true\n streaming_min_groups: 1\nstreaming_min_groups controls the minimum number of prompt groups scored per chunk. Setting it to 1 gives maximum granularity.\n\n\nZero-advantage batch skipping\nWhen all advantages in a micro-batch are zero (every completion in the group got the same reward), there is no learning signal. This feature skips the forward/backward pass entirely for such micro-batches.\ntrl:\n skip_zero_advantage_batches: true # default\nThis is enabled by default and logged as skipped_zero_adv_batches in training metrics. It is a safety net, not a major optimization – it only saves significant time when the model cannot solve any prompts in the batch.\n\n\nReplay buffer\nThe replay buffer caches rollout groups that had learning signal (non-zero reward variance) and replaces zero-signal groups in later batches. This improves data utilization when many prompts yield no reward variance.\ntrl:\n replay_buffer_size: 100\n replay_recompute_logps: true\n\n\n\n\n\n\nWarning\n\n\n\nWhen replay_recompute_logps: false, replayed data uses stale log-probabilities which creates an IS mismatch. Keep the default true unless you have a specific reason to disable it.\n\n\n\n\nDeferred re-rolling\nPrompts where the model gets zero reward for all generations are buffered and re-injected into later batches, when the model may have improved enough to produce useful completions.\ntrl:\n reroll_start_fraction: 0.5 # Start re-rolling after 50% of training\n reroll_max_groups: 1 # Max groups to replace per batch\nSet reroll_start_fraction: 1.0 to disable. This is most useful for tasks where the model starts weak but steadily improves.\n\n\nParallel reward workers\nReward functions that use signal.alarm() (like math_verify) only work in the main thread. Parallel reward workers run each function in its own subprocess:\ntrl:\n reward_num_workers: 4\nWork is sharded across workers by prompt group. For simple reward functions, a single worker is usually sufficient – the overhead of IPC can exceed the computation time.", + "crumbs": [ + "How To Guides", + "GRPO Training" + ] + }, + { + "objectID": "docs/grpo.html#importance-sampling-and-off-policy-correction", + "href": "docs/grpo.html#importance-sampling-and-off-policy-correction", + "title": "GRPO Training", + "section": "Importance Sampling and Off-Policy Correction", + "text": "Importance Sampling and Off-Policy Correction\nWhen using async prefetch, completions are generated from a slightly older policy. IS correction adjusts the gradient to account for this mismatch.\ntrl:\n vllm_importance_sampling_correction: true\n importance_sampling_level: token # 'token' recommended (especially with Liger kernel)\n off_policy_mask_threshold: 0.5 # KL threshold — masks sequences that are too off-policy\nUse token level IS. Sequence-level has numerical issues with Liger’s chunked computation. The off_policy_mask_threshold (OPSM) is a safety net that drops sequences where KL divergence exceeds the threshold — 0.5 is a reasonable starting point.\nFor detailed coverage of IS modes (token_mask, token_truncate, etc.), capping, and bias-corrected KL, see vLLM Serving — IS Correction.", + "crumbs": [ + "How To Guides", + "GRPO Training" + ] + }, + { + "objectID": "docs/grpo.html#scaling", + "href": "docs/grpo.html#scaling", + "title": "GRPO Training", + "section": "Scaling", + "text": "Scaling\n\nFP8 training\nFP8 quantization halves model VRAM usage with minimal impact on training quality. It does not significantly speed up computation for small models but allows larger models to fit in memory.\nfp8: true\ntorch_compile: true\n\n\n\n\n\n\nWarning\n\n\n\nFP8 requires patching for zero-padding edge cases. The act_quant_kernel can produce NaN when input is all zeros (padding positions). If you see NaN in grad norms, check whether your padding token embedding is non-zero.\n\n\n\n\nFSDP (Fully Sharded Data Parallel)\nFSDP distributes model parameters across multiple GPUs for training while vLLM runs on a separate GPU:\nfsdp:\n - full_shard\n - auto_wrap\nfsdp_config:\n fsdp_transformer_layer_cls_to_wrap: Qwen2DecoderLayer\ngradient_checkpointing_kwargs:\n use_reentrant: false\nLaunch with:\n# GPU 0: vLLM\nCUDA_VISIBLE_DEVICES=0 axolotl vllm-serve config.yaml\n\n# GPUs 0,1: Training (FSDP will use both visible GPUs)\nCUDA_VISIBLE_DEVICES=0,1 axolotl train config.yaml\n\n\n\n\n\n\nWarning\n\n\n\nasync_prefetch: true can deadlock with FSDP because background threads perform unsynchronized FSDP collectives across ranks. With multi-GPU FSDP, only rank 0 generates in the background thread and results are broadcast to all ranks. If you still see hangs, set async_prefetch: false.\n\n\n\n\nDeepSpeed ZeRO-3\ndeepspeed: deepspeed_configs/zero3_bf16.json\ngradient_checkpointing_kwargs:\n use_reentrant: true # Required -- non-reentrant causes CheckpointError with ZeRO-3\n\n\n\n\n\n\nNote\n\n\n\nDeepSpeed ZeRO-3 requires use_reentrant: true for gradient checkpointing. This is the opposite of the FSDP recommendation. Non-reentrant checkpointing causes tensor metadata mismatches during recomputation with ZeRO-3’s parameter partitioning.\n\n\n\n\nMulti-GPU considerations\n\n\n\n\n\n\n\nConcern\nRecommendation\n\n\n\n\nvLLM GPU allocation\nDedicate one or more GPUs to vLLM; do not share with trainer GPUs\n\n\nWeight sync contention\nUse vllm_lora_sync: true to avoid NCCL contention between training and vLLM\n\n\nFSDP + async\nUse async_prefetch: false or rely on rank-0-only background generation\n\n\nDeepSpeed + gradient checkpoint\nMust use use_reentrant: true\n\n\nOOM during scoring\nReduce micro_batch_size or num_generations. The logits tensor scales with batch_size * vocab_size", + "crumbs": [ + "How To Guides", + "GRPO Training" + ] + }, + { + "objectID": "docs/grpo.html#monitoring-and-debugging", + "href": "docs/grpo.html#monitoring-and-debugging", + "title": "GRPO Training", + "section": "Monitoring and Debugging", + "text": "Monitoring and Debugging\nFor detailed metric ranges, failure diagnosis, and OOM debugging, see Training Stability & Debugging.\nQuick health checks during GRPO training:\n\nrewards/*/mean should be > 0.15 within 20 steps — if it stays at 0, test your reward function standalone\nreward_std should be > 0 on most steps — all-zero means no learning signal\nentropy in 0.05-0.5 — below 0.01 suggests mode collapse\ngrad_norm in 0.001-1.0 — > 10 is unstable, 0.0 is expected when zero-advantage skip fires\n\n\n\n\n\n\n\nTip\n\n\n\nPipe training output to a log file: axolotl train config.yaml 2>&1 | tee /tmp/training.log", + "crumbs": [ + "How To Guides", + "GRPO Training" + ] + }, + { + "objectID": "docs/grpo.html#configuration-reference", + "href": "docs/grpo.html#configuration-reference", + "title": "GRPO Training", + "section": "Configuration Reference", + "text": "Configuration Reference\nAll GRPO-specific options live under the trl: key in your config. Standard training options (learning_rate, micro_batch_size, etc.) are set at the top level as usual.\n\nCore GRPO\n\n\n\n\n\n\n\n\n\nOption\nType\nDefault\nDescription\n\n\n\n\nuse_vllm\nbool\nfalse\nEnable vLLM for generation\n\n\nvllm_mode\n\"server\" or \"colocate\"\nnull\nvLLM deployment mode\n\n\nvllm_server_host\nstr\n\"0.0.0.0\"\nvLLM server hostname\n\n\nvllm_server_port\nint\n8000\nvLLM server port\n\n\nvllm_server_timeout\nint\nnull\nTimeout (seconds) for vLLM responses\n\n\nnum_generations\nint\nnull\nCompletions generated per prompt\n\n\ngeneration_batch_size\nint\nnull\nNumber of unique prompts per generation step\n\n\nmax_completion_length\nint\nnull\nMaximum tokens per completion\n\n\nbeta\nfloat\nnull\nKL penalty coefficient\n\n\nnum_iterations\nint\nnull\nIterations per batch (mu in the GRPO paper)\n\n\nepsilon\nfloat\nnull\nPPO clipping lower bound\n\n\nepsilon_high\nfloat\nnull\nPPO clipping upper bound\n\n\nloss_type\nstr\nnull\nLoss formulation: grpo, bnpo, or dr_grpo\n\n\nscale_rewards\nbool\ntrue\nNormalize rewards by standard deviation\n\n\nmask_truncated_completions\nbool\nfalse\nExclude truncated completions from loss\n\n\n\n\n\nReward functions\n\n\n\n\n\n\n\n\n\nOption\nType\nDefault\nDescription\n\n\n\n\nreward_funcs\nlist[str]\nnull\nImport paths to reward functions or HF model IDs\n\n\nreward_weights\nlist[float]\nnull\nRelative weights for each reward function\n\n\nmulti_objective_aggregation\nstr\nnull\n\"sum_then_normalize\" (GRPO) or \"normalize_then_sum\" (GDPO)\n\n\nrollout_func\nstr\nnull\nImport path to custom rollout function for OpenEnv-style tasks\n\n\n\n\n\nGeneration parameters\n\n\n\n\n\n\n\n\n\nOption\nType\nDefault\nDescription\n\n\n\n\ntemperature\nfloat\nnull\nSampling temperature\n\n\ntop_p\nfloat\nnull\nNucleus sampling probability\n\n\ntop_k\nint\nnull\nTop-k sampling\n\n\nmin_p\nfloat\nnull\nMinimum probability threshold\n\n\nrepetition_penalty\nfloat\nnull\nPenalty for repeated tokens\n\n\ngeneration_kwargs\ndict\nnull\nAdditional vLLM SamplingParams (e.g., stop_token_ids)\n\n\nchat_template_kwargs\ndict\nnull\nChat template kwargs (e.g., {enable_thinking: false})\n\n\nvllm_guided_decoding_regex\nstr\nnull\nRegex constraint for guided decoding\n\n\n\n\n\nAsync pipeline\n\n\n\n\n\n\n\n\n\nOption\nType\nDefault\nDescription\n\n\n\n\nuse_data_producer\nbool\nfalse\nEnable data producer protocol (required for async features)\n\n\nasync_prefetch\nbool\nfalse\nGenerate next batch in background thread\n\n\nprefetch_depth\nint\nnull\nNumber of batches to prefetch ahead\n\n\nvllm_sync_interval\nint\nnull\nSync LoRA weights to vLLM every N steps\n\n\nvllm_lora_sync\nbool\nfalse\nUse filesystem LoRA sync instead of NCCL merge\n\n\nstreaming_partial_batch\nbool\nnull\nScore prompt groups incrementally\n\n\nstreaming_min_groups\nint\nnull\nMinimum groups per streaming chunk\n\n\nskip_zero_advantage_batches\nbool\ntrue\nSkip micro-batches with zero learning signal\n\n\nreward_num_workers\nint\n1\nSubprocess workers for reward computation\n\n\nvllm_enable_sleep_mode\nbool\nnull\nOffload vLLM weights when idle (colocate mode)\n\n\n\n\n\nImportance sampling\n\n\n\n\n\n\n\n\n\nOption\nType\nDefault\nDescription\n\n\n\n\nvllm_importance_sampling_correction\nbool\nnull\nEnable IS correction for async distribution shift\n\n\nimportance_sampling_level\n\"token\" or \"sequence\"\nnull\nGranularity of IS ratios. Use token with Liger\n\n\nvllm_importance_sampling_mode\nstr\nnull\ntoken_mask, token_truncate, sequence_mask, or sequence_truncate\n\n\nvllm_importance_sampling_cap\nfloat\nnull\nCap C for IS ratio clipping/masking\n\n\noff_policy_mask_threshold\nfloat\nnull\nKL threshold for off-policy sequence masking (OPSM)\n\n\nuse_bias_correction_kl\nbool\nnull\nApply IS correction to KL divergence term\n\n\n\n\n\nReplay and re-roll\n\n\n\n\n\n\n\n\n\nOption\nType\nDefault\nDescription\n\n\n\n\nreplay_buffer_size\nint\n0\nMax cached high-signal groups. 0 = disabled\n\n\nreplay_recompute_logps\nbool\ntrue\nRecompute log-probs for replayed data with current model\n\n\nreroll_start_fraction\nfloat\n1.0\nStart re-rolling failed prompts after this fraction of training. 1.0 = disabled\n\n\nreroll_max_groups\nint\n1\nMax prompt groups to replace with re-rolls per batch\n\n\n\n\n\nReference model\n\n\n\n\n\n\n\n\n\nOption\nType\nDefault\nDescription\n\n\n\n\nsync_ref_model\nbool\nfalse\nPeriodically sync reference model with training model\n\n\nref_model_mixup_alpha\nfloat\n0.9\nEMA coefficient for reference model sync\n\n\nref_model_sync_steps\nint\n64\nSync reference model every N steps\n\n\n\n\n\nLogging\n\n\n\n\n\n\n\n\n\nOption\nType\nDefault\nDescription\n\n\n\n\nlog_completions\nbool\nfalse\nLog sample completions to W&B\n\n\nnum_completions_to_print\nint\nnull\nNumber of completions to print per step\n\n\nuse_liger_loss\nbool\nnull\nUse Liger fused kernel for GRPO loss (reduces VRAM)", + "crumbs": [ + "How To Guides", + "GRPO Training" + ] + }, + { + "objectID": "docs/choosing_method.html", + "href": "docs/choosing_method.html", + "title": "Which Fine-Tuning Method Should I Use?", + "section": "", + "text": "Axolotl supports four broad categories of fine-tuning, each suited to different data types, objectives, and resource constraints.\n\n\n\n\n\n\n\n\nMethod\nWhat It Does\nData You Need\n\n\n\n\nSupervised Fine-Tuning (SFT)\nTeaches the model to produce specific outputs given inputs\nInput-output pairs (instructions, conversations, completions)\n\n\nPreference Learning (DPO/KTO/ORPO)\nSteers the model toward preferred outputs and away from dispreferred ones\nChosen/rejected response pairs (DPO, ORPO) or binary labels (KTO)\n\n\nReinforcement Learning (GRPO)\nOptimizes the model against a reward signal through online generation\nA reward function (code or model-based) and a prompt dataset\n\n\nReward Modeling\nTrains a model to score responses, for use as a reward signal in RL\nPreference pairs ranked by quality\n\n\n\nEach method is configured through a YAML file with rl: <method> (or omitted for SFT). All methods support LoRA, QLoRA, and full fine-tuning unless otherwise noted.", + "crumbs": [ + "Getting Started", + "Which Fine-Tuning Method Should I Use?" + ] + }, + { + "objectID": "docs/choosing_method.html#sec-overview", + "href": "docs/choosing_method.html#sec-overview", + "title": "Which Fine-Tuning Method Should I Use?", + "section": "", + "text": "Axolotl supports four broad categories of fine-tuning, each suited to different data types, objectives, and resource constraints.\n\n\n\n\n\n\n\n\nMethod\nWhat It Does\nData You Need\n\n\n\n\nSupervised Fine-Tuning (SFT)\nTeaches the model to produce specific outputs given inputs\nInput-output pairs (instructions, conversations, completions)\n\n\nPreference Learning (DPO/KTO/ORPO)\nSteers the model toward preferred outputs and away from dispreferred ones\nChosen/rejected response pairs (DPO, ORPO) or binary labels (KTO)\n\n\nReinforcement Learning (GRPO)\nOptimizes the model against a reward signal through online generation\nA reward function (code or model-based) and a prompt dataset\n\n\nReward Modeling\nTrains a model to score responses, for use as a reward signal in RL\nPreference pairs ranked by quality\n\n\n\nEach method is configured through a YAML file with rl: <method> (or omitted for SFT). All methods support LoRA, QLoRA, and full fine-tuning unless otherwise noted.", + "crumbs": [ + "Getting Started", + "Which Fine-Tuning Method Should I Use?" + ] + }, + { + "objectID": "docs/choosing_method.html#sec-decision-tree", + "href": "docs/choosing_method.html#sec-decision-tree", + "title": "Which Fine-Tuning Method Should I Use?", + "section": "2 Decision Tree", + "text": "2 Decision Tree\nUse the following flowchart to choose your method. Start at the top and follow the path that matches your situation.\nDo you have a reward function (code-based or model-based)?\n├── YES\n│ └── Use GRPO (rl: grpo)\n│ The model generates its own completions and learns from reward scores.\n│ Best for: math, code, reasoning, tasks with verifiable answers.\n│ See: rlhf.qmd#grpo\n│\n└── NO\n │\n Do you have preference pairs (chosen vs. rejected responses)?\n ├── YES\n │ │\n │ Are they paired (same prompt, one chosen, one rejected)?\n │ ├── YES → Use DPO (rl: dpo)\n │ │ Direct optimization without a separate reward model.\n │ │ See: rlhf.qmd#dpo\n │ │\n │ └── NO (only binary good/bad labels)\n │ └── Use KTO (rl: kto)\n │ Works with unpaired preference data.\n │ See: rlhf.qmd#kto\n │\n └── NO\n │\n Do you have input-output examples?\n ├── YES → Use SFT\n │ The simplest and most common method.\n │ See: getting-started.qmd\n │\n └── NO\n └── You need to create training data first.\n Consider generating preference pairs with an LLM judge,\n or writing a reward function for GRPO.\n\n\n\n\n\n\nTip\n\n\n\nWhen in doubt, start with SFT. It is the most straightforward method and works well for most tasks. You can always move to preference learning or RL later to further refine behavior.\n\n\n\n2.1 Method Comparison at a Glance\n\n\n\n\n\n\n\n\n\n\nCriterion\nSFT\nDPO\nKTO\nGRPO\n\n\n\n\nData complexity\nLow (input-output pairs)\nMedium (preference pairs)\nMedium (binary labels)\nLow (prompts + reward code)\n\n\nCompute cost\nLow\nMedium\nMedium\nHigh (requires vLLM server)\n\n\nLearning signal\nSupervised\nContrastive\nContrastive\nOnline reward\n\n\nOnline generation\nNo\nNo\nNo\nYes\n\n\nReward model needed\nNo\nNo\nNo\nNo (uses reward functions)\n\n\nBest for\nTask adaptation, instruction following\nSafety, style alignment\nUnpaired preference data\nReasoning, math, code\n\n\n\n\n\n\n\n\n\nNote\n\n\n\nORPO is an alternative to DPO that combines SFT and preference optimization in a single training stage, removing the need for a separate SFT step. Configure with rl: orpo. See rlhf.qmd for details.", + "crumbs": [ + "Getting Started", + "Which Fine-Tuning Method Should I Use?" + ] + }, + { + "objectID": "docs/choosing_method.html#sec-adapter-selection", + "href": "docs/choosing_method.html#sec-adapter-selection", + "title": "Which Fine-Tuning Method Should I Use?", + "section": "3 Adapter Selection", + "text": "3 Adapter Selection\nOnce you have chosen a method, decide how to apply the parameter updates. The three main options trade off VRAM usage against model quality.\n\n3.1 QLoRA\n\nHow it works: The base model is loaded in 4-bit (NF4) quantization. Small low-rank adapter matrices are trained in higher precision on top.\nVRAM savings: Roughly 4x reduction in model memory compared to full fine-tuning.\nQuality: Slight degradation due to quantization noise, but often negligible for task-specific fine-tuning.\nWhen to use: When your GPU cannot fit the model in full precision, or when you want fast experimentation.\n\nadapter: qlora\nload_in_4bit: true\nlora_r: 32\nlora_alpha: 64\nlora_target_linear: true\n\n\n3.2 LoRA\n\nHow it works: The base model is loaded at full precision (or 8-bit). Low-rank adapter matrices are trained alongside.\nVRAM savings: Roughly 2-3x reduction compared to full fine-tuning (model weights are frozen, only adapters + optimizer states for adapters are stored).\nQuality: Very close to full fine-tuning for most tasks, especially with higher rank values.\nWhen to use: When you have enough VRAM for the base model but not for full optimizer states.\n\nadapter: lora\nlora_r: 32\nlora_alpha: 64\nlora_target_linear: true\n\n\n\n\n\n\nTip\n\n\n\nFor GRPO training, LoRA is strongly recommended. The vLLM server needs to sync weights from the trainer, and LoRA sync (trl.vllm_lora_sync: true) is far more efficient than syncing full merged weights. See vLLM Serving for details.\n\n\n\n\n3.3 Full Fine-Tuning\n\nHow it works: All model parameters are updated during training. No adapters.\nVRAM savings: None. Requires memory for model weights, gradients, and optimizer states (roughly 4x model size in bf16 with AdamW).\nQuality: Highest potential quality, especially for large distribution shifts.\nWhen to use: When you have ample GPU memory or multi-GPU setups, and need maximum performance. Also required for pre-training.\n\n# No adapter or load_in_* lines needed\nmicro_batch_size: 1\ngradient_accumulation_steps: 16\n\n\n3.4 Quick Comparison\n\n\n\n\n\n\n\n\n\n\nQLoRA\nLoRA\nFull\n\n\n\n\nTrainable params\n~0.1-1%\n~0.1-1%\n100%\n\n\nModel memory\n~25% of full\n~50-100% of full\n100%\n\n\nOptimizer memory\nTiny (adapters only)\nTiny (adapters only)\n2x model size (AdamW)\n\n\nTraining speed\nSlower (dequantization overhead)\nBaseline\nFaster per-step (no adapter overhead)\n\n\nInference\nMerge or serve with adapter\nMerge or serve with adapter\nDirect\n\n\nMulti-GPU required?\nRarely\nFor 13B+ models\nFor 7B+ models", + "crumbs": [ + "Getting Started", + "Which Fine-Tuning Method Should I Use?" + ] + }, + { + "objectID": "docs/choosing_method.html#sec-hardware-mapping", + "href": "docs/choosing_method.html#sec-hardware-mapping", + "title": "Which Fine-Tuning Method Should I Use?", + "section": "4 Hardware Mapping", + "text": "4 Hardware Mapping\nThe tables below provide approximate GPU memory requirements. Actual usage depends on context length, batch size, and optimizer choice.\n\n4.1 SFT / Preference Learning\n\n\n\nModel Size\nQLoRA (4-bit)\nLoRA (bf16)\nFull (bf16 + AdamW)\n\n\n\n\n1-3B\n6-8 GB\n8-12 GB\n24-32 GB\n\n\n7-8B\n10-14 GB\n16-24 GB\n60-80 GB\n\n\n13-14B\n16-20 GB\n28-40 GB\n120+ GB\n\n\n30-34B\n24-32 GB\n64-80 GB\n2-4x 80 GB\n\n\n70-72B\n40-48 GB\n2x 80 GB\n4-8x 80 GB\n\n\n\n\n\n\n\n\n\nImportant\n\n\n\nThese estimates assume a short context length (512-2048 tokens) and micro_batch_size of 1-2. Longer sequences and larger batches increase memory significantly due to activations. Use gradient checkpointing to reduce activation memory at the cost of ~30% slower training.\n\n\n\n\n4.2 GRPO (RL Training)\nGRPO requires additional GPU(s) for the vLLM generation server. Plan for at least two GPUs: one for training, one for vLLM.\n\n\n\n\n\n\n\n\n\nModel Size\nTraining GPU (LoRA, bf16)\nvLLM GPU\nTotal GPUs\n\n\n\n\n0.5-3B\n1x 24 GB\n1x 24 GB\n2x 24 GB\n\n\n7-8B\n1x 80 GB\n1x 80 GB\n2x 80 GB\n\n\n13-14B\n1-2x 80 GB\n1-2x 80 GB\n2-4x 80 GB\n\n\n30-72B\n2-4x 80 GB (FSDP/DeepSpeed)\n2-4x 80 GB (tensor parallel)\n4-8x 80 GB\n\n\n\n\n\n\n\n\n\nTip\n\n\n\nFor single-GPU GRPO, use vllm_mode: colocate with vllm_enable_sleep_mode: true. The vLLM engine shares the GPU and offloads VRAM when not generating. This works for smaller models (up to ~3B on a 24 GB GPU) but is slower than the two-GPU server mode.\n\n\n\n\n4.3 Multi-GPU Threshold\nYou need multi-GPU training when:\n\nFull fine-tuning of models 7B+ (use FSDP or DeepSpeed ZeRO)\nLoRA of models 30B+ (or 13B+ with long contexts)\nGRPO almost always (separate vLLM server), unless using colocate mode\n\nSee Multi-GPU Training for FSDP and DeepSpeed configuration.", + "crumbs": [ + "Getting Started", + "Which Fine-Tuning Method Should I Use?" + ] + }, + { + "objectID": "docs/choosing_method.html#sec-quick-links", + "href": "docs/choosing_method.html#sec-quick-links", + "title": "Which Fine-Tuning Method Should I Use?", + "section": "5 Quick Links", + "text": "5 Quick Links\n\n\n\nMethod\nConfig Key\nDocumentation\nExample Config\n\n\n\n\nSFT\n(default, no rl: key)\nGetting Started\nexamples/llama-3/lora-1b.yml\n\n\nDPO\nrl: dpo\nRLHF - DPO\nSee rlhf.qmd\n\n\nKTO\nrl: kto\nRLHF - KTO\nSee rlhf.qmd\n\n\nORPO\nrl: orpo\nRLHF - ORPO\nSee rlhf.qmd\n\n\nGRPO\nrl: grpo\nRLHF - GRPO, vLLM Serving\nSee rlhf.qmd\n\n\nReward Modeling\nrl: reward_trainer\nReward Modelling\nSee reward_modelling.qmd\n\n\n\n\n5.1 Related Guides\n\nConfiguration Reference – Full list of all config options\nDataset Formats – How to structure your training data\nOptimizations – Flash attention, gradient checkpointing, mixed precision\nMulti-GPU Training – FSDP and DeepSpeed setup\nvLLM Serving – Setting up vLLM for GRPO training", + "crumbs": [ + "Getting Started", + "Which Fine-Tuning Method Should I Use?" ] }, { @@ -4658,46 +5209,246 @@ ] }, { - "objectID": "docs/checkpoint_saving.html", - "href": "docs/checkpoint_saving.html", - "title": "Checkpoint Saving", + "objectID": "docs/agents/preference_tuning.html", + "href": "docs/agents/preference_tuning.html", + "title": "Preference Learning (RLHF) — Agent Reference", "section": "", - "text": "Axolotl supports on-demand checkpoint saving during training. You can trigger checkpoints via file-based triggers (for programmatic control) or Control+C (for interactive use)." + "text": "Reference for DPO, IPO, KTO, ORPO, and SimPO. For config templates and dataset format examples, see rlhf.qmd. For GRPO, see grpo.qmd. For EBFT, see ebft.qmd.\n\n\n\n\n\n\n\n\n\n\n\nMethod\nData Requirement\nKey Idea\nBest For\n\n\n\n\nDPO\nPaired (chosen + rejected)\nImplicit reward via preference pairs\nGeneral alignment, most common\n\n\nIPO\nPaired (chosen + rejected)\nDPO with different loss (avoids overfitting)\nWhen DPO overfits\n\n\nKTO\nUnpaired (completion + binary label)\nKahneman-Tversky loss, no pairs needed\nWhen you only have thumbs-up/down\n\n\nORPO\nPaired (chosen + rejected)\nCombined SFT + preference, no ref model\nSingle-stage alignment, saves VRAM\n\n\nSimPO\nPaired (chosen + rejected)\nLength-normalized, no ref model\nSimple setup, length-robust\n\n\n\nDefault: start with DPO. All methods require sample_packing: false.\n\n\n\n┌──────────────┐ ┌───────────────┐ ┌───────────────┐\n│ Policy Model │ │ Reference │ │ Preference │\n│ (trainable) │ │ Model (frozen)│ │ Dataset │\n└──────┬───────┘ └──────┬────────┘ └──────┬────────┘\n └──────────┬───────┘ │\n v │\n Forward pass on chosen + rejected <─────┘\n │\n Preference Loss (DPO/IPO/KTO/...)\n │\n Backprop + Update\n\nException: ORPO and SimPO do NOT use a reference model (~50% less VRAM).\nNo vLLM server needed (unlike GRPO). Offline RL with pre-collected preference data.\n\n\n\n\nPaired preference data (chosen + rejected)?\n\nDefault → rl: dpo\nOverfitting → rl: ipo\nVRAM-limited → rl: orpo (no ref model)\nLength-sensitive → rl: simpo (no ref model)\n\nOnly binary labels (good/bad)? → rl: kto\nSingle-stage training (no separate SFT)? → rl: orpo\n\n\n\n\n\n\n\n\n\n\n\n\n\nDPO\nIPO\nKTO\nORPO\nSimPO\n\n\n\n\nReference model\nYes\nYes\nYes\nNo\nNo\n\n\nVRAM overhead\n~2x model\n~2x model\n~2x model\n~1x model\n~1x model\n\n\nTRL trainer class\nDPOTrainer\nDPOTrainer\nKTOTrainer\nORPOTrainer\nCPOTrainer\n\n\n\n\n\n\nThe type field resolves to a Python function:\ntype: \"chatml.intel\"\n → axolotl.prompt_strategies.dpo.chatml.intel(cfg, **kwargs)\n → returns transform_fn(sample) → {\"prompt\", \"chosen\", \"rejected\"}\n\ntype: \"chat_template.default\"\n → axolotl.prompt_strategies.dpo.chat_template.default(cfg, dataset_idx, **kwargs)\n\ntype: {\"field_prompt\": \"prompt\", ...} (dict)\n → axolotl.prompt_strategies.dpo.user_defined.default(...)\nModule base: axolotl.prompt_strategies.{rl_method} — replace dpo with kto or orpo.\n\n\n\n\n\n\n\n\n\n\n\nMetric\nHealthy Range\nProblem\n\n\n\n\ntrain/loss\nDecreasing, 0.3-0.7\nFlat or increasing = broken data or too high LR\n\n\nrewards/chosen\nIncreasing\nFlat = model not learning preferences\n\n\nrewards/rejected\nDecreasing\nIncreasing = model prefers wrong responses\n\n\nrewards/margins\nPositive and increasing\nNegative = prefers rejected over chosen\n\n\nrewards/accuracies\n> 0.5, toward 0.7+\n< 0.5 = worse than random\n\n\nlogps/rejected\nDecreasing\nIncreasing = reward hacking\n\n\ngrad_norm\n0.01 - 10.0\n> 100 = exploding gradients\n\n\n\nMethod-specific: DPO/IPO watch rewards/margins; KTO loss is noisier; ORPO monitor SFT + odds ratio components; SimPO check length-normalized reward separation.\n\n\n\n\n\n\n\n\n\n\nIssue\nFix\n\n\n\n\nSample packing crash\nSet sample_packing: false (required for all preference methods)\n\n\nKTO KeyError: 'label'\nEnsure dataset has boolean label column\n\n\nORPO/KTO KeyError during tokenization\nAdd remove_unused_columns: false\n\n\nORPO template not applied\nORPO requires explicit chat_template setting\n\n\nOOM with ref model (DPO/IPO/KTO)\nUse LoRA/QLoRA, or switch to ORPO/SimPO (no ref model)\n\n\nIPO + label_smoothing\nDo not set dpo_label_smoothing when rl: ipo\n\n\n\nFull troubleshooting: training_stability.qmd\n\n\n\nsrc/axolotl/\n core/trainers/dpo/ # DPO trainer, args, strategy\n core/builders/rl.py # HFRLTrainerBuilder — routes rl type → trainer class\n core/training_args.py # AxolotlKTOConfig, AxolotlORPOConfig, AxolotlCPOConfig\n prompt_strategies/\n dpo/ # DPO/IPO/SimPO dataset strategies\n chat_template.py # chat_template.default, chat_template.argilla_chat\n chatml.py # chatml.default/intel/icr/argilla_chat/prompt_pairs/ultra\n llama3.py # llama3 variants (same subtypes as chatml)\n user_defined.py # Custom field mapping\n passthrough.py # No transform\n kto/ # KTO dataset strategies (chatml, llama3, user_defined)\n orpo/ # ORPO dataset strategies (chat_template.argilla)\n utils/schemas/enums.py # RLType enum (dpo, ipo, kto, orpo, simpo, grpo, gdpo, ebft)\n utils/schemas/config.py # All rl/dpo/kto/orpo/simpo config fields\n\ndocs/rlhf.qmd # Full user docs: all dataset formats, config templates\ndocs/choosing_method.qmd # SFT vs DPO vs GRPO decision guide\nexamples/qwen2/dpo.yaml # DPO example\nexamples/llama-3/qlora-1b-kto.yaml # KTO example" }, { - "objectID": "docs/checkpoint_saving.html#overview", - "href": "docs/checkpoint_saving.html#overview", - "title": "Checkpoint Saving", + "objectID": "docs/agents/preference_tuning.html#method-overview", + "href": "docs/agents/preference_tuning.html#method-overview", + "title": "Preference Learning (RLHF) — Agent Reference", "section": "", - "text": "Axolotl supports on-demand checkpoint saving during training. You can trigger checkpoints via file-based triggers (for programmatic control) or Control+C (for interactive use)." + "text": "Method\nData Requirement\nKey Idea\nBest For\n\n\n\n\nDPO\nPaired (chosen + rejected)\nImplicit reward via preference pairs\nGeneral alignment, most common\n\n\nIPO\nPaired (chosen + rejected)\nDPO with different loss (avoids overfitting)\nWhen DPO overfits\n\n\nKTO\nUnpaired (completion + binary label)\nKahneman-Tversky loss, no pairs needed\nWhen you only have thumbs-up/down\n\n\nORPO\nPaired (chosen + rejected)\nCombined SFT + preference, no ref model\nSingle-stage alignment, saves VRAM\n\n\nSimPO\nPaired (chosen + rejected)\nLength-normalized, no ref model\nSimple setup, length-robust\n\n\n\nDefault: start with DPO. All methods require sample_packing: false." }, { - "objectID": "docs/checkpoint_saving.html#file-based-checkpoint-trigger", - "href": "docs/checkpoint_saving.html#file-based-checkpoint-trigger", - "title": "Checkpoint Saving", - "section": "2 File-Based Checkpoint Trigger", - "text": "2 File-Based Checkpoint Trigger\n\n2.1 Configuration\nEnable in your config:\ndynamic_checkpoint:\n enabled: true\n check_interval: 100 # Optional: check every N steps (default: 100)\n trigger_file_path: \"axolotl_checkpoint.save\" # Optional: custom filename\nOptions:\n- enabled: true to enable (required)\n- check_interval: Steps between file checks. Default: 100. Lower = faster response, higher I/O overhead.\n- trigger_file_path: Custom trigger filename. Default: axolotl_checkpoint.save\n\n\n2.2 How It Works\n\nRank 0 checks for trigger file every check_interval steps in output_dir\nWhen detected, file is deleted and checkpoint is saved\nIn distributed training, rank 0 broadcasts to synchronize all ranks\n\n\n\n2.3 Usage\nCommand line:\ntouch /path/to/output_dir/axolotl_checkpoint.save\nProgrammatic:\nfrom pathlib import Path\nPath(\"/path/to/output_dir/axolotl_checkpoint.save\").touch()\nCheckpoint saves within the next check_interval steps. The trigger file is auto-deleted after detection, so you can create it multiple times.\nCustom filename:\ndynamic_checkpoint:\n enabled: true\n trigger_file_path: \"my_trigger.save\"\ntouch /path/to/output_dir/my_trigger.save" + "objectID": "docs/agents/preference_tuning.html#architecture", + "href": "docs/agents/preference_tuning.html#architecture", + "title": "Preference Learning (RLHF) — Agent Reference", + "section": "", + "text": "┌──────────────┐ ┌───────────────┐ ┌───────────────┐\n│ Policy Model │ │ Reference │ │ Preference │\n│ (trainable) │ │ Model (frozen)│ │ Dataset │\n└──────┬───────┘ └──────┬────────┘ └──────┬────────┘\n └──────────┬───────┘ │\n v │\n Forward pass on chosen + rejected <─────┘\n │\n Preference Loss (DPO/IPO/KTO/...)\n │\n Backprop + Update\n\nException: ORPO and SimPO do NOT use a reference model (~50% less VRAM).\nNo vLLM server needed (unlike GRPO). Offline RL with pre-collected preference data." }, { - "objectID": "docs/checkpoint_saving.html#controlc-sigint-checkpoint", - "href": "docs/checkpoint_saving.html#controlc-sigint-checkpoint", - "title": "Checkpoint Saving", - "section": "3 Control+C (SIGINT) Checkpoint", - "text": "3 Control+C (SIGINT) Checkpoint\nPressing Ctrl+C during training saves the model state and exits gracefully. Note: This saves only the model weights, not optimizer state. For resumable checkpoints, use the file-based trigger." + "objectID": "docs/agents/preference_tuning.html#method-selection", + "href": "docs/agents/preference_tuning.html#method-selection", + "title": "Preference Learning (RLHF) — Agent Reference", + "section": "", + "text": "Paired preference data (chosen + rejected)?\n\nDefault → rl: dpo\nOverfitting → rl: ipo\nVRAM-limited → rl: orpo (no ref model)\nLength-sensitive → rl: simpo (no ref model)\n\nOnly binary labels (good/bad)? → rl: kto\nSingle-stage training (no separate SFT)? → rl: orpo\n\n\n\n\n\n\n\n\n\n\n\n\n\nDPO\nIPO\nKTO\nORPO\nSimPO\n\n\n\n\nReference model\nYes\nYes\nYes\nNo\nNo\n\n\nVRAM overhead\n~2x model\n~2x model\n~2x model\n~1x model\n~1x model\n\n\nTRL trainer class\nDPOTrainer\nDPOTrainer\nKTOTrainer\nORPOTrainer\nCPOTrainer" }, { - "objectID": "docs/checkpoint_saving.html#best-practices", - "href": "docs/checkpoint_saving.html#best-practices", - "title": "Checkpoint Saving", - "section": "4 Best Practices", - "text": "4 Best Practices\n\nCheck interval: Lower values (10-50) for fast training, default 100 for slower training\nDistributed training: Create trigger file once; rank 0 handles synchronization\nResume: Dynamic checkpoints can be resumed like regular checkpoints via resume_from_checkpoint" + "objectID": "docs/agents/preference_tuning.html#prompt-strategy-resolution", + "href": "docs/agents/preference_tuning.html#prompt-strategy-resolution", + "title": "Preference Learning (RLHF) — Agent Reference", + "section": "", + "text": "The type field resolves to a Python function:\ntype: \"chatml.intel\"\n → axolotl.prompt_strategies.dpo.chatml.intel(cfg, **kwargs)\n → returns transform_fn(sample) → {\"prompt\", \"chosen\", \"rejected\"}\n\ntype: \"chat_template.default\"\n → axolotl.prompt_strategies.dpo.chat_template.default(cfg, dataset_idx, **kwargs)\n\ntype: {\"field_prompt\": \"prompt\", ...} (dict)\n → axolotl.prompt_strategies.dpo.user_defined.default(...)\nModule base: axolotl.prompt_strategies.{rl_method} — replace dpo with kto or orpo." }, { - "objectID": "docs/checkpoint_saving.html#example", - "href": "docs/checkpoint_saving.html#example", - "title": "Checkpoint Saving", - "section": "5 Example", - "text": "5 Example\noutput_dir: ./outputs/lora-out\nsave_steps: 500 # Scheduled checkpoints\n\ndynamic_checkpoint:\n enabled: true\n check_interval: 50\nThis enables scheduled checkpoints every 500 steps plus on-demand saves via file trigger (checked every 50 steps)." + "objectID": "docs/agents/preference_tuning.html#healthy-training-indicators", + "href": "docs/agents/preference_tuning.html#healthy-training-indicators", + "title": "Preference Learning (RLHF) — Agent Reference", + "section": "", + "text": "Metric\nHealthy Range\nProblem\n\n\n\n\ntrain/loss\nDecreasing, 0.3-0.7\nFlat or increasing = broken data or too high LR\n\n\nrewards/chosen\nIncreasing\nFlat = model not learning preferences\n\n\nrewards/rejected\nDecreasing\nIncreasing = model prefers wrong responses\n\n\nrewards/margins\nPositive and increasing\nNegative = prefers rejected over chosen\n\n\nrewards/accuracies\n> 0.5, toward 0.7+\n< 0.5 = worse than random\n\n\nlogps/rejected\nDecreasing\nIncreasing = reward hacking\n\n\ngrad_norm\n0.01 - 10.0\n> 100 = exploding gradients\n\n\n\nMethod-specific: DPO/IPO watch rewards/margins; KTO loss is noisier; ORPO monitor SFT + odds ratio components; SimPO check length-normalized reward separation." + }, + { + "objectID": "docs/agents/preference_tuning.html#known-issues", + "href": "docs/agents/preference_tuning.html#known-issues", + "title": "Preference Learning (RLHF) — Agent Reference", + "section": "", + "text": "Issue\nFix\n\n\n\n\nSample packing crash\nSet sample_packing: false (required for all preference methods)\n\n\nKTO KeyError: 'label'\nEnsure dataset has boolean label column\n\n\nORPO/KTO KeyError during tokenization\nAdd remove_unused_columns: false\n\n\nORPO template not applied\nORPO requires explicit chat_template setting\n\n\nOOM with ref model (DPO/IPO/KTO)\nUse LoRA/QLoRA, or switch to ORPO/SimPO (no ref model)\n\n\nIPO + label_smoothing\nDo not set dpo_label_smoothing when rl: ipo\n\n\n\nFull troubleshooting: training_stability.qmd" + }, + { + "objectID": "docs/agents/preference_tuning.html#file-map", + "href": "docs/agents/preference_tuning.html#file-map", + "title": "Preference Learning (RLHF) — Agent Reference", + "section": "", + "text": "src/axolotl/\n core/trainers/dpo/ # DPO trainer, args, strategy\n core/builders/rl.py # HFRLTrainerBuilder — routes rl type → trainer class\n core/training_args.py # AxolotlKTOConfig, AxolotlORPOConfig, AxolotlCPOConfig\n prompt_strategies/\n dpo/ # DPO/IPO/SimPO dataset strategies\n chat_template.py # chat_template.default, chat_template.argilla_chat\n chatml.py # chatml.default/intel/icr/argilla_chat/prompt_pairs/ultra\n llama3.py # llama3 variants (same subtypes as chatml)\n user_defined.py # Custom field mapping\n passthrough.py # No transform\n kto/ # KTO dataset strategies (chatml, llama3, user_defined)\n orpo/ # ORPO dataset strategies (chat_template.argilla)\n utils/schemas/enums.py # RLType enum (dpo, ipo, kto, orpo, simpo, grpo, gdpo, ebft)\n utils/schemas/config.py # All rl/dpo/kto/orpo/simpo config fields\n\ndocs/rlhf.qmd # Full user docs: all dataset formats, config templates\ndocs/choosing_method.qmd # SFT vs DPO vs GRPO decision guide\nexamples/qwen2/dpo.yaml # DPO example\nexamples/llama-3/qlora-1b-kto.yaml # KTO example" + }, + { + "objectID": "docs/agents/reward_modelling.html", + "href": "docs/agents/reward_modelling.html", + "title": "Reward Modelling — Agent Reference", + "section": "", + "text": "Train models to score responses for use as reward signals in RL. For full docs, see reward_modelling.qmd.\n\n\n\n\nTrain a classifier to predict preference over entire interactions. Uses AutoModelForSequenceClassification.\nbase_model: google/gemma-2-2b\nmodel_type: AutoModelForSequenceClassification\nnum_labels: 1\nreward_model: true\nchat_template: gemma\ndatasets:\n - path: argilla/distilabel-intel-orca-dpo-pairs\n type: bradley_terry.chat_template\nDataset format: {\"system\": \"...\", \"input\": \"...\", \"chosen\": \"...\", \"rejected\": \"...\"}\n\n\n\nTrain a token classifier to score each reasoning step. Uses AutoModelForTokenClassification.\nbase_model: Qwen/Qwen2.5-3B\nmodel_type: AutoModelForTokenClassification\nnum_labels: 2\nprocess_reward_model: true\ndatasets:\n - path: trl-lib/math_shepherd\n type: stepwise_supervised\nDataset format: see stepwise_supervised.qmd.\n\n\n\n\nsrc/axolotl/\n core/builders/causal.py # Handles reward_model flag in trainer builder\n prompt_strategies/bradley_terry/ # Bradley-Terry prompt strategies\n prompt_strategies/stepwise_supervised.py # PRM dataset strategy\n utils/schemas/config.py # reward_model, process_reward_model config fields" + }, + { + "objectID": "docs/agents/reward_modelling.html#types", + "href": "docs/agents/reward_modelling.html#types", + "title": "Reward Modelling — Agent Reference", + "section": "", + "text": "Train a classifier to predict preference over entire interactions. Uses AutoModelForSequenceClassification.\nbase_model: google/gemma-2-2b\nmodel_type: AutoModelForSequenceClassification\nnum_labels: 1\nreward_model: true\nchat_template: gemma\ndatasets:\n - path: argilla/distilabel-intel-orca-dpo-pairs\n type: bradley_terry.chat_template\nDataset format: {\"system\": \"...\", \"input\": \"...\", \"chosen\": \"...\", \"rejected\": \"...\"}\n\n\n\nTrain a token classifier to score each reasoning step. Uses AutoModelForTokenClassification.\nbase_model: Qwen/Qwen2.5-3B\nmodel_type: AutoModelForTokenClassification\nnum_labels: 2\nprocess_reward_model: true\ndatasets:\n - path: trl-lib/math_shepherd\n type: stepwise_supervised\nDataset format: see stepwise_supervised.qmd." + }, + { + "objectID": "docs/agents/reward_modelling.html#file-map", + "href": "docs/agents/reward_modelling.html#file-map", + "title": "Reward Modelling — Agent Reference", + "section": "", + "text": "src/axolotl/\n core/builders/causal.py # Handles reward_model flag in trainer builder\n prompt_strategies/bradley_terry/ # Bradley-Terry prompt strategies\n prompt_strategies/stepwise_supervised.py # PRM dataset strategy\n utils/schemas/config.py # reward_model, process_reward_model config fields" + }, + { + "objectID": "docs/optimizations.html", + "href": "docs/optimizations.html", + "title": "Optimizations Guide", + "section": "", + "text": "Axolotl includes numerous optimizations to speed up training, reduce memory usage, and handle large models.\nThis guide provides a high-level overview and directs you to the detailed documentation for each feature.", + "crumbs": [ + "How To Guides", + "Optimizations Guide" + ] + }, + { + "objectID": "docs/optimizations.html#speed-optimizations", + "href": "docs/optimizations.html#speed-optimizations", + "title": "Optimizations Guide", + "section": "Speed Optimizations", + "text": "Speed Optimizations\nThese optimizations focus on increasing training throughput and reducing total training time.\n\nSample Packing\nImproves GPU utilization by combining multiple short sequences into a single packed sequence for training. This requires enabling one of the attention implementations below.\n\nConfig: sample_packing: true\nLearn more: Sample Packing\n\n\n\nAttention Implementations\nUsing an optimized attention implementation is critical for training speed.\n\nFlash Attention 2: flash_attention: true. (Recommended) The industry standard for fast attention on modern GPUs. Requires Ampere or higher. For AMD, check AMD Support.\nFlex Attention: flex_attention: true.\nSDP Attention: sdp_attention: true. PyTorch’s native implementation.\nXformers: xformers_attention: true. Works with FP16.\n\nNote: You should only enable one attention backend.\n\n\nLoRA Optimizations\nLeverages optimized kernels to accelerate LoRA training and reduce memory usage.\n\nLearn more: LoRA Optimizations Documentation", + "crumbs": [ + "How To Guides", + "Optimizations Guide" + ] + }, + { + "objectID": "docs/optimizations.html#memory-optimizations", + "href": "docs/optimizations.html#memory-optimizations", + "title": "Optimizations Guide", + "section": "Memory Optimizations", + "text": "Memory Optimizations\nThese techniques help you fit larger models or use bigger batch sizes on your existing hardware.\n\nParameter Efficient Finetuning (LoRA & QLoRA)\nDrastically reduces memory by training a small set of “adapter” parameters instead of the full model. This is the most common and effective memory-saving technique.\n\nExamples: Find configs with lora or qlora in the examples directory.\nConfig Reference: See adapter, load_in_4bit, and load_in_8bit in the Configuration Reference.\n\n\n\nGradient Checkpointing & Activation Offloading\nThese techniques save VRAM by changing how activations are handled.\n\nGradient Checkpointing: re-computes activations during the backward pass, trading compute time for VRAM.\nActivation Offloading: moves activations to CPU RAM or disk, trading I/O overhead for VRAM.\nLearn more: Gradient Checkpointing and Offloading Docs\n\n\n\nLayer Offloading\nOffloads frozen (non-trainable) decoder layer parameters to CPU and streams them back to GPU one layer at a time during forward/backward passes using CUDA stream prefetching. Especially effective for LoRA/QLoRA where most parameters are frozen.\n\nConfig: layer_offloading: true\nLearn more: Layer Offloading Docs\n\n\n\nCut Cross Entropy (CCE)\nReduces VRAM usage by using an optimized cross-entropy loss calculation.\n\nLearn more: Custom Integrations - CCE\n\n\n\nLiger Kernels\nProvides efficient Triton kernels to improve training speed and reduce memory usage.\n\nLearn more: Custom Integrations - Liger Kernels\n\n\n\nExpert Kernels\nOptimized kernel implementations for Mixture of Experts (MoE) model training.\n\nScatterMoE: Triton-based MoE kernels with fused LoRA support.\nSonicMoE: CUTLASS-based MoE kernels for NVIDIA Hopper and Blackwell GPUs.\nLearn more: Custom Integrations - Kernels Integration", + "crumbs": [ + "How To Guides", + "Optimizations Guide" + ] + }, + { + "objectID": "docs/optimizations.html#long-context-models", + "href": "docs/optimizations.html#long-context-models", + "title": "Optimizations Guide", + "section": "Long Context Models", + "text": "Long Context Models\nTechniques to train models on sequences longer than their original context window.\n\nRoPE Scaling\nExtends a model’s context window by interpolating its Rotary Position Embeddings.\n\nConfig: Pass the rope_scaling config under the overrides_of_model_config:. To learn how to set RoPE, check the respective model config.\n\n\n\nSequence Parallelism\nSplits long sequences across multiple GPUs, enabling training with sequence lengths that would not fit on a single device.\n\nLearn more: Sequence Parallelism Documentation\n\n\n\nArtic Long Sequence Training (ALST)\nALST is a recipe that combines several techniques to train long-context models efficiently. It typically involves:\n\nTiledMLP to reduce memory usage in MLP layers.\nTiled Loss functions (like CCE.\nActivation Offloading to CPU.\nExample: ALST Example Configuration", + "crumbs": [ + "How To Guides", + "Optimizations Guide" + ] + }, + { + "objectID": "docs/optimizations.html#large-models-distributed-training", + "href": "docs/optimizations.html#large-models-distributed-training", + "title": "Optimizations Guide", + "section": "Large Models (Distributed Training)", + "text": "Large Models (Distributed Training)\nTo train models that don’t fit on a single GPU, you’ll need to use a distributed training strategy like FSDP or DeepSpeed. These frameworks shard the model weights, gradients, and optimizer states across multiple GPUs and nodes.\n\nLearn more: Multi-GPU Guide\nLearn more: Multi-Node Guide\n\n\nN-D Parallelism (Beta)\nFor advanced scaling, Axolotl allows you to compose different parallelism techniques (e.g., Data, Tensor, Sequence Parallelism). This is a powerful approach to train an extremely large model by overcoming multiple bottlenecks at once.\n\nLearn more: N-D Parallelism Guide", + "crumbs": [ + "How To Guides", + "Optimizations Guide" + ] + }, + { + "objectID": "docs/optimizations.html#quantization", + "href": "docs/optimizations.html#quantization", + "title": "Optimizations Guide", + "section": "Quantization", + "text": "Quantization\nTechniques to reduce the precision of model weights for memory savings.\n\n4-bit Training (QLoRA)\nThe recommended approach for quantization-based training. It loads the base model in 4-bit using bitsandbytes and then trains QLoRA adapters. See Adapter Finetuning for details.\n\n\nFP8 Training\nEnables training with 8-bit floating point precision on supported hardware (e.g., NVIDIA Hopper series GPUs) for significant speed and memory gains.\n\nExample: Llama 3 FP8 FSDP Example\n\n\n\nQuantization Aware Training (QAT)\nSimulates quantization effects during training, helping the model adapt and potentially improving the final accuracy of the quantized model.\n\nLearn more: QAT Documentation\n\n\n\nGPTQ\nAllows you to finetune LoRA adapters on top of a model that has already been quantized using the GPTQ method.\n\nExample: GPTQ LoRA Example\n\n\n\nMoE Expert Quantization\nQuantizes MoE expert weights on load to reduce VRAM when training MoE models with adapters. Required for Transformers v5+ MoE models where experts use fused nn.Parameter tensors.\n\nConfig: quantize_moe_experts: true\nLearn more: MoE Expert Quantization", + "crumbs": [ + "How To Guides", + "Optimizations Guide" + ] + }, + { + "objectID": "docs/training_stability.html", + "href": "docs/training_stability.html", + "title": "Training Stability & Debugging", + "section": "", + "text": "This guide covers practical techniques for monitoring training health, diagnosing instability, and resolving common failures in both supervised fine-tuning (SFT) and reinforcement learning (GRPO/EBFT) workflows.", + "crumbs": [ + "Troubleshooting", + "Training Stability & Debugging" + ] + }, + { + "objectID": "docs/training_stability.html#monitoring-training", + "href": "docs/training_stability.html#monitoring-training", + "title": "Training Stability & Debugging", + "section": "Monitoring Training", + "text": "Monitoring Training\n\nKey Metrics for SFT\nEvery SFT run should be monitored through at least these four metrics:\n\n\n\n\n\n\n\n\nMetric\nWhat It Tells You\nHealthy Range\n\n\n\n\ntrain/loss\nHow well the model fits training data\nDecreasing; typically 0.5–2.0 for chat fine-tuning\n\n\neval/loss\nGeneralization performance\nTracks train loss with small gap; divergence signals overfitting\n\n\ngrad_norm\nGradient magnitude\n0.1–10.0; spikes above 100 indicate instability\n\n\nlearning_rate\nCurrent LR from scheduler\nShould follow expected schedule (warmup then decay)\n\n\n\n\n\n\n\n\n\nTipSet Up Logging Early\n\n\n\nEnable W&B or TensorBoard from the start. Debugging a failed run without metrics is guesswork.\nwandb_project: my-project\nwandb_run_id: # optional, for resuming\nlogging_steps: 1\n\n\n\n\nKey Metrics for RL (GRPO)\nGRPO training logs a richer set of metrics. These are the critical ones:\n\n\n\n\n\n\n\n\nMetric\nHealthy Range\nRed Flag\n\n\n\n\nrewards/<name>/mean\n> 0.15 within 20 steps\nStays at 0 – reward function is broken or task is too hard\n\n\nreward_std\n> 0 on most steps\nAlways 0 – no learning signal (all completions get the same reward)\n\n\nfrac_reward_zero_std\n< 0.8\n1.0 on every step – zero-advantage skip fires constantly, no gradient updates\n\n\ngrad_norm\n0.001–1.0\n0.0 is acceptable occasionally (zero-adv skip); > 10.0 is unstable\n\n\nentropy\n0.05–0.5\n< 0.01 suggests mode collapse; > 1.0 suggests the model is not converging\n\n\nkl\n0.0–0.5\n> 2.0 suggests policy has diverged too far from reference\n\n\nsampling/sampling_logp_difference/mean\n< 0.1\n> 1.0 means policy has diverged far from vLLM server weights\n\n\nsampling/importance_sampling_ratio/min\n> 0.1\nNear 0 indicates stale off-policy data; increase vllm_sync_interval\n\n\nclip_ratio/region_mean\n< 0.1\n> 0.3 means PPO clipping is too aggressive\n\n\ncompletions/mean_length\nTask-dependent\nMonotonically increasing to max length suggests reward hacking\n\n\ncompletions/clipped_ratio\n< 0.3\n> 0.8 means most completions hit max_completion_length – increase it\n\n\n\n\n\n\n\n\n\nNoteEBFT-Specific Metrics\n\n\n\nFor EBFT training, also monitor ebft/alignment (should trend upward, healthy 0.3–0.9), ebft/diversity (healthy 0.01–0.1; > 1.0 indicates mode collapse), and ebft/cfm_loss (should trend downward, < 10).", + "crumbs": [ + "Troubleshooting", + "Training Stability & Debugging" + ] + }, + { + "objectID": "docs/training_stability.html#sft-stability", + "href": "docs/training_stability.html#sft-stability", + "title": "Training Stability & Debugging", + "section": "SFT Stability", + "text": "SFT Stability\n\nLoss Plateau\nSymptom: Loss stops decreasing early in training, well above expected values.\nCauses and fixes:\n\nLearning rate too low: Increase by 2–5x. Typical ranges: full fine-tune 1e-5 to 5e-5, LoRA 1e-4 to 3e-4.\nInsufficient warmup: Set warmup_steps to 5–10% of total steps. Too-aggressive learning at the start can push the model into a flat region.\nData quality: Check that labels are correctly masked. Use axolotl preprocess and inspect tokenized samples to confirm only the target tokens are trainable.\nWeight decay too high: Default 0.01 is usually fine. Values above 0.1 can suppress learning in LoRA.\n\n\n\nLoss Spikes\nSymptom: Loss suddenly jumps by 2–10x then (possibly) recovers.\nCauses and fixes:\n\nBad data samples: A single malformed or extremely long example can cause a spike. Enable sample_packing: false temporarily and check if spikes correlate with specific batches.\nLearning rate too high: Reduce by 2–5x, or increase warmup.\nGradient accumulation mismatch: Effective batch size = micro_batch_size * gradient_accumulation_steps * num_gpus. Very large effective batch sizes amplify gradient noise.\nMixed precision issues: With bf16: true, some operations can lose precision. If spikes are severe, try fp32 for diagnosis.\n\n\n\nOverfitting\nSymptom: Train loss keeps decreasing but eval loss starts increasing.\nFixes:\n\nIncrease val_set_size (e.g., 0.05) and monitor eval/loss.\nReduce num_epochs or max_steps.\nIncrease weight_decay (try 0.01–0.1).\nUse a smaller LoRA rank (lora_r). Typical values: 8–32.\nIncrease dropout: lora_dropout: 0.05.", + "crumbs": [ + "Troubleshooting", + "Training Stability & Debugging" + ] + }, + { + "objectID": "docs/training_stability.html#rlgrpo-stability", + "href": "docs/training_stability.html#rlgrpo-stability", + "title": "Training Stability & Debugging", + "section": "RL/GRPO Stability", + "text": "RL/GRPO Stability\n\nReward Never Increases\nIf rewards/*/mean stays at 0 for more than 20 steps:\n\nTest reward function standalone: Run it outside training with known inputs to verify it returns nonzero values.\ncd experiments && python -c \"import my_rewards; print(my_rewards.accuracy_reward(...))\"\nCheck dataset columns: The reward function receives **kwargs containing dataset columns. Verify the columns it needs (e.g., answer) are not removed by the dataset transform.\nCheck completion content: Enable log_completions: true in the trl: config and inspect logged completions in W&B. If completions are empty or incoherent, the model may be too weak for the task.\nVerify vLLM is serving the right model: Hit the vLLM health endpoint and confirm the model name matches your config.\n\n\n\nEntropy Collapse (Mode Collapse)\nSymptom: entropy drops below 0.01; all completions become nearly identical.\nFixes:\n\nIncrease temperature in generation kwargs (try 0.8–1.0).\nReduce learning rate.\nAdd a KL penalty term (beta parameter in GRPO config).\nCheck that num_generations is sufficient (16+ gives better advantage estimates).\n\n\n\nIS Ratio Divergence\nSymptom: sampling/importance_sampling_ratio/min drops near 0, or sampling/sampling_logp_difference/mean exceeds 1.0.\nThis means the policy has diverged significantly from the weights used by vLLM for generation. The importance sampling correction becomes unreliable.\nFixes:\n\nDecrease vllm_sync_interval (sync weights more often).\nEnable off_policy_mask_threshold (e.g., 0.5) to mask stale off-policy samples.\nUse importance_sampling_level: token for finer-grained correction.\n\n\n\nGradient Norm Instability\nSymptom: grad_norm oscillates wildly or exceeds 10.0 regularly.\nFixes:\n\nEnable gradient clipping: max_grad_norm: 1.0 (default in most configs).\nReduce learning rate.\nIncrease gradient_accumulation_steps to smooth out noisy batches.\nCheck for NaN issues (see next section).", + "crumbs": [ + "Troubleshooting", + "Training Stability & Debugging" + ] + }, + { + "objectID": "docs/training_stability.html#nan-and-inf-handling", + "href": "docs/training_stability.html#nan-and-inf-handling", + "title": "Training Stability & Debugging", + "section": "NaN and Inf Handling", + "text": "NaN and Inf Handling\n\nCommon Causes\n\n\n\n\n\n\n\n\nCause\nWhere It Manifests\nDetection\n\n\n\n\nFP8 zero-scale division\nForward pass logits\ngrad_norm: nan, loss becomes NaN immediately\n\n\nGradient explosion\nBackward pass\ngrad_norm spikes to inf, then loss goes NaN\n\n\nBad data (empty sequences)\nLogprob computation\nNaN in specific batches only\n\n\nNumerical overflow in log-softmax\nLoss computation\nLarge negative logprobs cause exp() overflow\n\n\n\n\n\nFP8-Specific NaN Issues\nFP8 quantization (fp8: true) can produce NaN when the activation quantization kernel divides by max(abs(x)) / 448. If the input tensor is all zeros (e.g., padding positions), the scale becomes 0, causing division by zero.\nFixes applied in axolotl:\n\nThe act_quant_kernel has a zero-guard: s = tl.where(s == 0, 1.0, s).\nA safety net nan_to_num(logits, nan=0.0) is applied in _get_per_token_logps_and_entropies.\nEmbedding padding is zero-padded for FP8 compatibility.\n\n\n\n\n\n\n\nImportantAfter Modifying Triton Kernels\n\n\n\nIf you patch any Triton JIT kernel (e.g., the FP8 quantization kernels in transformers), you must clear the Triton cache for changes to take effect:\nrm -rf ~/.triton/cache\n\n\n\n\nGeneral NaN Debugging Steps\n\nEnable anomaly detection (slow, but pinpoints the source):\ntorch.autograd.set_detect_anomaly(True)\nCheck grad_norm: If it goes to NaN, the backward pass is the problem. If loss is NaN but grad_norm was fine on the previous step, the forward pass is the problem.\nReduce to single GPU, single batch: Eliminate distributed training variables.\nInspect data: Print the batch that triggers NaN. Look for empty sequences, extreme token IDs, or unexpected padding patterns.", + "crumbs": [ + "Troubleshooting", + "Training Stability & Debugging" + ] + }, + { + "objectID": "docs/training_stability.html#oom-debugging", + "href": "docs/training_stability.html#oom-debugging", + "title": "Training Stability & Debugging", + "section": "OOM Debugging", + "text": "OOM Debugging\nOut-of-memory errors are the most common training failure. Use this systematic approach, from least to most disruptive:\n\nStep 1: Reduce Batch Size\nThe single highest-impact change. VRAM scales roughly linearly with batch size.\nmicro_batch_size: 1 # Start here\ngradient_accumulation_steps: 16 # Increase to maintain effective batch size\nFor GRPO specifically, the logits tensor for policy logprob computation can be very large. batch_size * num_generations * seq_len * vocab_size in bf16. For example, with num_generations: 16 and micro_batch_size: 8, the logits tensor alone is:\n8 * 16 * 2048 * 151936 * 2 bytes = ~75 GB (way too large)\nReduce micro_batch_size to 2–4 for GRPO.\n\n\nStep 2: Enable Gradient Checkpointing\nTrades compute for memory by recomputing activations during the backward pass instead of storing them.\ngradient_checkpointing: true\ngradient_checkpointing_kwargs:\n use_reentrant: false # Recommended default\n\n\n\n\n\n\nWarningReentrant Checkpointing Exceptions\n\n\n\nSome configurations require use_reentrant: true:\n\nDeepSpeed ZeRO-3 (non-reentrant causes CheckpointError)\nEBFT strided mode with flex_attention\n\n\n\n\n\nStep 3: Use Quantization\nLoad the base model in reduced precision:\n# 4-bit QLoRA\nadapter: qlora\nload_in_4bit: true\n\n# 8-bit\nload_in_8bit: true\n\n# FP8 (saves ~50% model VRAM, same compute speed as bf16)\nfp8: true\n\n\nStep 4: Reduce Sequence Length\nsequence_len: 1024 # Down from 2048 or 4096\nFor GRPO, also reduce max_completion_length. Memory scales quadratically with sequence length when using standard attention.\n\n\nStep 5: Use Flash Attention\nReduces attention memory from O(n^2) to O(n):\nflash_attention: true\n\n\nStep 6: Offload with DeepSpeed\nFor extreme cases, offload optimizer states or parameters to CPU:\ndeepspeed: deepspeed_configs/zero3_bf16.json\n\n\nDiagnosing the Specific Culprit\nUse the profiler_steps config option to capture GPU memory snapshots:\nprofiler_steps: [1, 2]\nThis generates PyTorch profiler traces you can inspect to see exactly which tensor allocation caused the OOM.", + "crumbs": [ + "Troubleshooting", + "Training Stability & Debugging" + ] + }, + { + "objectID": "docs/training_stability.html#common-errors", + "href": "docs/training_stability.html#common-errors", + "title": "Training Stability & Debugging", + "section": "Common Errors", + "text": "Common Errors\n\n\n\nError Message\nLikely Cause\nFix\n\n\n\n\nexitcode: -9\nSystem RAM exhaustion\nReduce dataset size, dataset_num_proc, or number of data workers\n\n\nexitcode: -7 (DeepSpeed)\nDeepSpeed version issue\npip install -U deepspeed\n\n\nCUDA out of memory\nGPU VRAM exhaustion\nFollow OOM debugging steps above\n\n\nRuntimeError: NCCL communicator was aborted\nGPU communication failure\nSee NCCL docs; check NCCL_DEBUG=INFO output\n\n\nValueError: Asking to pad but the tokenizer does not have a padding token\nMissing pad token\nAdd special_tokens: { pad_token: \"<\\|endoftext\\|>\" } to config\n\n\n'DummyOptim' object has no attribute 'step'\nDeepSpeed on single GPU\nRemove deepspeed: section from config\n\n\nunable to load strategy X then None is not callable\nReward module not importable\nRun cd experiments && python -c \"import my_rewards\" to check\n\n\ngeneration_batch_size not divisible by num_generations\nmicro_batch_size too small\nSet micro_batch_size >= num_generations and make it divisible\n\n\n'weight' must be 2-D\nFSDP1 flattened parameters\nUse fsdp_version: 2 or skip unwrap_model when FSDP is enabled\n\n\nCheckpointError (tensor count mismatch)\nNon-reentrant checkpointing + ZeRO-3 or flex_attention\nSet use_reentrant: true in gradient_checkpointing_kwargs\n\n\nBFloat16 TypeError during weight sync\nNumPy does not support bf16\nFixed in axolotl’s weight_serde.py (auto bf16 to fp16 conversion)\n\n\nContent end boundary is before start boundary\nChat template parsing issue\nCheck eos_token matches template; file a GitHub issue if persistent\n\n\nCAS service error during data processing\nHuggingFace XET issue\nSet export HF_HUB_DISABLE_XET=1\n\n\nTraining hangs (multi-GPU)\nFSDP + async prefetch deadlock\nSet async_prefetch: false with FSDP", + "crumbs": [ + "Troubleshooting", + "Training Stability & Debugging" + ] + }, + { + "objectID": "docs/training_stability.html#profiling", + "href": "docs/training_stability.html#profiling", + "title": "Training Stability & Debugging", + "section": "Profiling", + "text": "Profiling\n\nPyTorch Profiler\nAxolotl supports PyTorch profiler integration via the config:\nprofiler_steps: [1, 2, 3]\nThis captures profiler traces for the specified steps. View them in TensorBoard:\ntensorboard --logdir output_dir/runs\nOr open the .json trace file in chrome://tracing.\n\n\nCUDA Memory Snapshots\nFor detailed memory analysis, use PyTorch’s memory snapshot API. Add this to your training script or use it interactively:\nimport torch\n\n# Enable memory history tracking\ntorch.cuda.memory._record_memory_history()\n\n# ... run your training step ...\n\n# Save snapshot\ntorch.cuda.memory._dump_snapshot(\"memory_snapshot.pickle\")\nVisualize with PyTorch’s memory visualizer:\npython -m torch.cuda.memory._viz memory_snapshot.pickle\n\n\nQuick GPU Memory Check\nDuring training, monitor GPU utilization in a separate terminal:\nwatch -n 1 nvidia-smi\nFor programmatic access within axolotl, the logged metrics memory/max_alloc and memory/max_reserved come from torch.cuda.max_memory_allocated() and torch.cuda.max_memory_reserved(). Note these report PyTorch’s view of memory, which may differ from nvidia-smi (see FAQ).", + "crumbs": [ + "Troubleshooting", + "Training Stability & Debugging" + ] + }, + { + "objectID": "docs/training_stability.html#wb-and-logging", + "href": "docs/training_stability.html#wb-and-logging", + "title": "Training Stability & Debugging", + "section": "W&B and Logging", + "text": "W&B and Logging\n\nEnabling Logging\nwandb_project: my-project\nwandb_entity: my-team # optional\nwandb_run_id: run-123 # optional, for resuming\nwandb_name: experiment-name # optional\nlogging_steps: 1 # log every step (recommended for RL)\n\n\nDebug Logging\nFor detailed axolotl-internal debug output:\nAXOLOTL_LOG_LEVEL=DEBUG axolotl train config.yaml 2>&1 | tee /tmp/training.log\n\n\n\n\n\n\nTipAlways Log to a File\n\n\n\nPipe training output to a log file so you can inspect it after the run:\naxolotl train config.yaml 2>&1 | tee /tmp/my_run.log\n\n\n\n\nWhat Axolotl Logs\nSFT metrics (logged every logging_steps):\n\ntrain/loss, eval/loss – training and validation loss\ntrain/grad_norm – gradient L2 norm (before clipping)\ntrain/learning_rate – current learning rate\nmemory/max_alloc, memory/max_reserved – peak GPU memory\n\nGRPO/RL metrics (logged every step):\n\nrewards/<name>/mean, rewards/<name>/std – per-reward-function statistics\nreward, reward_std – aggregated reward across all reward functions\nfrac_reward_zero_std – fraction of prompt groups where all completions got the same reward\ncompletions/mean_length, completions/min_length, completions/max_length – completion token lengths\ncompletions/clipped_ratio – fraction of completions that hit the max length\ncompletions/mean_terminated_length, completions/min_terminated_length, completions/max_terminated_length – lengths of naturally terminated completions\nkl – KL divergence between policy and reference\nentropy – policy entropy (measure of output diversity)\nclip_ratio/region_mean, clip_ratio/low_mean, clip_ratio/high_mean – PPO clipping statistics\nsampling/sampling_logp_difference/mean, sampling/sampling_logp_difference/max – log-probability difference between policy and sampling distribution\nsampling/importance_sampling_ratio/min, sampling/importance_sampling_ratio/mean, sampling/importance_sampling_ratio/max – IS ratio statistics for off-policy correction\nnum_tokens – total tokens processed\n\n\n\nReading W&B Charts\nFor a healthy GRPO run, expect to see:\n\nreward/mean: Gradual upward trend. May start near 0 and reach 0.3–0.8 depending on task difficulty. Not monotonic – fluctuations are normal.\nentropy: Gradual decrease from initial values (often 0.3–0.6) as the model becomes more confident. Should not collapse to near-zero.\ngrad_norm: Mostly in the 0.001–1.0 range. Occasional 0.0 values are fine (zero-advantage skip). Persistent values above 10.0 need investigation.\nkl: Starts near 0 and grows slowly. If it shoots up rapidly, the policy is diverging from the reference.\ncompletions/mean_length: Should reflect the task’s natural answer length. If it steadily increases to max_completion_length, the model may be reward-hacking by generating longer outputs.", + "crumbs": [ + "Troubleshooting", + "Training Stability & Debugging" + ] }, { "objectID": "docs/cli.html", @@ -5816,7 +6567,7 @@ "href": "docs/rlhf.html", "title": "RLHF (Beta)", "section": "", - "text": "Reinforcement Learning from Human Feedback is a method whereby a language model is optimized from data using human\nfeedback. Various methods include, but not limited to:\n\nDirect Preference Optimization (DPO)\nIdentity Preference Optimization (IPO)\nKahneman-Tversky Optimization (KTO)\nOdds Ratio Preference Optimization (ORPO)\nGroup Relative Policy Optimization (GRPO)\nGroup Reward-Decoupled Policy Optimization (GDPO)\nEnergy-Based Fine-Tuning (EBFT)\nNeMo Gym Integration", + "text": "Reinforcement Learning from Human Feedback is a method whereby a language model is optimized from data using human\nfeedback. Various methods include, but not limited to:\n\nDirect Preference Optimization (DPO)\nIdentity Preference Optimization (IPO)\nKahneman-Tversky Optimization (KTO)\nOdds Ratio Preference Optimization (ORPO)\nGroup Relative Policy Optimization (GRPO) — see also the GRPO deep dive for async features, custom rewards, and scaling\nGroup Reward-Decoupled Policy Optimization (GDPO)\nEnergy-Based Fine-Tuning (EBFT) — see also the EBFT guide for detailed mode comparisons and configuration\nNeMo Gym Integration\n\nFor help choosing between these methods, see Choosing a Fine-Tuning Method.", "crumbs": [ "How To Guides", "RLHF (Beta)" @@ -5827,7 +6578,7 @@ "href": "docs/rlhf.html#overview", "title": "RLHF (Beta)", "section": "", - "text": "Reinforcement Learning from Human Feedback is a method whereby a language model is optimized from data using human\nfeedback. Various methods include, but not limited to:\n\nDirect Preference Optimization (DPO)\nIdentity Preference Optimization (IPO)\nKahneman-Tversky Optimization (KTO)\nOdds Ratio Preference Optimization (ORPO)\nGroup Relative Policy Optimization (GRPO)\nGroup Reward-Decoupled Policy Optimization (GDPO)\nEnergy-Based Fine-Tuning (EBFT)\nNeMo Gym Integration", + "text": "Reinforcement Learning from Human Feedback is a method whereby a language model is optimized from data using human\nfeedback. Various methods include, but not limited to:\n\nDirect Preference Optimization (DPO)\nIdentity Preference Optimization (IPO)\nKahneman-Tversky Optimization (KTO)\nOdds Ratio Preference Optimization (ORPO)\nGroup Relative Policy Optimization (GRPO) — see also the GRPO deep dive for async features, custom rewards, and scaling\nGroup Reward-Decoupled Policy Optimization (GDPO)\nEnergy-Based Fine-Tuning (EBFT) — see also the EBFT guide for detailed mode comparisons and configuration\nNeMo Gym Integration\n\nFor help choosing between these methods, see Choosing a Fine-Tuning Method.", "crumbs": [ "How To Guides", "RLHF (Beta)" @@ -5838,7 +6589,7 @@ "href": "docs/rlhf.html#rlhf-using-axolotl", "title": "RLHF (Beta)", "section": "RLHF using Axolotl", - "text": "RLHF using Axolotl\n\n\n\n\n\n\nImportant\n\n\n\nThis is a BETA feature and many features are not fully implemented. You are encouraged to open new PRs to improve the integration and functionality.\n\n\nWe rely on the TRL library for implementations of various RL training methods, which we wrap around to expose in axolotl. Each method has their own supported ways of loading datasets and prompt formats.\n\n\n\n\n\n\nTip\n\n\n\nYou can find what each method supports by going into src/axolotl/prompt_strategies/{method} where {method} is one of our supported methods. The type: can be retrieved from {method}.{function_name}.\n\n\n\nDPO\nExample config:\nrl: dpo\ndatasets:\n - path: Intel/orca_dpo_pairs\n split: train\n type: chatml.intel\n - path: argilla/ultrafeedback-binarized-preferences\n split: train\n type: chatml\nDPO supports the following types with the following dataset format:\n\nchatml.argilla\n{\n \"system\": \"...\", // optional\n \"instruction\": \"...\",\n \"chosen_response\": \"...\",\n \"rejected_response\": \"...\"\n}\n\n\nchatml.argilla_chat\n{\n \"chosen\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ],\n \"rejected\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\nchatml.icr\n{\n \"system\": \"...\", // optional\n \"input\": \"...\",\n \"chosen\": \"...\",\n \"rejected\": \"...\"\n}\n\n\nchatml.intel\n{\n \"system\": \"...\", // optional\n \"question\": \"...\",\n \"chosen\": \"...\",\n \"rejected\": \"...\"\n}\n\n\nchatml.prompt_pairs\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"chosen\": \"...\",\n \"rejected\": \"...\"\n}\n\n\nchatml.ultra\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"chosen\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ],\n \"rejected\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\nllama3.argilla\n{\n \"system\": \"...\", // optional\n \"instruction\": \"...\",\n \"chosen_response\": \"...\",\n \"rejected_response\": \"...\"\n}\n\n\nllama3.argilla_chat\n{\n \"chosen\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ],\n \"rejected\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\nllama3.icr\n{\n \"system\": \"...\", // optional\n \"input\": \"...\",\n \"chosen\": \"...\",\n \"rejected\": \"...\"\n}\n\n\nllama3.intel\n{\n \"system\": \"...\", // optional\n \"question\": \"...\",\n \"chosen\": \"...\",\n \"rejected\": \"...\"\n}\n\n\nllama3.prompt_pairs\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"chosen\": \"...\",\n \"rejected\": \"...\"\n}\n\n\nllama3.ultra\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"chosen\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ],\n \"rejected\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\nzephyr.nectar\n{\n \"prompt\": \"...\",\n \"answers\": [\n {\n \"answer\": \"...\",\n \"rank\": 1\n },\n {\n \"answer\": \"...\",\n \"rank\": 2\n }\n // ... more answers with ranks\n ]\n}\n\n\nchat_template.argilla_chat\n{\n \"chosen\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ],\n \"rejected\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\nchat_template.default\nrl: dpo\ndatasets:\n - path: ...\n split: train\n type: chat_template.default\n field_messages: \"messages\"\n field_chosen: \"chosen\"\n field_rejected: \"rejected\"\n message_property_mappings:\n role: role\n content: content\n roles:\n user: [\"user\"]\n assistant: [\"assistant\"]\n system: [\"system\"]\nSample input format:\n{\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"...\"\n },\n {\n \"role\": \"user\",\n \"content\": \"...\"\n },\n // ... more messages\n ],\n \"chosen\": {\n \"role\": \"assistant\",\n \"content\": \"...\"\n },\n \"rejected\": {\n \"role\": \"assistant\",\n \"content\": \"...\"\n }\n}\n\n\nuser_defined.default\nFor custom behaviors,\nrl: dpo\ndatasets:\n - path: ...\n split: train\n type:\n field_prompt: \"prompt\"\n field_system: \"system\"\n field_chosen: \"chosen\"\n field_rejected: \"rejected\"\n prompt_format: \"{prompt}\"\n chosen_format: \"{chosen}\"\n rejected_format: \"{rejected}\"\nThe input format is a simple JSON input with customizable fields based on the above config.\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"chosen\": \"...\",\n \"rejected\": \"...\"\n}\n\n\n\nIPO\nAs IPO is just DPO with a different loss function, all supported dataset formats for DPO are also supported for IPO.\nrl: ipo\n\n\nORPO\nPaper: https://arxiv.org/abs/2403.07691\nrl: orpo\norpo_alpha: 0.1\nremove_unused_columns: false\n\nchat_template: chatml\ndatasets:\n - path: argilla/ultrafeedback-binarized-preferences-cleaned\n type: chat_template.argilla\nORPO supports the following types with the following dataset format:\n\nchat_template.argilla\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\", // if available, will be taken as user message for single-turn instead of from list below\n\n // chosen/rejected should be same till last content and only even-number of alternating user/assistant turns\n \"chosen\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ],\n \"rejected\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\n\nKTO\nrl: kto\nrl_beta: 0.1 # default\nkto_desirable_weight: 1.0 # default\nkto_undesirable_weight: 1.0 # default\n\nremove_unused_columns: false\n\ndatasets:\n - path: argilla/ultrafeedback-binarized-preferences-cleaned-kto\n type: llama3.ultra\n split: train\n\ngradient_checkpointing: true\ngradient_checkpointing_kwargs:\n use_reentrant: true\nKTO supports the following types with the following dataset format:\n\nchatml.argilla\n{\n \"system\": \"...\", // optional\n \"instruction\": \"...\",\n \"completion\": \"...\"\n}\n\n\nchatml.argilla_chat\n{\n \"chosen\": [\n {\"role\": \"user\", \"content\": \"...\"}\n ],\n \"completion\": [\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\nchatml.intel\n{\n \"system\": \"...\", // optional\n \"question\": \"...\",\n \"completion\": \"...\"\n}\n\n\nchatml.prompt_pairs\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"completion\": \"...\"\n}\n\n\nchatml.ultra\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"completion\": \"...\"\n}\n\n\nllama3.argilla\n{\n \"system\": \"...\", // optional\n \"instruction\": \"...\",\n \"completion\": \"...\"\n}\n\n\nllama3.argilla_chat\n{\n \"completion\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\nllama3.intel\n{\n \"system\": \"...\", // optional\n \"question\": \"...\",\n \"completion\": \"...\"\n}\n\n\nllama3.prompt_pairs\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"completion\": \"...\"\n}\n\n\nllama3.ultra\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"completion\": \"...\"\n}\n\n\nuser_defined.default\nFor custom behaviors,\nrl: kto\ndatasets:\n - path: ...\n split: train\n type:\n field_prompt: \"prompt\"\n field_system: \"system\"\n field_completion: \"completion\"\n field_label: \"label\"\n prompt_format: \"{prompt}\"\n completion_format: \"{completion}\"\nThe input format is a simple JSON input with customizable fields based on the above config.\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"completion\": \"...\",\n \"label\": \"...\"\n}\n\n\n\nGRPO\n\n\n\n\n\n\nTip\n\n\n\nCheck out our GRPO cookbook.\n\n\nIn the latest GRPO implementation, vLLM is used to significantly speedup trajectory generation during training. In this example, we’re using 4 GPUs - 2 for training, and 2 for vLLM:\n\n\n\n\n\n\nImportant\n\n\n\nMake sure you’ve installed the correct version of vLLM by including it as an extra when installing axolotl, e.g. pip install axolotl[vllm].\n\n\nbase_model: Qwen/Qwen2.5-1.5B-Instruct\n\nvllm:\n host: 0.0.0.0\n port: 8000\n tensor_parallel_size: 2\n gpu_memory_utilization: 0.85\n dtype: auto\n # max_model_len: # you may find it useful to set the vLLM model context length if you know this beforehand\n\nrl: grpo\ntrl:\n use_vllm: true\n vllm_server_host: 0.0.0.0\n vllm_server_port: 8000\n vllm_server_timeout: 300\nCUDA_VISIBLE_DEVICES=2,3 axolotl vllm-serve grpo.yaml\nYour vLLM instance will now attempt to spin up, and it’s time to kick off training utilizing our remaining two GPUs. In another terminal, execute:\nCUDA_VISIBLE_DEVICES=0,1 axolotl train grpo.yaml --num-processes 2\n\n\n\n\n\n\nNote\n\n\n\nDue to TRL’s implementation with vLLM, the vLLM instance must use the last N GPUs instead of the first N GPUs. This is why in the example above, we use CUDA_VISIBLE_DEVICES=2,3 for the vLLM instance.\n\n\n\nReward functions\nGRPO uses custom reward functions and transformations. Please have them ready locally.\nFor example, to load OpenAI’s GSM8K and use a random reward for completions:\n# rewards.py\nimport random\n\ndef rand_reward_func(completions, **kwargs) -> list[float]:\n return [random.uniform(0, 1) for _ in completions]\n\ndef oai_gsm8k_transform(cfg, *args, **kwargs):\n def transform_fn(example, tokenizer=None):\n label = example[\"answer\"].split(\"####\")[-1].strip().replace(\",\", \"\")\n return {\n \"prompt\": [{\"role\": \"user\", \"content\": example[\"question\"]},],\n \"answer\": label,\n }\n return transform_fn, {\"remove_columns\": [\"question\"]}\nrl: grpo\n\ntrl:\n beta: 0.001\n max_completion_length: 256\n use_vllm: True\n num_generations: 4\n reward_funcs: [\"rewards.rand_reward_func\"] # format: '{file_name}.{fn_name}'\n reward_weights: [1.0]\ndatasets:\n - path: openai/gsm8k\n name: main\n type: rewards.oai_gsm8k_transform # format: '{file_name}.{fn_name}'\nTo see other examples of custom reward functions, please see TRL GRPO Docs.\nTo see all configs, please see TRLConfig.\n\n\nOpenEnv Rollout Functions\nGRPO supports custom rollout functions for OpenEnv-style environments, enabling interactive tasks like web browsing, code execution, or tool use. This allows you to implement custom generation logic that interacts with external environments.\nFor example, to implement a simple math-solving environment with step-by-step verification:\n# math_env.py\nimport re\n\ndef math_solver_rollout(model, processing_class, prompts, generation_config=None):\n \"\"\"\n Custom rollout function that generates step-by-step math solutions.\n\n Args:\n model: The language model\n processing_class: The tokenizer/processing_class\n prompts: List of prompt dicts (with 'messages' key for chat format)\n generation_config: Optional generation configuration\n\n Returns:\n List of completion strings\n \"\"\"\n completions = []\n\n for prompt in prompts:\n # Apply chat template to prompt\n messages = prompt.get(\"messages\", [])\n formatted_prompt = processing_class.apply_chat_template(\n messages, processing_class=False, add_generation_prompt=True\n )\n\n # Generate step-by-step solution\n full_response = \"\"\n for step in range(5): # Max 5 reasoning steps\n current_input = formatted_prompt + full_response + \"\\nNext step:\"\n inputs = processing_class(current_input, return_tensors=\"pt\").to(model.device)\n\n outputs = model.generate(\n **inputs,\n max_new_tokens=100,\n generation_config=generation_config,\n )\n step_text = processing_class.decode(\n outputs[0][inputs.input_ids.shape[1]:],\n skip_special_tokens=True\n )\n\n # Check if solution is complete\n if \"FINAL ANSWER:\" in step_text:\n full_response += step_text\n break\n full_response += step_text + \"\\n\"\n\n completions.append(full_response)\n\n return completions\n\ndef math_reward(prompts, completions, answers, **kwargs):\n \"\"\"Reward function that checks mathematical correctness\"\"\"\n rewards = []\n for completion, correct_answer in zip(completions, answers):\n # Extract predicted answer\n match = re.search(r\"FINAL ANSWER:\\s*(.+)\", completion)\n predicted = match.group(1).strip() if match else \"\"\n\n # Compare with correct answer\n reward = 1.0 if predicted == str(correct_answer) else 0.0\n rewards.append(reward)\n\n return rewards\n\ndef math_transform(cfg, *args, **kwargs):\n \"\"\"Transform dataset to GRPO format with answer field\"\"\"\n def transform_fn(example, processing_class=None):\n return {\n \"prompt\": [{\"role\": \"user\", \"content\": example[\"question\"]}],\n \"answer\": str(example[\"answer\"]),\n }\n return transform_fn, {\"remove_columns\": [\"question\"]}\nrl: grpo\n\ntrl:\n beta: 0.001\n max_completion_length: 512\n num_generations: 4\n rollout_func: \"math_env.math_solver_rollout\" # Custom rollout function\n reward_funcs: [\"math_env.math_reward\"]\n reward_weights: [1.0]\n\ndatasets:\n - path: openai/gsm8k\n name: main\n type: math_env.math_transform\nThe rollout_func parameter accepts a fully qualified name (e.g., module_name.function_name) that points to a callable function in your local directory. The function receives:\n\nmodel: The language model\nprocessing_class: The tokenizer/processing class\nprompts: List of prompt dictionaries\ngeneration_config (optional): Generation configuration\n\nAnd should return a list of completion strings.\nFor more OpenEnv examples, see TRL OpenEnv Documentation.\n\n\nGRPO with DAPO/Dr. GRPO loss\nThe DAPO paper and subsequently Dr. GRPO paper proposed an alternative loss function for GRPO to remediate the penalty in longer responses.\ntrl:\n loss_type: dr_grpo\n # Normalizes loss based on max completion length (default: 256)\n max_completion_length:\nFor more information, see GRPO docs.\n\n\nAsync GRPO\nAsync GRPO overlaps vLLM generation with training by producing rollouts in a background thread. While the model trains on the current batch, the next batch is already being generated. This can significantly reduce wall-clock time per step.\ntrl:\n use_data_producer: true # Enable data producer protocol\n use_vllm: true\n async_prefetch: true # Generate rollouts in background thread\n prefetch_depth: 1 # Number of rollouts to prefetch\n vllm_sync_interval: 2 # Sync weights to vLLM every N steps\n\n\n\n\n\n\nNote\n\n\n\nBecause the background thread generates completions with slightly stale model weights, async GRPO uses importance sampling correction to account for the distribution shift. This is controlled by vllm_importance_sampling_correction: true (default when async is enabled).\n\n\n\nvLLM LoRA Sync\nBy default, weight sync to vLLM merges the LoRA adapter into the base model and broadcasts all parameters via NCCL. LoRA sync is a faster alternative that saves only the adapter weights to the filesystem and has vLLM load them natively using Punica kernels.\nadapter: lora\nlora_r: 32\nlora_alpha: 64\nlora_target_linear: true\n\ntrl:\n vllm_lora_sync: true # Enable native LoRA sync\nWhen vllm_lora_sync: true is set, axolotl automatically selects the LoRA-aware vLLM serve module. Start vLLM as usual:\nCUDA_VISIBLE_DEVICES=0 axolotl vllm-serve config.yaml\nThen start training on a separate GPU:\nCUDA_VISIBLE_DEVICES=1 axolotl train config.yaml\n\n\n\n\n\n\nTip\n\n\n\nLoRA sync is especially beneficial with multi-GPU training (FSDP/DeepSpeed), where NCCL merge-sync can cause GPU contention with vLLM generation.\n\n\n\n\nStreaming Partial Batch\nInstead of scoring the entire batch at once, streaming mode scores one prompt group at a time. This enables finer-grained zero-advantage skipping and reduces peak memory usage during scoring.\ntrl:\n streaming_partial_batch: true\n\n\nImportance Sampling Correction\nWhen using async prefetch, completions are generated from a slightly older version of the model. Importance sampling (IS) correction adjusts the policy gradient to account for this distribution shift.\ntrl:\n vllm_importance_sampling_correction: true # Enable IS correction\n importance_sampling_level: token # 'token' or 'sequence'\n off_policy_mask_threshold: 0.5 # Mask sequences with IS ratio below this\n\nimportance_sampling_level: token applies per-token IS ratios (recommended with Liger kernel)\nimportance_sampling_level: sequence applies per-sequence IS ratios\noff_policy_mask_threshold masks out sequences where the IS ratio indicates they are too far off-policy\n\n\n\nReplay Buffer\nThe replay buffer caches rollout groups that had learning signal (non-zero reward variance) and uses them to replace zero-signal groups in later batches.\ntrl:\n replay_buffer_size: 100 # Max cached groups (0 = disabled)\n replay_recompute_logps: true # Recompute log-probs for replayed data (recommended)\n\n\n\n\n\n\nNote\n\n\n\nWhen replay_recompute_logps: true (default), old log-probabilities are recomputed using the current model weights. This fixes the IS mismatch that would otherwise occur when replaying stale data.\n\n\n\n\nDeferred Re-rolling\nFailed prompts (where the model produces zero reward for all generations) are buffered and re-injected into later batches when the model may be better equipped to solve them.\ntrl:\n reroll_start_fraction: 0.5 # Start re-rolling after 50% of training\n reroll_max_groups: 1 # Max groups to replace per batch\n\n\nZero-Advantage Batch Skipping\nWhen all advantages in a micro-batch are zero (no learning signal), the forward/backward pass is skipped entirely. This is enabled by default and logged as skipped_zero_adv_batches=1.\ntrl:\n skip_zero_advantage_batches: true # default\n\n\nParallel Reward Workers\nReward functions that use signal.alarm() (e.g., math_verify) must run in the main thread. Parallel reward workers use subprocesses to work around this limitation while enabling concurrent reward computation.\ntrl:\n reward_num_workers: 4 # Number of subprocess workers (1 = no parallelism)\n\n\nFull Async GRPO Example\nbase_model: Qwen/Qwen2.5-1.5B-Instruct\n\nvllm:\n host: 0.0.0.0\n port: 8000\n gpu_memory_utilization: 0.35\n dtype: auto\n\nadapter: lora\nlora_r: 32\nlora_alpha: 64\nlora_target_linear: true\n\nrl: grpo\ntrl:\n use_data_producer: true\n use_vllm: true\n async_prefetch: true\n prefetch_depth: 1\n vllm_sync_interval: 2\n vllm_lora_sync: true\n streaming_partial_batch: true\n vllm_importance_sampling_correction: true\n off_policy_mask_threshold: 0.5\n importance_sampling_level: token\n num_generations: 8\n max_completion_length: 512\n reward_funcs:\n - rewards.accuracy_reward\n reroll_start_fraction: 0.5\n replay_buffer_size: 100\n reward_num_workers: 4\n skip_zero_advantage_batches: true\n\ndatasets:\n - path: AI-MO/NuminaMath-TIR\n type: rewards.prompt_transform\n split: train\n\ngradient_accumulation_steps: 4\nmicro_batch_size: 2\nmax_steps: 500\nlearning_rate: 1e-5\nbf16: true\ngradient_checkpointing: true\n# Terminal 1: Start vLLM on GPU 0\nCUDA_VISIBLE_DEVICES=0 axolotl vllm-serve config.yaml\n\n# Terminal 2: Train on GPU 1\nCUDA_VISIBLE_DEVICES=1 axolotl train config.yaml\n\n\nMulti-GPU Async GRPO\nAsync GRPO supports FSDP and DeepSpeed ZeRO-3 for multi-GPU training. vLLM runs on one GPU while training is distributed across the remaining GPUs.\nFSDP:\nfsdp:\n - full_shard\n - auto_wrap\nfsdp_config:\n fsdp_transformer_layer_cls_to_wrap: Qwen2DecoderLayer\ngradient_checkpointing_kwargs:\n use_reentrant: false\nDeepSpeed ZeRO-3:\ndeepspeed: deepspeed_configs/zero3_bf16.json\ngradient_checkpointing_kwargs:\n use_reentrant: true # Required for ZeRO-3\n# Terminal 1: Start vLLM on GPU 0\nCUDA_VISIBLE_DEVICES=0 axolotl vllm-serve config.yaml\n\n# Terminal 2: Train on GPUs 0,1\nCUDA_VISIBLE_DEVICES=0,1 accelerate launch --num_processes 2 -m axolotl.cli.train config.yaml\n\n\n\n\n\n\nImportant\n\n\n\nWith multi-GPU async prefetch, only rank 0 generates completions in the background thread. Results are broadcast to all ranks on the main thread. This avoids FSDP/DeepSpeed collective deadlocks from unsynchronized background threads.\n\n\n\n\n\n\nGDPO\nGDPO (Group Reward-Decoupled Policy Optimization) extends GRPO for multi-reward training. It addresses the reward advantage collapse problem by normalizing each reward function independently before combining them.\n\n\n\n\n\n\nTip\n\n\n\nUse GDPO when training with multiple reward functions. For single reward, GRPO and GDPO produce equivalent results.\n\n\nPaper: https://arxiv.org/pdf/2501.05242\nGDPO uses TRL’s native multi_objective_aggregation parameter under the hood. When you set rl: gdpo, axolotl automatically configures TRL to use normalize_then_sum aggregation.\nbase_model: Qwen/Qwen2.5-1.5B-Instruct\n\nvllm:\n host: 0.0.0.0\n port: 8000\n tensor_parallel_size: 2\n gpu_memory_utilization: 0.85\n\nrl: gdpo\n\ntrl:\n beta: 0.001\n max_completion_length: 256\n use_vllm: true\n num_generations: 4\n reward_funcs:\n - rewards.format_reward\n - rewards.correctness_reward\n reward_weights: [1.0, 2.0]\n\ndatasets:\n - path: openai/gsm8k\n name: main\n type: rewards.oai_gsm8k_transform\nYou can also use GRPO with explicit aggregation control:\nrl: grpo\ntrl:\n multi_objective_aggregation: normalize_then_sum # GDPO behavior\n # or: sum_then_normalize # Default GRPO behavior\n\nGDPO vs GRPO\n\n\n\n\n\n\n\n\nAspect\nGRPO\nGDPO\n\n\n\n\nAggregation\nsum_then_normalize\nnormalize_then_sum\n\n\nMulti-reward\nMay collapse advantages\nPreserves reward signals\n\n\nSingle reward\nStandard behavior\nEquivalent to GRPO\n\n\n\n\n\nWhy GDPO?\nWhen using multiple rewards with GRPO, different reward combinations can produce identical advantages:\n# Example: format + correctness rewards\n[format=0, correct=3] → sum=3\n[format=1, correct=2] → sum=3 ← GRPO sees these as equal!\n[format=2, correct=1] → sum=3\n[format=3, correct=0] → sum=3\nGDPO normalizes each reward independently, preserving their relative differences.\n\n\nReward Functions\nGDPO uses the same reward function format as GRPO:\n# rewards.py\ndef format_reward(completions, **kwargs) -> list[float]:\n return [1.0 if len(c) > 10 else 0.0 for c in completions]\n\ndef correctness_reward(completions, answers, **kwargs) -> list[float]:\n rewards = []\n for completion, answer in zip(completions, answers):\n # Your scoring logic here\n rewards.append(score)\n return rewards\n\n\nSequence Parallelism\nGDPO supports sequence parallelism for long-context training:\nrl: gdpo\ncontext_parallel_size: 2\n\n\n\nSimPO\nSimPO uses CPOTrainer but with alternative loss function.\nrl: simpo\nrl_beta: 0.1 # default in CPOTrainer\ncpo_alpha: 1.0 # default in CPOTrainer\nsimpo_gamma: 0.5 # default in CPOTrainer\nThis method uses the same dataset format as DPO.\n\n\nEBFT\nEBFT (Energy-Based Fine-Tuning) fine-tunes language models by optimizing a feature-matching loss rather than relying on external reward functions. A frozen copy of the model extracts embeddings from both generated and ground-truth completions, and the generator is updated via REINFORCE to match the ground-truth feature moments.\nPaper: “Matching Features, Not Tokens: Energy-Based Fine-Tuning of Language Models” (Jelassi et al., 2026)\nKey advantages:\n\nNo reward model or verifier required — works on any (prompt, completion) data\nApplicable to non-verifiable tasks (code, translation, creative writing)\nOperates on model rollouts (not teacher forcing), reducing distribution shift\n\nEBFT supports two modes:\n\nStructured mode: For QA/instruction data with prompt + completion pairs. Uses vLLM for generation (like GRPO).\nStrided mode: For unstructured text without prompt/completion splits. Uses strided block-parallel generation with flex_attention — no vLLM needed.\n\n\nStructured Mode\nbase_model: Qwen/Qwen3-4B\n\nrl: ebft\n\nebft:\n feature_layers: [0.25, 0.5, 0.75] # Extract features at 25%, 50%, 75% depth\n embed_method: last_token\n use_whitening: false\n alignment_coef: 1.0 # Cosine similarity reward weight\n diversity_coef: 1.0 # Pairwise dot product penalty\n ce_coef: 0.0 # Cross-entropy on GT tokens (0 = off)\n\ntrl:\n num_generations: 4\n max_completion_length: 256\n temperature: 0.7\n use_vllm: true\n vllm_server_host: 0.0.0.0\n vllm_server_port: 8000\n vllm_lora_sync: true # LoRA adapter sync (recommended)\n vllm_sync_interval: 3\n use_data_producer: true\n async_prefetch: true # Set false for sync mode\n scale_rewards: true\n loss_type: grpo\n epsilon: 0.2\n\nvllm:\n gpu_memory_utilization: 0.5\n max_model_len: 2048\n\ndatasets:\n - path: nvidia/OpenCodeInstruct\n type: ebft_opencode.transform\n split: train[:500]\n\nadapter: lora\nlora_r: 16\nlora_alpha: 32\nlora_target_linear: true\n# Terminal 1: Start vLLM\nCUDA_VISIBLE_DEVICES=0 axolotl vllm-serve config.yaml\n\n# Terminal 2: Train\nCUDA_VISIBLE_DEVICES=1 axolotl train config.yaml\n\n\nStrided Mode\nFor unstructured text (raw code, prose). No vLLM needed — runs on a single GPU.\nbase_model: meta-llama/Llama-3.2-1B\n\nrl: ebft\n\nebft:\n mode: strided\n stride: 8\n context_length: 8\n generate_max_len: 8\n n_samples_per_prompt: 4\n temperature: 0.6\n feature_layers: [0.25, 0.5, 0.75]\n embed_method: last_token\n use_whitening: true\n alignment_coef: 1.0\n diversity_coef: 1.0\n rl_coef: 1.0\n ce_coef: 0.03\n advantage_estimator: rloo\n\ndatasets:\n - path: nvidia/OpenCodeInstruct\n type: ebft_strided_structured.transform\n split: train[:1%]\n\nflash_attention: false\nflex_attention: true # Strided mode uses flex_attention\ngradient_checkpointing: true\ngradient_checkpointing_kwargs:\n use_reentrant: true # Required for flex_attention\nCUDA_VISIBLE_DEVICES=0 axolotl train config.yaml\n\n\n\n\n\n\nTip\n\n\n\nSee examples/ebft/ for complete example configs covering Llama 1B/3B/8B and Qwen3 4B/8B models in both modes.\n\n\n\n\nEBFT Configuration Reference\n\n\n\n\n\n\n\n\nParameter\nDefault\nDescription\n\n\n\n\nebft.feature_layers\n[0.25, 0.5, 0.75]\nLayer depths for feature extraction (fractional)\n\n\nebft.embed_method\nlast_token\nFeature pooling: last_token, mean_pooling, concat\n\n\nebft.use_whitening\nfalse\nSVD whitening of feature dimensions\n\n\nebft.alignment_coef\n1.0\nCosine similarity reward weight\n\n\nebft.diversity_coef\n1.0\nPairwise dot product penalty weight\n\n\nebft.ce_coef\n0.0\nCross-entropy loss on ground-truth tokens\n\n\nebft.mode\nstructured\nstructured (vLLM) or strided (no vLLM)\n\n\nebft.stride\n—\nTokens between anchor points (strided mode)\n\n\nebft.context_length\n—\nContext window per block (strided mode)\n\n\nebft.generate_max_len\n—\nTokens to generate per block (strided mode)\n\n\nebft.n_samples_per_prompt\n—\nRollouts per document (strided mode)\n\n\nebft.advantage_estimator\ngrpo\ngrpo or rloo (strided mode)\n\n\n\n\n\n\nNeMo Gym Integration\nNeMo Gym provides 50+ verified RL environments (math, coding, tool-use, reasoning) with deterministic reward signals. The axolotl integration supports both single-turn (call /verify after generation) and multi-turn (agent-based tool execution via /run).\n\nSingle-Turn (Simplest)\nFor environments that only need answer verification (math, coding challenges). No agent server needed — the reward function calls /verify directly on the resource server.\nbase_model: Qwen/Qwen2.5-0.5B-Instruct\n\nrl: grpo\nchat_template: tokenizer_default\n\ntrl:\n use_vllm: false # Colocate mode (single GPU)\n num_generations: 4\n max_completion_length: 128\n temperature: 0.9\n reward_funcs:\n - axolotl.integrations.nemo_gym.rewards.reward_nemo_gym_verify\n\nplugins:\n - axolotl.integrations.nemo_gym.NemoGymPlugin\n\nnemo_gym_enabled: true\nnemo_gym_dir: ~/Gym\nnemo_gym_auto_start: false\nnemo_gym_head_port: 11000\nnemo_gym_datasets:\n - path: resources_servers/reasoning_gym/data/train_basic_arithmetic.jsonl\n server_name: reasoning_gym\n\ndatasets:\n - path: ~/Gym/resources_servers/reasoning_gym/data/train_basic_arithmetic.jsonl\n type: chat_template\n field_messages: responses_create_params.input\n message_field_content: content\n message_field_role: role\n# Terminal 1: Start NeMo Gym resource server\ncd ~/Gym && .venv/bin/ng_run \\\n \"+config_paths=[resources_servers/reasoning_gym/configs/resources_only.yaml]\" \\\n \"+skip_venv_if_present=true\"\n\n# Terminal 2: Train\nCUDA_VISIBLE_DEVICES=0 axolotl train config.yaml\n\n\n\n\n\n\nNote\n\n\n\nnemo_gym_datasets.path is relative to nemo_gym_dir. Don’t use absolute paths or they will be double-joined.\n\n\n\n\nMulti-Turn with Async GRPO (Recommended)\nFor environments with tool-use (weather, search, databases). An agent server orchestrates multi-turn interactions: generate → parse tool calls → execute tools → feed results back → repeat until done.\nbase_model: Qwen/Qwen3-0.6B\n\nrl: grpo\nchat_template: tokenizer_default\n\nadapter: lora\nlora_r: 16\nlora_alpha: 32\nlora_target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]\n\ntrl:\n use_vllm: true\n vllm_mode: server\n vllm_server_host: localhost\n vllm_server_port: 8000\n vllm_lora_sync: true\n vllm_sync_interval: 5\n use_data_producer: true\n async_prefetch: true # 3x speedup\n num_generations: 4\n max_completion_length: 512\n temperature: 0.8\n reward_funcs:\n - axolotl.integrations.nemo_gym.rewards.reward_env\n\nplugins:\n - axolotl.integrations.nemo_gym.NemoGymPlugin\n\nnemo_gym_enabled: true\nnemo_gym_auto_start: false\nnemo_gym_head_port: 11000\nnemo_gym_multi_turn: true\nnemo_gym_verify_timeout: 120\nnemo_gym_datasets:\n - path: resources_servers/example_single_tool_call/data/weather_tool_calling.jsonl\n server_name: example_single_tool_call\n\ndatasets:\n - path: ~/Gym/resources_servers/example_single_tool_call/data/weather_tool_calling.jsonl\n type: chat_template\n field_messages: responses_create_params.input\n message_field_content: content\n message_field_role: role\n\nvllm:\n gpu_memory_utilization: 0.85\n max_model_len: 2048\nMulti-turn requires three services running:\n# Terminal 1: vLLM with LoRA + tool calling\nVLLM_ALLOW_RUNTIME_LORA_UPDATING=1 CUDA_VISIBLE_DEVICES=0 \\\n python -m vllm.entrypoints.openai.api_server \\\n --model Qwen/Qwen3-0.6B --max-model-len 2048 \\\n --gpu-memory-utilization 0.85 \\\n --enable-lora --max-lora-rank 64 \\\n --enable-auto-tool-choice --tool-call-parser hermes\n\n# Terminal 2: NeMo Gym servers (resource + model proxy + agent)\ncd ~/Gym && .venv/bin/ng_run \\\n \"+config_paths=[configs/axolotl_tool_calling.yaml]\" \\\n \"+skip_venv_if_present=true\"\n\n# Terminal 3: Training\nCUDA_VISIBLE_DEVICES=1 axolotl train config.yaml\n\n\n\n\n\n\nImportant\n\n\n\nMulti-turn requires a NeMo Gym agent config YAML that defines three components: a resource server (tools + /verify), a model server proxy (forwards to your vLLM), and an agent server (orchestrates /run). See the NeMo Gym README for agent config format.\n\n\n\n\nNeMo Gym Prerequisites\n# Clone and set up NeMo Gym\ngit clone https://github.com/NVIDIA-NeMo/Gym.git ~/Gym\ncd ~/Gym\nuv venv --python 3.12 && source .venv/bin/activate && uv sync\n\n# Fix pycosat build (GCC 13+)\nCFLAGS=\"\" uv pip install pycosat --python .venv/bin/python --no-build-isolation\n\n\nNeMo Gym Configuration Reference\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nnemo_gym_enabled\nbool\n—\nEnable the NeMo Gym integration\n\n\nnemo_gym_dir\nstr\n~/Gym\nPath to NeMo Gym repo\n\n\nnemo_gym_auto_start\nbool\ntrue\nAuto-start resource servers\n\n\nnemo_gym_head_port\nint\n11000\nHead server port\n\n\nnemo_gym_multi_turn\nbool\nfalse\nEnable multi-turn via agent /run\n\n\nnemo_gym_verify_timeout\nint\n30\nPer-request timeout (seconds)\n\n\nnemo_gym_datasets\nlist\nrequired\nDataset configs with path and server_name\n\n\n\n\n\nReward Functions\n\n\n\n\n\n\n\n\nFunction\nMode\nDescription\n\n\n\n\naxolotl.integrations.nemo_gym.rewards.reward_nemo_gym_verify\nSingle-turn\nCalls /verify, returns binary reward\n\n\naxolotl.integrations.nemo_gym.rewards.reward_env\nMulti-turn\nPassthrough reward from agent /run\n\n\n\n\n\n\nUsing local dataset files\ndatasets:\n - ds_type: json\n data_files:\n - orca_rlhf.jsonl\n split: train\n type: chatml.intel\n\n\nTRL auto-unwrapping for PEFT\nTRL supports auto-unwrapping PEFT models for RL training paradigms which rely on a reference model. This significantly reduces memory pressure as an additional refreference model does not need to be loaded, and reference model log-probabilities can be obtained by disabling PEFT adapters. This is enabled by default. To turn it off, pass the following config:\n# load ref model when adapter training.\nrl_adapter_ref_model: true", + "text": "RLHF using Axolotl\n\n\n\n\n\n\nImportant\n\n\n\nThis is a BETA feature and many features are not fully implemented. You are encouraged to open new PRs to improve the integration and functionality.\n\n\nWe rely on the TRL library for implementations of various RL training methods, which we wrap around to expose in axolotl. Each method has their own supported ways of loading datasets and prompt formats.\n\n\n\n\n\n\nTip\n\n\n\nYou can find what each method supports by going into src/axolotl/prompt_strategies/{method} where {method} is one of our supported methods. The type: can be retrieved from {method}.{function_name}.\n\n\n\nDPO\nExample config:\nrl: dpo\ndatasets:\n - path: Intel/orca_dpo_pairs\n split: train\n type: chatml.intel\n - path: argilla/ultrafeedback-binarized-preferences\n split: train\n type: chatml\nDPO supports the following types with the following dataset format:\n\nchatml.argilla\n{\n \"system\": \"...\", // optional\n \"instruction\": \"...\",\n \"chosen_response\": \"...\",\n \"rejected_response\": \"...\"\n}\n\n\nchatml.argilla_chat\n{\n \"chosen\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ],\n \"rejected\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\nchatml.icr\n{\n \"system\": \"...\", // optional\n \"input\": \"...\",\n \"chosen\": \"...\",\n \"rejected\": \"...\"\n}\n\n\nchatml.intel\n{\n \"system\": \"...\", // optional\n \"question\": \"...\",\n \"chosen\": \"...\",\n \"rejected\": \"...\"\n}\n\n\nchatml.prompt_pairs\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"chosen\": \"...\",\n \"rejected\": \"...\"\n}\n\n\nchatml.ultra\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"chosen\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ],\n \"rejected\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\nllama3.argilla\n{\n \"system\": \"...\", // optional\n \"instruction\": \"...\",\n \"chosen_response\": \"...\",\n \"rejected_response\": \"...\"\n}\n\n\nllama3.argilla_chat\n{\n \"chosen\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ],\n \"rejected\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\nllama3.icr\n{\n \"system\": \"...\", // optional\n \"input\": \"...\",\n \"chosen\": \"...\",\n \"rejected\": \"...\"\n}\n\n\nllama3.intel\n{\n \"system\": \"...\", // optional\n \"question\": \"...\",\n \"chosen\": \"...\",\n \"rejected\": \"...\"\n}\n\n\nllama3.prompt_pairs\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"chosen\": \"...\",\n \"rejected\": \"...\"\n}\n\n\nllama3.ultra\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"chosen\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ],\n \"rejected\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\nzephyr.nectar\n{\n \"prompt\": \"...\",\n \"answers\": [\n {\n \"answer\": \"...\",\n \"rank\": 1\n },\n {\n \"answer\": \"...\",\n \"rank\": 2\n }\n // ... more answers with ranks\n ]\n}\n\n\nchat_template.argilla_chat\n{\n \"chosen\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ],\n \"rejected\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\nchat_template.default\nrl: dpo\ndatasets:\n - path: ...\n split: train\n type: chat_template.default\n field_messages: \"messages\"\n field_chosen: \"chosen\"\n field_rejected: \"rejected\"\n message_property_mappings:\n role: role\n content: content\n roles:\n user: [\"user\"]\n assistant: [\"assistant\"]\n system: [\"system\"]\nSample input format:\n{\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"...\"\n },\n {\n \"role\": \"user\",\n \"content\": \"...\"\n },\n // ... more messages\n ],\n \"chosen\": {\n \"role\": \"assistant\",\n \"content\": \"...\"\n },\n \"rejected\": {\n \"role\": \"assistant\",\n \"content\": \"...\"\n }\n}\n\n\nuser_defined.default\nFor custom behaviors,\nrl: dpo\ndatasets:\n - path: ...\n split: train\n type:\n field_prompt: \"prompt\"\n field_system: \"system\"\n field_chosen: \"chosen\"\n field_rejected: \"rejected\"\n prompt_format: \"{prompt}\"\n chosen_format: \"{chosen}\"\n rejected_format: \"{rejected}\"\nThe input format is a simple JSON input with customizable fields based on the above config.\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"chosen\": \"...\",\n \"rejected\": \"...\"\n}\n\n\n\nIPO\nAs IPO is just DPO with a different loss function, all supported dataset formats for DPO are also supported for IPO.\nrl: ipo\n\n\nORPO\nPaper: https://arxiv.org/abs/2403.07691\nrl: orpo\norpo_alpha: 0.1\nremove_unused_columns: false\n\nchat_template: chatml\ndatasets:\n - path: argilla/ultrafeedback-binarized-preferences-cleaned\n type: chat_template.argilla\nORPO supports the following types with the following dataset format:\n\nchat_template.argilla\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\", // if available, will be taken as user message for single-turn instead of from list below\n\n // chosen/rejected should be same till last content and only even-number of alternating user/assistant turns\n \"chosen\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ],\n \"rejected\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\n\nKTO\nrl: kto\nrl_beta: 0.1 # default\nkto_desirable_weight: 1.0 # default\nkto_undesirable_weight: 1.0 # default\n\nremove_unused_columns: false\n\ndatasets:\n - path: argilla/ultrafeedback-binarized-preferences-cleaned-kto\n type: llama3.ultra\n split: train\n\ngradient_checkpointing: true\ngradient_checkpointing_kwargs:\n use_reentrant: true\nKTO supports the following types with the following dataset format:\n\nchatml.argilla\n{\n \"system\": \"...\", // optional\n \"instruction\": \"...\",\n \"completion\": \"...\"\n}\n\n\nchatml.argilla_chat\n{\n \"chosen\": [\n {\"role\": \"user\", \"content\": \"...\"}\n ],\n \"completion\": [\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\nchatml.intel\n{\n \"system\": \"...\", // optional\n \"question\": \"...\",\n \"completion\": \"...\"\n}\n\n\nchatml.prompt_pairs\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"completion\": \"...\"\n}\n\n\nchatml.ultra\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"completion\": \"...\"\n}\n\n\nllama3.argilla\n{\n \"system\": \"...\", // optional\n \"instruction\": \"...\",\n \"completion\": \"...\"\n}\n\n\nllama3.argilla_chat\n{\n \"completion\": [\n {\"role\": \"user\", \"content\": \"...\"},\n {\"role\": \"assistant\", \"content\": \"...\"}\n ]\n}\n\n\nllama3.intel\n{\n \"system\": \"...\", // optional\n \"question\": \"...\",\n \"completion\": \"...\"\n}\n\n\nllama3.prompt_pairs\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"completion\": \"...\"\n}\n\n\nllama3.ultra\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"completion\": \"...\"\n}\n\n\nuser_defined.default\nFor custom behaviors,\nrl: kto\ndatasets:\n - path: ...\n split: train\n type:\n field_prompt: \"prompt\"\n field_system: \"system\"\n field_completion: \"completion\"\n field_label: \"label\"\n prompt_format: \"{prompt}\"\n completion_format: \"{completion}\"\nThe input format is a simple JSON input with customizable fields based on the above config.\n{\n \"system\": \"...\", // optional\n \"prompt\": \"...\",\n \"completion\": \"...\",\n \"label\": \"...\"\n}\n\n\n\nGRPO\n\n\n\n\n\n\nTip\n\n\n\nCheck out our GRPO cookbook. For a comprehensive guide covering async training, custom rewards, importance sampling, and scaling, see the GRPO deep dive.\n\n\nIn the latest GRPO implementation, vLLM is used to significantly speedup trajectory generation during training. In this example, we’re using 4 GPUs - 2 for training, and 2 for vLLM:\n\n\n\n\n\n\nImportant\n\n\n\nMake sure you’ve installed the correct version of vLLM by including it as an extra when installing axolotl, e.g. pip install axolotl[vllm].\n\n\nbase_model: Qwen/Qwen2.5-1.5B-Instruct\n\nvllm:\n host: 0.0.0.0\n port: 8000\n tensor_parallel_size: 2\n gpu_memory_utilization: 0.85\n dtype: auto\n # max_model_len: # you may find it useful to set the vLLM model context length if you know this beforehand\n\nrl: grpo\ntrl:\n use_vllm: true\n vllm_server_host: 0.0.0.0\n vllm_server_port: 8000\n vllm_server_timeout: 300\nCUDA_VISIBLE_DEVICES=2,3 axolotl vllm-serve grpo.yaml\nYour vLLM instance will now attempt to spin up, and it’s time to kick off training utilizing our remaining two GPUs. In another terminal, execute:\nCUDA_VISIBLE_DEVICES=0,1 axolotl train grpo.yaml --num-processes 2\n\n\n\n\n\n\nNote\n\n\n\nDue to TRL’s implementation with vLLM, the vLLM instance must use the last N GPUs instead of the first N GPUs. This is why in the example above, we use CUDA_VISIBLE_DEVICES=2,3 for the vLLM instance.\n\n\n\nReward functions\nGRPO uses custom reward functions and transformations. Please have them ready locally.\nFor example, to load OpenAI’s GSM8K and use a random reward for completions:\n# rewards.py\nimport random\n\ndef rand_reward_func(completions, **kwargs) -> list[float]:\n return [random.uniform(0, 1) for _ in completions]\n\ndef oai_gsm8k_transform(cfg, *args, **kwargs):\n def transform_fn(example, tokenizer=None):\n label = example[\"answer\"].split(\"####\")[-1].strip().replace(\",\", \"\")\n return {\n \"prompt\": [{\"role\": \"user\", \"content\": example[\"question\"]},],\n \"answer\": label,\n }\n return transform_fn, {\"remove_columns\": [\"question\"]}\nrl: grpo\n\ntrl:\n beta: 0.001\n max_completion_length: 256\n use_vllm: True\n num_generations: 4\n reward_funcs: [\"rewards.rand_reward_func\"] # format: '{file_name}.{fn_name}'\n reward_weights: [1.0]\ndatasets:\n - path: openai/gsm8k\n name: main\n type: rewards.oai_gsm8k_transform # format: '{file_name}.{fn_name}'\nTo see other examples of custom reward functions, please see TRL GRPO Docs.\nTo see all configs, please see TRLConfig.\n\n\nOpenEnv Rollout Functions\nGRPO supports custom rollout functions for OpenEnv-style environments, enabling interactive tasks like web browsing, code execution, or tool use. This allows you to implement custom generation logic that interacts with external environments.\nFor example, to implement a simple math-solving environment with step-by-step verification:\n# math_env.py\nimport re\n\ndef math_solver_rollout(model, processing_class, prompts, generation_config=None):\n \"\"\"\n Custom rollout function that generates step-by-step math solutions.\n\n Args:\n model: The language model\n processing_class: The tokenizer/processing_class\n prompts: List of prompt dicts (with 'messages' key for chat format)\n generation_config: Optional generation configuration\n\n Returns:\n List of completion strings\n \"\"\"\n completions = []\n\n for prompt in prompts:\n # Apply chat template to prompt\n messages = prompt.get(\"messages\", [])\n formatted_prompt = processing_class.apply_chat_template(\n messages, processing_class=False, add_generation_prompt=True\n )\n\n # Generate step-by-step solution\n full_response = \"\"\n for step in range(5): # Max 5 reasoning steps\n current_input = formatted_prompt + full_response + \"\\nNext step:\"\n inputs = processing_class(current_input, return_tensors=\"pt\").to(model.device)\n\n outputs = model.generate(\n **inputs,\n max_new_tokens=100,\n generation_config=generation_config,\n )\n step_text = processing_class.decode(\n outputs[0][inputs.input_ids.shape[1]:],\n skip_special_tokens=True\n )\n\n # Check if solution is complete\n if \"FINAL ANSWER:\" in step_text:\n full_response += step_text\n break\n full_response += step_text + \"\\n\"\n\n completions.append(full_response)\n\n return completions\n\ndef math_reward(prompts, completions, answers, **kwargs):\n \"\"\"Reward function that checks mathematical correctness\"\"\"\n rewards = []\n for completion, correct_answer in zip(completions, answers):\n # Extract predicted answer\n match = re.search(r\"FINAL ANSWER:\\s*(.+)\", completion)\n predicted = match.group(1).strip() if match else \"\"\n\n # Compare with correct answer\n reward = 1.0 if predicted == str(correct_answer) else 0.0\n rewards.append(reward)\n\n return rewards\n\ndef math_transform(cfg, *args, **kwargs):\n \"\"\"Transform dataset to GRPO format with answer field\"\"\"\n def transform_fn(example, processing_class=None):\n return {\n \"prompt\": [{\"role\": \"user\", \"content\": example[\"question\"]}],\n \"answer\": str(example[\"answer\"]),\n }\n return transform_fn, {\"remove_columns\": [\"question\"]}\nrl: grpo\n\ntrl:\n beta: 0.001\n max_completion_length: 512\n num_generations: 4\n rollout_func: \"math_env.math_solver_rollout\" # Custom rollout function\n reward_funcs: [\"math_env.math_reward\"]\n reward_weights: [1.0]\n\ndatasets:\n - path: openai/gsm8k\n name: main\n type: math_env.math_transform\nThe rollout_func parameter accepts a fully qualified name (e.g., module_name.function_name) that points to a callable function in your local directory. The function receives:\n\nmodel: The language model\nprocessing_class: The tokenizer/processing class\nprompts: List of prompt dictionaries\ngeneration_config (optional): Generation configuration\n\nAnd should return a list of completion strings.\nFor more OpenEnv examples, see TRL OpenEnv Documentation.\n\n\nGRPO with DAPO/Dr. GRPO loss\nThe DAPO paper and subsequently Dr. GRPO paper proposed an alternative loss function for GRPO to remediate the penalty in longer responses.\ntrl:\n loss_type: dr_grpo\n # Normalizes loss based on max completion length (default: 256)\n max_completion_length:\nFor more information, see GRPO docs.\n\n\nAsync GRPO\nAsync GRPO overlaps vLLM generation with training by producing rollouts in a background thread. While the model trains on the current batch, the next batch is already being generated. This can significantly reduce wall-clock time per step.\ntrl:\n use_data_producer: true # Enable data producer protocol\n use_vllm: true\n async_prefetch: true # Generate rollouts in background thread\n prefetch_depth: 1 # Number of rollouts to prefetch\n vllm_sync_interval: 2 # Sync weights to vLLM every N steps\n\n\n\n\n\n\nNote\n\n\n\nBecause the background thread generates completions with slightly stale model weights, async GRPO uses importance sampling correction to account for the distribution shift. This is controlled by vllm_importance_sampling_correction: true (default when async is enabled).\n\n\n\nvLLM LoRA Sync\nBy default, weight sync to vLLM merges the LoRA adapter into the base model and broadcasts all parameters via NCCL. LoRA sync is a faster alternative that saves only the adapter weights to the filesystem and has vLLM load them natively using Punica kernels.\nadapter: lora\nlora_r: 32\nlora_alpha: 64\nlora_target_linear: true\n\ntrl:\n vllm_lora_sync: true # Enable native LoRA sync\nWhen vllm_lora_sync: true is set, axolotl automatically selects the LoRA-aware vLLM serve module. Start vLLM as usual:\nCUDA_VISIBLE_DEVICES=0 axolotl vllm-serve config.yaml\nThen start training on a separate GPU:\nCUDA_VISIBLE_DEVICES=1 axolotl train config.yaml\n\n\n\n\n\n\nTip\n\n\n\nLoRA sync is especially beneficial with multi-GPU training (FSDP/DeepSpeed), where NCCL merge-sync can cause GPU contention with vLLM generation.\n\n\n\n\nStreaming Partial Batch\nInstead of scoring the entire batch at once, streaming mode scores one prompt group at a time. This enables finer-grained zero-advantage skipping and reduces peak memory usage during scoring.\ntrl:\n streaming_partial_batch: true\n\n\nImportance Sampling Correction\nWhen using async prefetch, completions are generated from a slightly older version of the model. Importance sampling (IS) correction adjusts the policy gradient to account for this distribution shift.\ntrl:\n vllm_importance_sampling_correction: true # Enable IS correction\n importance_sampling_level: token # 'token' or 'sequence'\n off_policy_mask_threshold: 0.5 # Mask sequences with IS ratio below this\n\nimportance_sampling_level: token applies per-token IS ratios (recommended with Liger kernel)\nimportance_sampling_level: sequence applies per-sequence IS ratios\noff_policy_mask_threshold masks out sequences where the IS ratio indicates they are too far off-policy\n\n\n\nReplay Buffer\nThe replay buffer caches rollout groups that had learning signal (non-zero reward variance) and uses them to replace zero-signal groups in later batches.\ntrl:\n replay_buffer_size: 100 # Max cached groups (0 = disabled)\n replay_recompute_logps: true # Recompute log-probs for replayed data (recommended)\n\n\n\n\n\n\nNote\n\n\n\nWhen replay_recompute_logps: true (default), old log-probabilities are recomputed using the current model weights. This fixes the IS mismatch that would otherwise occur when replaying stale data.\n\n\n\n\nDeferred Re-rolling\nFailed prompts (where the model produces zero reward for all generations) are buffered and re-injected into later batches when the model may be better equipped to solve them.\ntrl:\n reroll_start_fraction: 0.5 # Start re-rolling after 50% of training\n reroll_max_groups: 1 # Max groups to replace per batch\n\n\nZero-Advantage Batch Skipping\nWhen all advantages in a micro-batch are zero (no learning signal), the forward/backward pass is skipped entirely. This is enabled by default and logged as skipped_zero_adv_batches=1.\ntrl:\n skip_zero_advantage_batches: true # default\n\n\nParallel Reward Workers\nReward functions that use signal.alarm() (e.g., math_verify) must run in the main thread. Parallel reward workers use subprocesses to work around this limitation while enabling concurrent reward computation.\ntrl:\n reward_num_workers: 4 # Number of subprocess workers (1 = no parallelism)\n\n\nFull Async GRPO Example\nbase_model: Qwen/Qwen2.5-1.5B-Instruct\n\nvllm:\n host: 0.0.0.0\n port: 8000\n gpu_memory_utilization: 0.35\n dtype: auto\n\nadapter: lora\nlora_r: 32\nlora_alpha: 64\nlora_target_linear: true\n\nrl: grpo\ntrl:\n use_data_producer: true\n use_vllm: true\n async_prefetch: true\n prefetch_depth: 1\n vllm_sync_interval: 2\n vllm_lora_sync: true\n streaming_partial_batch: true\n vllm_importance_sampling_correction: true\n off_policy_mask_threshold: 0.5\n importance_sampling_level: token\n num_generations: 8\n max_completion_length: 512\n reward_funcs:\n - rewards.accuracy_reward\n reroll_start_fraction: 0.5\n replay_buffer_size: 100\n reward_num_workers: 4\n skip_zero_advantage_batches: true\n\ndatasets:\n - path: AI-MO/NuminaMath-TIR\n type: rewards.prompt_transform\n split: train\n\ngradient_accumulation_steps: 4\nmicro_batch_size: 2\nmax_steps: 500\nlearning_rate: 1e-5\nbf16: true\ngradient_checkpointing: true\n# Terminal 1: Start vLLM on GPU 0\nCUDA_VISIBLE_DEVICES=0 axolotl vllm-serve config.yaml\n\n# Terminal 2: Train on GPU 1\nCUDA_VISIBLE_DEVICES=1 axolotl train config.yaml\n\n\nMulti-GPU Async GRPO\nAsync GRPO supports FSDP and DeepSpeed ZeRO-3 for multi-GPU training. vLLM runs on one GPU while training is distributed across the remaining GPUs.\nFSDP:\nfsdp:\n - full_shard\n - auto_wrap\nfsdp_config:\n fsdp_transformer_layer_cls_to_wrap: Qwen2DecoderLayer\ngradient_checkpointing_kwargs:\n use_reentrant: false\nDeepSpeed ZeRO-3:\ndeepspeed: deepspeed_configs/zero3_bf16.json\ngradient_checkpointing_kwargs:\n use_reentrant: true # Required for ZeRO-3\n# Terminal 1: Start vLLM on GPU 0\nCUDA_VISIBLE_DEVICES=0 axolotl vllm-serve config.yaml\n\n# Terminal 2: Train on GPUs 0,1\nCUDA_VISIBLE_DEVICES=0,1 axolotl train config.yaml\n\n\n\n\n\n\nImportant\n\n\n\nWith multi-GPU async prefetch, only rank 0 generates completions in the background thread. Results are broadcast to all ranks on the main thread. This avoids FSDP/DeepSpeed collective deadlocks from unsynchronized background threads.\n\n\n\n\n\n\nGDPO\nGDPO (Group Reward-Decoupled Policy Optimization) extends GRPO for multi-reward training. It addresses the reward advantage collapse problem by normalizing each reward function independently before combining them.\n\n\n\n\n\n\nTip\n\n\n\nUse GDPO when training with multiple reward functions. For single reward, GRPO and GDPO produce equivalent results.\n\n\nPaper: https://arxiv.org/pdf/2501.05242\nGDPO uses TRL’s native multi_objective_aggregation parameter under the hood. When you set rl: gdpo, axolotl automatically configures TRL to use normalize_then_sum aggregation.\nbase_model: Qwen/Qwen2.5-1.5B-Instruct\n\nvllm:\n host: 0.0.0.0\n port: 8000\n tensor_parallel_size: 2\n gpu_memory_utilization: 0.85\n\nrl: gdpo\n\ntrl:\n beta: 0.001\n max_completion_length: 256\n use_vllm: true\n num_generations: 4\n reward_funcs:\n - rewards.format_reward\n - rewards.correctness_reward\n reward_weights: [1.0, 2.0]\n\ndatasets:\n - path: openai/gsm8k\n name: main\n type: rewards.oai_gsm8k_transform\nYou can also use GRPO with explicit aggregation control:\nrl: grpo\ntrl:\n multi_objective_aggregation: normalize_then_sum # GDPO behavior\n # or: sum_then_normalize # Default GRPO behavior\n\nGDPO vs GRPO\n\n\n\n\n\n\n\n\nAspect\nGRPO\nGDPO\n\n\n\n\nAggregation\nsum_then_normalize\nnormalize_then_sum\n\n\nMulti-reward\nMay collapse advantages\nPreserves reward signals\n\n\nSingle reward\nStandard behavior\nEquivalent to GRPO\n\n\n\n\n\nWhy GDPO?\nWhen using multiple rewards with GRPO, different reward combinations can produce identical advantages:\n# Example: format + correctness rewards\n[format=0, correct=3] → sum=3\n[format=1, correct=2] → sum=3 ← GRPO sees these as equal!\n[format=2, correct=1] → sum=3\n[format=3, correct=0] → sum=3\nGDPO normalizes each reward independently, preserving their relative differences.\n\n\nReward Functions\nGDPO uses the same reward function format as GRPO:\n# rewards.py\ndef format_reward(completions, **kwargs) -> list[float]:\n return [1.0 if len(c) > 10 else 0.0 for c in completions]\n\ndef correctness_reward(completions, answers, **kwargs) -> list[float]:\n rewards = []\n for completion, answer in zip(completions, answers):\n # Your scoring logic here\n rewards.append(score)\n return rewards\n\n\nSequence Parallelism\nGDPO supports sequence parallelism for long-context training:\nrl: gdpo\ncontext_parallel_size: 2\n\n\n\nSimPO\nSimPO uses CPOTrainer but with alternative loss function.\nrl: simpo\nrl_beta: 0.1 # default in CPOTrainer\ncpo_alpha: 1.0 # default in CPOTrainer\nsimpo_gamma: 0.5 # default in CPOTrainer\nThis method uses the same dataset format as DPO.\n\n\nEBFT\n\n\n\n\n\n\nTip\n\n\n\nFor a detailed guide on EBFT modes, feature extraction, and configuration, see the EBFT guide.\n\n\nEBFT (Energy-Based Fine-Tuning) fine-tunes language models by optimizing a feature-matching loss rather than relying on external reward functions. A frozen copy of the model extracts embeddings from both generated and ground-truth completions, and the generator is updated via REINFORCE to match the ground-truth feature moments.\nPaper: “Matching Features, Not Tokens: Energy-Based Fine-Tuning of Language Models” (Jelassi et al., 2026)\nKey advantages:\n\nNo reward model or verifier required — works on any (prompt, completion) data\nApplicable to non-verifiable tasks (code, translation, creative writing)\nOperates on model rollouts (not teacher forcing), reducing distribution shift\n\nEBFT supports two modes:\n\nStructured mode: For QA/instruction data with prompt + completion pairs. Uses vLLM for generation (like GRPO).\nStrided mode: For unstructured text without prompt/completion splits. Uses strided block-parallel generation with flex_attention — no vLLM needed.\n\n\nStructured Mode\nbase_model: Qwen/Qwen3-4B\n\nrl: ebft\n\nebft:\n feature_layers: [0.25, 0.5, 0.75] # Extract features at 25%, 50%, 75% depth\n embed_method: last_token\n use_whitening: false\n alignment_coef: 1.0 # Cosine similarity reward weight\n diversity_coef: 1.0 # Pairwise dot product penalty\n ce_coef: 0.0 # Cross-entropy on GT tokens (0 = off)\n\ntrl:\n num_generations: 4\n max_completion_length: 256\n temperature: 0.7\n use_vllm: true\n vllm_server_host: 0.0.0.0\n vllm_server_port: 8000\n vllm_lora_sync: true # LoRA adapter sync (recommended)\n vllm_sync_interval: 3\n use_data_producer: true\n async_prefetch: true # Set false for sync mode\n scale_rewards: true\n loss_type: grpo\n epsilon: 0.2\n\nvllm:\n gpu_memory_utilization: 0.5\n max_model_len: 2048\n\ndatasets:\n - path: nvidia/OpenCodeInstruct\n type: ebft_opencode.transform\n split: train[:500]\n\nadapter: lora\nlora_r: 16\nlora_alpha: 32\nlora_target_linear: true\n# Terminal 1: Start vLLM\nCUDA_VISIBLE_DEVICES=0 axolotl vllm-serve config.yaml\n\n# Terminal 2: Train\nCUDA_VISIBLE_DEVICES=1 axolotl train config.yaml\n\n\nStrided Mode\nFor unstructured text (raw code, prose). No vLLM needed — runs on a single GPU.\nbase_model: meta-llama/Llama-3.2-1B\n\nrl: ebft\n\nebft:\n mode: strided\n stride: 8\n context_length: 8\n generate_max_len: 8\n n_samples_per_prompt: 4\n temperature: 0.6\n feature_layers: [0.25, 0.5, 0.75]\n embed_method: last_token\n use_whitening: true\n alignment_coef: 1.0\n diversity_coef: 1.0\n rl_coef: 1.0\n ce_coef: 0.03\n advantage_estimator: rloo\n\ndatasets:\n - path: nvidia/OpenCodeInstruct\n type: ebft_strided_structured.transform\n split: train[:1%]\n\nflash_attention: false\nflex_attention: true # Strided mode uses flex_attention\ngradient_checkpointing: true\ngradient_checkpointing_kwargs:\n use_reentrant: true # Required for flex_attention\nCUDA_VISIBLE_DEVICES=0 axolotl train config.yaml\n\n\n\n\n\n\nTip\n\n\n\nSee examples/ebft/ for complete example configs covering Llama 1B/3B/8B and Qwen3 4B/8B models in both modes.\n\n\n\n\nEBFT Configuration Reference\n\n\n\n\n\n\n\n\nParameter\nDefault\nDescription\n\n\n\n\nebft.feature_layers\n[0.25, 0.5, 0.75]\nLayer depths for feature extraction (fractional)\n\n\nebft.embed_method\nlast_token\nFeature pooling: last_token, mean_pooling, concat\n\n\nebft.use_whitening\nfalse\nSVD whitening of feature dimensions\n\n\nebft.alignment_coef\n1.0\nCosine similarity reward weight\n\n\nebft.diversity_coef\n1.0\nPairwise dot product penalty weight\n\n\nebft.ce_coef\n0.0\nCross-entropy loss on ground-truth tokens\n\n\nebft.mode\nstructured\nstructured (vLLM) or strided (no vLLM)\n\n\nebft.stride\n—\nTokens between anchor points (strided mode)\n\n\nebft.context_length\n—\nContext window per block (strided mode)\n\n\nebft.generate_max_len\n—\nTokens to generate per block (strided mode)\n\n\nebft.n_samples_per_prompt\n—\nRollouts per document (strided mode)\n\n\nebft.advantage_estimator\ngrpo\ngrpo or rloo (strided mode)\n\n\n\n\n\n\nNeMo Gym Integration\nNeMo Gym provides 50+ verified RL environments (math, coding, tool-use, reasoning) with deterministic reward signals. The axolotl integration supports both single-turn (call /verify after generation) and multi-turn (agent-based tool execution via /run).\n\nSingle-Turn (Simplest)\nFor environments that only need answer verification (math, coding challenges). No agent server needed — the reward function calls /verify directly on the resource server.\nbase_model: Qwen/Qwen2.5-0.5B-Instruct\n\nrl: grpo\nchat_template: tokenizer_default\n\ntrl:\n use_vllm: false # Colocate mode (single GPU)\n num_generations: 4\n max_completion_length: 128\n temperature: 0.9\n reward_funcs:\n - axolotl.integrations.nemo_gym.rewards.reward_nemo_gym_verify\n\nplugins:\n - axolotl.integrations.nemo_gym.NemoGymPlugin\n\nnemo_gym_enabled: true\nnemo_gym_dir: ~/Gym\nnemo_gym_auto_start: false\nnemo_gym_head_port: 11000\nnemo_gym_datasets:\n - path: resources_servers/reasoning_gym/data/train_basic_arithmetic.jsonl\n server_name: reasoning_gym\n\ndatasets:\n - path: ~/Gym/resources_servers/reasoning_gym/data/train_basic_arithmetic.jsonl\n type: chat_template\n field_messages: responses_create_params.input\n message_field_content: content\n message_field_role: role\n# Terminal 1: Start NeMo Gym resource server\ncd ~/Gym && .venv/bin/ng_run \\\n \"+config_paths=[resources_servers/reasoning_gym/configs/resources_only.yaml]\" \\\n \"+skip_venv_if_present=true\"\n\n# Terminal 2: Train\nCUDA_VISIBLE_DEVICES=0 axolotl train config.yaml\n\n\n\n\n\n\nNote\n\n\n\nnemo_gym_datasets.path is relative to nemo_gym_dir. Don’t use absolute paths or they will be double-joined.\n\n\n\n\nMulti-Turn with Async GRPO (Recommended)\nFor environments with tool-use (weather, search, databases). An agent server orchestrates multi-turn interactions: generate → parse tool calls → execute tools → feed results back → repeat until done.\nbase_model: Qwen/Qwen3-0.6B\n\nrl: grpo\nchat_template: tokenizer_default\n\nadapter: lora\nlora_r: 16\nlora_alpha: 32\nlora_target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj]\n\ntrl:\n use_vllm: true\n vllm_mode: server\n vllm_server_host: localhost\n vllm_server_port: 8000\n vllm_lora_sync: true\n vllm_sync_interval: 5\n use_data_producer: true\n async_prefetch: true # 3x speedup\n num_generations: 4\n max_completion_length: 512\n temperature: 0.8\n reward_funcs:\n - axolotl.integrations.nemo_gym.rewards.reward_env\n\nplugins:\n - axolotl.integrations.nemo_gym.NemoGymPlugin\n\nnemo_gym_enabled: true\nnemo_gym_auto_start: false\nnemo_gym_head_port: 11000\nnemo_gym_multi_turn: true\nnemo_gym_verify_timeout: 120\nnemo_gym_datasets:\n - path: resources_servers/example_single_tool_call/data/weather_tool_calling.jsonl\n server_name: example_single_tool_call\n\ndatasets:\n - path: ~/Gym/resources_servers/example_single_tool_call/data/weather_tool_calling.jsonl\n type: chat_template\n field_messages: responses_create_params.input\n message_field_content: content\n message_field_role: role\n\nvllm:\n gpu_memory_utilization: 0.85\n max_model_len: 2048\nMulti-turn requires three services running:\n# Terminal 1: vLLM with LoRA + tool calling\nVLLM_ALLOW_RUNTIME_LORA_UPDATING=1 CUDA_VISIBLE_DEVICES=0 \\\n python -m vllm.entrypoints.openai.api_server \\\n --model Qwen/Qwen3-0.6B --max-model-len 2048 \\\n --gpu-memory-utilization 0.85 \\\n --enable-lora --max-lora-rank 64 \\\n --enable-auto-tool-choice --tool-call-parser hermes\n\n# Terminal 2: NeMo Gym servers (resource + model proxy + agent)\ncd ~/Gym && .venv/bin/ng_run \\\n \"+config_paths=[configs/axolotl_tool_calling.yaml]\" \\\n \"+skip_venv_if_present=true\"\n\n# Terminal 3: Training\nCUDA_VISIBLE_DEVICES=1 axolotl train config.yaml\n\n\n\n\n\n\nImportant\n\n\n\nMulti-turn requires a NeMo Gym agent config YAML that defines three components: a resource server (tools + /verify), a model server proxy (forwards to your vLLM), and an agent server (orchestrates /run). See the NeMo Gym README for agent config format.\n\n\n\n\nNeMo Gym Prerequisites\n# Clone and set up NeMo Gym\ngit clone https://github.com/NVIDIA-NeMo/Gym.git ~/Gym\ncd ~/Gym\nuv venv --python 3.12 && source .venv/bin/activate && uv sync\n\n# Fix pycosat build (GCC 13+)\nCFLAGS=\"\" uv pip install pycosat --python .venv/bin/python --no-build-isolation\n\n\nNeMo Gym Configuration Reference\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nnemo_gym_enabled\nbool\n—\nEnable the NeMo Gym integration\n\n\nnemo_gym_dir\nstr\n~/Gym\nPath to NeMo Gym repo\n\n\nnemo_gym_auto_start\nbool\ntrue\nAuto-start resource servers\n\n\nnemo_gym_head_port\nint\n11000\nHead server port\n\n\nnemo_gym_multi_turn\nbool\nfalse\nEnable multi-turn via agent /run\n\n\nnemo_gym_verify_timeout\nint\n30\nPer-request timeout (seconds)\n\n\nnemo_gym_datasets\nlist\nrequired\nDataset configs with path and server_name\n\n\n\n\n\nReward Functions\n\n\n\n\n\n\n\n\nFunction\nMode\nDescription\n\n\n\n\naxolotl.integrations.nemo_gym.rewards.reward_nemo_gym_verify\nSingle-turn\nCalls /verify, returns binary reward\n\n\naxolotl.integrations.nemo_gym.rewards.reward_env\nMulti-turn\nPassthrough reward from agent /run\n\n\n\n\n\n\nUsing local dataset files\ndatasets:\n - ds_type: json\n data_files:\n - orca_rlhf.jsonl\n split: train\n type: chatml.intel\n\n\nTRL auto-unwrapping for PEFT\nTRL supports auto-unwrapping PEFT models for RL training paradigms which rely on a reference model. This significantly reduces memory pressure as an additional refreference model does not need to be loaded, and reference model log-probabilities can be obtained by disabling PEFT adapters. This is enabled by default. To turn it off, pass the following config:\n# load ref model when adapter training.\nrl_adapter_ref_model: true", "crumbs": [ "How To Guides", "RLHF (Beta)" diff --git a/sitemap.xml b/sitemap.xml index fd3957168..9bf11c8e2 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -2,942 +2,982 @@ https://docs.axolotl.ai/FAQS.html - 2026-04-01T22:21:40.928Z + 2026-04-02T12:02:09.344Z https://docs.axolotl.ai/docs/dataset-formats/template_free.html - 2026-04-01T22:21:40.930Z + 2026-04-02T12:02:09.348Z https://docs.axolotl.ai/docs/dataset-formats/conversation.html - 2026-04-01T22:21:40.930Z + 2026-04-02T12:02:09.347Z https://docs.axolotl.ai/docs/dataset-formats/pretraining.html - 2026-04-01T22:21:40.930Z + 2026-04-02T12:02:09.348Z https://docs.axolotl.ai/docs/dataset-formats/index.html - 2026-04-01T22:21:40.930Z + 2026-04-02T12:02:09.348Z https://docs.axolotl.ai/docs/api/cli.args.html - 2026-04-01T22:25:08.926Z + 2026-04-02T12:05:41.667Z https://docs.axolotl.ai/docs/api/prompt_strategies.orcamini.html - 2026-04-01T22:25:09.428Z + 2026-04-02T12:05:42.175Z https://docs.axolotl.ai/docs/api/cli.preprocess.html - 2026-04-01T22:25:09.018Z + 2026-04-02T12:05:41.762Z https://docs.axolotl.ai/docs/api/utils.collators.core.html - 2026-04-01T22:25:10.300Z + 2026-04-02T12:05:43.042Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.llama3.html - 2026-04-01T22:25:09.463Z + 2026-04-02T12:05:42.210Z https://docs.axolotl.ai/docs/api/utils.schemas.enums.html - 2026-04-01T22:25:10.047Z + 2026-04-02T12:05:42.795Z https://docs.axolotl.ai/docs/api/utils.lora.html - 2026-04-01T22:25:09.775Z + 2026-04-02T12:05:42.522Z https://docs.axolotl.ai/docs/api/common.datasets.html - 2026-04-01T22:25:10.297Z + 2026-04-02T12:05:43.038Z https://docs.axolotl.ai/docs/api/monkeypatch.relora.html - 2026-04-01T22:25:09.642Z + 2026-04-02T12:05:42.389Z https://docs.axolotl.ai/docs/api/core.builders.base.html - 2026-04-01T22:25:08.754Z + 2026-04-02T12:05:41.500Z https://docs.axolotl.ai/docs/api/prompt_strategies.input_output.html - 2026-04-01T22:25:09.409Z + 2026-04-02T12:05:42.156Z https://docs.axolotl.ai/docs/api/integrations.lm_eval.args.html - 2026-04-01T22:25:10.268Z + 2026-04-02T12:05:43.012Z https://docs.axolotl.ai/docs/api/cli.inference.html - 2026-04-01T22:25:08.984Z + 2026-04-02T12:05:41.726Z https://docs.axolotl.ai/docs/api/monkeypatch.gradient_checkpointing.offload_disk.html - 2026-04-01T22:25:09.758Z + 2026-04-02T12:05:42.505Z https://docs.axolotl.ai/docs/api/core.datasets.chat.html - 2026-04-01T22:25:08.822Z + 2026-04-02T12:05:41.570Z https://docs.axolotl.ai/docs/api/core.chat.format.shared.html - 2026-04-01T22:25:08.816Z + 2026-04-02T12:05:41.563Z https://docs.axolotl.ai/docs/api/logging_config.html - 2026-04-01T22:25:08.746Z + 2026-04-02T12:05:41.492Z https://docs.axolotl.ai/docs/api/prompt_strategies.chat_template.html - 2026-04-01T22:25:09.334Z + 2026-04-02T12:05:42.080Z https://docs.axolotl.ai/docs/api/utils.collators.mamba.html - 2026-04-01T22:25:10.328Z + 2026-04-02T12:05:43.069Z https://docs.axolotl.ai/docs/api/cli.config.html - 2026-04-01T22:25:08.960Z + 2026-04-02T12:05:41.702Z https://docs.axolotl.ai/docs/api/loaders.model.html - 2026-04-01T22:25:09.196Z + 2026-04-02T12:05:41.944Z https://docs.axolotl.ai/docs/api/prompt_strategies.kto.chatml.html - 2026-04-01T22:25:09.503Z + 2026-04-02T12:05:42.249Z https://docs.axolotl.ai/docs/api/cli.quantize.html - 2026-04-01T22:25:09.024Z + 2026-04-02T12:05:41.768Z https://docs.axolotl.ai/docs/api/prompt_strategies.bradley_terry.llama3.html - 2026-04-01T22:25:09.535Z + 2026-04-02T12:05:42.282Z https://docs.axolotl.ai/docs/api/integrations.spectrum.args.html - 2026-04-01T22:25:10.274Z + 2026-04-02T12:05:43.016Z https://docs.axolotl.ai/docs/api/prompt_strategies.messages.chat.html - 2026-04-01T22:25:09.442Z + 2026-04-02T12:05:42.188Z https://docs.axolotl.ai/docs/api/utils.callbacks.perplexity.html - 2026-04-01T22:25:10.393Z + 2026-04-02T12:05:43.134Z https://docs.axolotl.ai/docs/api/monkeypatch.lora_kernels.html - 2026-04-01T22:25:09.677Z + 2026-04-02T12:05:42.424Z https://docs.axolotl.ai/docs/api/monkeypatch.data.batch_dataset_fetcher.html - 2026-04-01T22:25:09.719Z + 2026-04-02T12:05:42.466Z https://docs.axolotl.ai/docs/api/loaders.patch_manager.html - 2026-04-01T22:25:09.239Z + 2026-04-02T12:05:41.986Z https://docs.axolotl.ai/docs/api/utils.model_shard_quant.html - 2026-04-01T22:25:09.782Z + 2026-04-02T12:05:42.529Z https://docs.axolotl.ai/docs/api/utils.schemas.multimodal.html - 2026-04-01T22:25:10.016Z + 2026-04-02T12:05:42.764Z https://docs.axolotl.ai/docs/api/utils.callbacks.profiler.html - 2026-04-01T22:25:10.398Z + 2026-04-02T12:05:43.139Z https://docs.axolotl.ai/docs/api/convert.html - 2026-04-01T22:25:08.681Z + 2026-04-02T12:05:41.427Z https://docs.axolotl.ai/docs/api/cli.utils.html - 2026-04-01T22:25:09.047Z + 2026-04-02T12:05:41.791Z https://docs.axolotl.ai/docs/api/kernels.lora.html - 2026-04-01T22:25:09.583Z + 2026-04-02T12:05:42.329Z https://docs.axolotl.ai/docs/api/monkeypatch.utils.html - 2026-04-01T22:25:09.685Z + 2026-04-02T12:05:42.432Z https://docs.axolotl.ai/docs/api/common.const.html - 2026-04-01T22:25:10.278Z + 2026-04-02T12:05:43.020Z https://docs.axolotl.ai/docs/api/utils.freeze.html - 2026-04-01T22:25:09.796Z + 2026-04-02T12:05:42.543Z https://docs.axolotl.ai/docs/api/utils.schemas.utils.html - 2026-04-01T22:25:10.054Z + 2026-04-02T12:05:42.802Z https://docs.axolotl.ai/docs/api/utils.callbacks.qat.html - 2026-04-01T22:25:10.417Z + 2026-04-02T12:05:43.158Z https://docs.axolotl.ai/docs/api/utils.data.sft.html - 2026-04-01T22:25:09.908Z + 2026-04-02T12:05:42.655Z https://docs.axolotl.ai/docs/api/monkeypatch.llama_attn_hijack_xformers.html - 2026-04-01T22:25:09.634Z + 2026-04-02T12:05:42.381Z https://docs.axolotl.ai/docs/api/core.trainers.grpo.sampler.html - 2026-04-01T22:25:09.182Z + 2026-04-02T12:05:41.930Z https://docs.axolotl.ai/docs/api/core.chat.messages.html - 2026-04-01T22:25:08.810Z + 2026-04-02T12:05:41.558Z https://docs.axolotl.ai/docs/api/core.trainers.mamba.html - 2026-04-01T22:25:09.142Z + 2026-04-02T12:05:41.886Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.passthrough.html - 2026-04-01T22:25:09.482Z + 2026-04-02T12:05:42.229Z https://docs.axolotl.ai/docs/api/kernels.swiglu.html - 2026-04-01T22:25:09.608Z + 2026-04-02T12:05:42.355Z https://docs.axolotl.ai/docs/api/prompt_strategies.pygmalion.html - 2026-04-01T22:25:09.436Z + 2026-04-02T12:05:42.183Z https://docs.axolotl.ai/docs/api/utils.schemas.peft.html - 2026-04-01T22:25:10.005Z + 2026-04-02T12:05:42.753Z https://docs.axolotl.ai/docs/api/utils.schemas.trl.html - 2026-04-01T22:25:10.009Z + 2026-04-02T12:05:42.757Z https://docs.axolotl.ai/docs/api/prompt_strategies.completion.html - 2026-04-01T22:25:09.402Z + 2026-04-02T12:05:42.148Z https://docs.axolotl.ai/docs/api/cli.vllm_serve.html - 2026-04-01T22:25:09.033Z + 2026-04-02T12:05:41.777Z https://docs.axolotl.ai/docs/api/utils.trainer.html - 2026-04-01T22:25:09.818Z + 2026-04-02T12:05:42.565Z https://docs.axolotl.ai/docs/api/utils.ctx_managers.sequence_parallel.html - 2026-04-01T22:25:09.290Z + 2026-04-02T12:05:42.037Z https://docs.axolotl.ai/docs/api/core.training_args.html - 2026-04-01T22:25:08.781Z + 2026-04-02T12:05:41.528Z https://docs.axolotl.ai/docs/api/evaluate.html - 2026-04-01T22:25:08.656Z + 2026-04-02T12:05:41.402Z https://docs.axolotl.ai/docs/api/utils.callbacks.comet_.html - 2026-04-01T22:25:10.409Z + 2026-04-02T12:05:43.150Z https://docs.axolotl.ai/docs/api/loaders.tokenizer.html - 2026-04-01T22:25:09.207Z + 2026-04-02T12:05:41.955Z https://docs.axolotl.ai/docs/api/monkeypatch.llama_attn_hijack_flash.html - 2026-04-01T22:25:09.632Z + 2026-04-02T12:05:42.379Z https://docs.axolotl.ai/docs/api/cli.cloud.modal_.html - 2026-04-01T22:25:09.045Z + 2026-04-02T12:05:41.789Z https://docs.axolotl.ai/docs/api/prompt_strategies.stepwise_supervised.html - 2026-04-01T22:25:09.415Z + 2026-04-02T12:05:42.161Z https://docs.axolotl.ai/docs/api/monkeypatch.btlm_attn_hijack_flash.html - 2026-04-01T22:25:09.686Z + 2026-04-02T12:05:42.433Z https://docs.axolotl.ai/docs/api/core.chat.format.llama3x.html - 2026-04-01T22:25:08.814Z + 2026-04-02T12:05:41.561Z https://docs.axolotl.ai/docs/api/utils.quantization.html - 2026-04-01T22:25:09.932Z + 2026-04-02T12:05:42.680Z https://docs.axolotl.ai/docs/api/monkeypatch.unsloth_.html - 2026-04-01T22:25:09.708Z + 2026-04-02T12:05:42.455Z https://docs.axolotl.ai/docs/api/prompt_strategies.orpo.chat_template.html - 2026-04-01T22:25:09.530Z + 2026-04-02T12:05:42.277Z https://docs.axolotl.ai/docs/api/cli.art.html - 2026-04-01T22:25:08.930Z + 2026-04-02T12:05:41.671Z https://docs.axolotl.ai/docs/api/loaders.processor.html - 2026-04-01T22:25:09.209Z + 2026-04-02T12:05:41.957Z https://docs.axolotl.ai/docs/api/cli.merge_sharded_fsdp_weights.html - 2026-04-01T22:25:09.008Z + 2026-04-02T12:05:41.751Z https://docs.axolotl.ai/docs/api/kernels.quantize.html - 2026-04-01T22:25:09.623Z + 2026-04-02T12:05:42.370Z https://docs.axolotl.ai/docs/api/core.trainers.utils.html - 2026-04-01T22:25:09.184Z + 2026-04-02T12:05:41.932Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.chat_template.html - 2026-04-01T22:25:09.450Z + 2026-04-02T12:05:42.197Z https://docs.axolotl.ai/docs/api/cli.delinearize_llama4.html - 2026-04-01T22:25:08.966Z + 2026-04-02T12:05:41.708Z https://docs.axolotl.ai/docs/faq.html - 2026-04-01T22:21:40.930Z + 2026-04-02T12:02:09.348Z https://docs.axolotl.ai/docs/expert_quantization.html - 2026-04-01T22:21:40.930Z - - - https://docs.axolotl.ai/docs/optimizations.html - 2026-04-01T22:21:40.933Z - - - https://docs.axolotl.ai/docs/multi-gpu.html - 2026-04-01T22:21:40.933Z - - - https://docs.axolotl.ai/docs/nd_parallelism.html - 2026-04-01T22:21:40.933Z - - - https://docs.axolotl.ai/docs/mac.html - 2026-04-01T22:21:40.933Z - - - https://docs.axolotl.ai/docs/reward_modelling.html - 2026-04-01T22:21:40.934Z - - - https://docs.axolotl.ai/docs/models/ministral3.html - 2026-04-01T22:25:33.390Z - - - https://docs.axolotl.ai/docs/models/hunyuan.html - 2026-04-01T22:25:33.397Z - - - https://docs.axolotl.ai/docs/models/smolvlm2.html - 2026-04-01T22:25:33.396Z - - - https://docs.axolotl.ai/docs/models/ministral3/vision.html - 2026-04-01T22:25:33.390Z - - - https://docs.axolotl.ai/docs/models/voxtral.html - 2026-04-01T22:25:33.392Z - - - https://docs.axolotl.ai/docs/models/ministral.html - 2026-04-01T22:25:33.392Z - - - https://docs.axolotl.ai/docs/models/granite4.html - 2026-04-01T22:25:33.396Z - - - https://docs.axolotl.ai/docs/models/phi.html - 2026-04-01T22:25:33.396Z - - - https://docs.axolotl.ai/docs/models/internvl3_5.html - 2026-04-01T22:25:33.388Z - - - https://docs.axolotl.ai/docs/models/magistral/think.html - 2026-04-01T22:25:33.391Z - - - https://docs.axolotl.ai/docs/models/mistral-small.html - 2026-04-01T22:25:33.392Z - - - https://docs.axolotl.ai/docs/models/gemma3n.html - 2026-04-01T22:25:33.395Z - - - https://docs.axolotl.ai/docs/models/arcee.html - 2026-04-01T22:25:33.389Z - - - https://docs.axolotl.ai/docs/models/llama-2.html - 2026-04-01T22:25:33.394Z - - - https://docs.axolotl.ai/docs/models/llama-4.html - 2026-04-01T22:25:33.393Z - - - https://docs.axolotl.ai/docs/models/seed-oss.html - 2026-04-01T22:25:33.396Z - - - https://docs.axolotl.ai/docs/models/jamba.html - 2026-04-01T22:25:33.398Z - - - https://docs.axolotl.ai/docs/nccl.html - 2026-04-01T22:21:40.933Z - - - https://docs.axolotl.ai/docs/debugging.html - 2026-04-01T22:21:40.930Z - - - https://docs.axolotl.ai/docs/dataset_preprocessing.html - 2026-04-01T22:21:40.930Z - - - https://docs.axolotl.ai/docs/ray-integration.html - 2026-04-01T22:21:40.934Z - - - https://docs.axolotl.ai/docs/custom_integrations.html - 2026-04-01T22:21:40.930Z - - - https://docs.axolotl.ai/docs/torchao.html - 2026-04-01T22:21:40.935Z - - - https://docs.axolotl.ai/docs/lr_groups.html - 2026-04-01T22:21:40.933Z - - - https://docs.axolotl.ai/docs/streaming.html - 2026-04-01T22:21:40.935Z - - - https://docs.axolotl.ai/docs/amd_hpc.html - 2026-04-01T22:21:40.930Z - - - https://docs.axolotl.ai/docs/installation.html - 2026-04-01T22:21:40.933Z - - - https://docs.axolotl.ai/docs/inference.html - 2026-04-01T22:21:40.933Z - - - https://docs.axolotl.ai/docs/getting-started.html - 2026-04-01T22:21:40.930Z - - - https://docs.axolotl.ai/docs/telemetry.html - 2026-04-01T22:21:40.935Z - - - https://docs.axolotl.ai/src/axolotl/integrations/cut_cross_entropy/ACKNOWLEDGEMENTS.html - 2026-04-01T22:21:40.963Z - - - https://docs.axolotl.ai/index.html - 2026-04-01T22:21:40.955Z - - - https://docs.axolotl.ai/examples/colab-notebooks/colab-axolotl-example.html - 2026-04-01T22:21:40.939Z - - - https://docs.axolotl.ai/src/axolotl/integrations/LICENSE.html - 2026-04-01T22:21:40.962Z - - - https://docs.axolotl.ai/docs/batch_vs_grad.html - 2026-04-01T22:21:40.930Z - - - https://docs.axolotl.ai/docs/sequence_parallelism.html - 2026-04-01T22:21:40.935Z - - - https://docs.axolotl.ai/docs/quantize.html - 2026-04-01T22:21:40.934Z - - - https://docs.axolotl.ai/docs/docker.html - 2026-04-01T22:21:40.930Z - - - https://docs.axolotl.ai/docs/attention.html - 2026-04-01T22:21:40.930Z - - - https://docs.axolotl.ai/docs/unsloth.html - 2026-04-01T22:21:40.935Z - - - https://docs.axolotl.ai/docs/qat.html - 2026-04-01T22:21:40.933Z - - - https://docs.axolotl.ai/docs/multi-node.html - 2026-04-01T22:21:40.933Z - - - https://docs.axolotl.ai/docs/optimizers.html - 2026-04-01T22:21:40.933Z - - - https://docs.axolotl.ai/docs/config-reference.html - 2026-04-01T22:25:32.398Z - - - https://docs.axolotl.ai/docs/gradient_checkpointing.html - 2026-04-01T22:21:40.930Z - - - https://docs.axolotl.ai/docs/multipack.html - 2026-04-01T22:21:40.933Z - - - https://docs.axolotl.ai/docs/models/LiquidAI.html - 2026-04-01T22:25:33.397Z - - - https://docs.axolotl.ai/docs/models/magistral.html - 2026-04-01T22:25:33.391Z - - - https://docs.axolotl.ai/docs/models/devstral.html - 2026-04-01T22:25:33.393Z - - - https://docs.axolotl.ai/docs/models/qwen3-next.html - 2026-04-01T22:25:33.394Z - - - https://docs.axolotl.ai/docs/models/mistral.html - 2026-04-01T22:25:33.393Z - - - https://docs.axolotl.ai/docs/models/plano.html - 2026-04-01T22:25:33.387Z - - - https://docs.axolotl.ai/docs/models/olmo3.html - 2026-04-01T22:25:33.388Z - - - https://docs.axolotl.ai/docs/models/magistral/vision.html - 2026-04-01T22:25:33.391Z - - - https://docs.axolotl.ai/docs/models/mimo.html - 2026-04-01T22:25:33.387Z - - - https://docs.axolotl.ai/docs/models/index.html - 2026-04-01T22:25:33.398Z - - - https://docs.axolotl.ai/docs/models/trinity.html - 2026-04-01T22:25:33.388Z - - - https://docs.axolotl.ai/docs/models/kimi-linear.html - 2026-04-01T22:25:33.387Z - - - https://docs.axolotl.ai/docs/models/orpheus.html - 2026-04-01T22:25:33.398Z - - - https://docs.axolotl.ai/docs/models/qwen3.html - 2026-04-01T22:25:33.394Z - - - https://docs.axolotl.ai/docs/models/ministral3/think.html - 2026-04-01T22:25:33.390Z - - - https://docs.axolotl.ai/docs/models/apertus.html - 2026-04-01T22:25:33.395Z - - - https://docs.axolotl.ai/docs/models/gpt-oss.html - 2026-04-01T22:25:33.395Z - - - https://docs.axolotl.ai/docs/mixed_precision.html - 2026-04-01T22:21:40.933Z - - - https://docs.axolotl.ai/docs/lora_optims.html - 2026-04-01T22:21:40.933Z - - - https://docs.axolotl.ai/docs/dataset_loading.html - 2026-04-01T22:21:40.930Z - - - https://docs.axolotl.ai/docs/input_output.html - 2026-04-01T22:21:40.933Z - - - https://docs.axolotl.ai/docs/fsdp_qlora.html - 2026-04-01T22:21:40.930Z + 2026-04-02T12:02:09.348Z https://docs.axolotl.ai/docs/checkpoint_saving.html - 2026-04-01T22:21:40.930Z + 2026-04-02T12:02:09.347Z + + + https://docs.axolotl.ai/docs/agents/pretraining.html + 2026-04-02T12:02:09.347Z + + + https://docs.axolotl.ai/docs/agents/grpo.html + 2026-04-02T12:02:09.347Z + + + https://docs.axolotl.ai/docs/agents/sft.html + 2026-04-02T12:02:09.347Z + + + https://docs.axolotl.ai/docs/multi-gpu.html + 2026-04-02T12:02:09.350Z + + + https://docs.axolotl.ai/docs/nd_parallelism.html + 2026-04-02T12:02:09.351Z + + + https://docs.axolotl.ai/docs/mac.html + 2026-04-02T12:02:09.350Z + + + https://docs.axolotl.ai/docs/reward_modelling.html + 2026-04-02T12:02:09.351Z + + + https://docs.axolotl.ai/docs/models/ministral3.html + 2026-04-02T12:06:06.437Z + + + https://docs.axolotl.ai/docs/models/hunyuan.html + 2026-04-02T12:06:06.445Z + + + https://docs.axolotl.ai/docs/models/smolvlm2.html + 2026-04-02T12:06:06.444Z + + + https://docs.axolotl.ai/docs/models/ministral3/vision.html + 2026-04-02T12:06:06.437Z + + + https://docs.axolotl.ai/docs/models/voxtral.html + 2026-04-02T12:06:06.440Z + + + https://docs.axolotl.ai/docs/models/ministral.html + 2026-04-02T12:06:06.439Z + + + https://docs.axolotl.ai/docs/models/granite4.html + 2026-04-02T12:06:06.444Z + + + https://docs.axolotl.ai/docs/models/phi.html + 2026-04-02T12:06:06.443Z + + + https://docs.axolotl.ai/docs/models/internvl3_5.html + 2026-04-02T12:06:06.435Z + + + https://docs.axolotl.ai/docs/models/magistral/think.html + 2026-04-02T12:06:06.438Z + + + https://docs.axolotl.ai/docs/models/mistral-small.html + 2026-04-02T12:06:06.439Z + + + https://docs.axolotl.ai/docs/models/gemma3n.html + 2026-04-02T12:06:06.442Z + + + https://docs.axolotl.ai/docs/models/arcee.html + 2026-04-02T12:06:06.436Z + + + https://docs.axolotl.ai/docs/models/llama-2.html + 2026-04-02T12:06:06.441Z + + + https://docs.axolotl.ai/docs/models/llama-4.html + 2026-04-02T12:06:06.441Z + + + https://docs.axolotl.ai/docs/models/seed-oss.html + 2026-04-02T12:06:06.443Z + + + https://docs.axolotl.ai/docs/models/jamba.html + 2026-04-02T12:06:06.445Z + + + https://docs.axolotl.ai/docs/nccl.html + 2026-04-02T12:02:09.351Z + + + https://docs.axolotl.ai/docs/multipack.html + 2026-04-02T12:02:09.351Z + + + https://docs.axolotl.ai/docs/debugging.html + 2026-04-02T12:02:09.348Z + + + https://docs.axolotl.ai/docs/dataset_preprocessing.html + 2026-04-02T12:02:09.348Z + + + https://docs.axolotl.ai/docs/vllm_serving.html + 2026-04-02T12:02:09.352Z + + + https://docs.axolotl.ai/docs/optimizers.html + 2026-04-02T12:02:09.351Z + + + https://docs.axolotl.ai/docs/ebft.html + 2026-04-02T12:02:09.348Z + + + https://docs.axolotl.ai/docs/torchao.html + 2026-04-02T12:02:09.352Z + + + https://docs.axolotl.ai/docs/lr_groups.html + 2026-04-02T12:02:09.350Z + + + https://docs.axolotl.ai/docs/streaming.html + 2026-04-02T12:02:09.352Z + + + https://docs.axolotl.ai/docs/amd_hpc.html + 2026-04-02T12:02:09.347Z + + + https://docs.axolotl.ai/docs/installation.html + 2026-04-02T12:02:09.350Z + + + https://docs.axolotl.ai/docs/inference.html + 2026-04-02T12:02:09.350Z + + + https://docs.axolotl.ai/docs/getting-started.html + 2026-04-02T12:02:09.348Z + + + https://docs.axolotl.ai/docs/telemetry.html + 2026-04-02T12:02:09.352Z + + + https://docs.axolotl.ai/src/axolotl/integrations/cut_cross_entropy/ACKNOWLEDGEMENTS.html + 2026-04-02T12:02:09.391Z + + + https://docs.axolotl.ai/index.html + 2026-04-02T12:02:09.380Z + + + https://docs.axolotl.ai/examples/colab-notebooks/colab-axolotl-example.html + 2026-04-02T12:02:09.359Z + + + https://docs.axolotl.ai/src/axolotl/integrations/LICENSE.html + 2026-04-02T12:02:09.389Z + + + https://docs.axolotl.ai/docs/batch_vs_grad.html + 2026-04-02T12:02:09.347Z + + + https://docs.axolotl.ai/docs/sequence_parallelism.html + 2026-04-02T12:02:09.352Z + + + https://docs.axolotl.ai/docs/quantize.html + 2026-04-02T12:02:09.351Z + + + https://docs.axolotl.ai/docs/docker.html + 2026-04-02T12:02:09.348Z + + + https://docs.axolotl.ai/docs/attention.html + 2026-04-02T12:02:09.347Z + + + https://docs.axolotl.ai/docs/unsloth.html + 2026-04-02T12:02:09.352Z + + + https://docs.axolotl.ai/docs/qat.html + 2026-04-02T12:02:09.351Z + + + https://docs.axolotl.ai/docs/multi-node.html + 2026-04-02T12:02:09.351Z + + + https://docs.axolotl.ai/docs/custom_integrations.html + 2026-04-02T12:02:09.347Z + + + https://docs.axolotl.ai/docs/ray-integration.html + 2026-04-02T12:02:09.351Z + + + https://docs.axolotl.ai/docs/config-reference.html + 2026-04-02T12:06:05.346Z + + + https://docs.axolotl.ai/docs/gradient_checkpointing.html + 2026-04-02T12:02:09.348Z + + + https://docs.axolotl.ai/docs/grpo.html + 2026-04-02T12:02:09.348Z + + + https://docs.axolotl.ai/docs/choosing_method.html + 2026-04-02T12:02:09.347Z + + + https://docs.axolotl.ai/docs/models/LiquidAI.html + 2026-04-02T12:06:06.444Z + + + https://docs.axolotl.ai/docs/models/magistral.html + 2026-04-02T12:06:06.438Z + + + https://docs.axolotl.ai/docs/models/devstral.html + 2026-04-02T12:06:06.440Z + + + https://docs.axolotl.ai/docs/models/qwen3-next.html + 2026-04-02T12:06:06.441Z + + + https://docs.axolotl.ai/docs/models/mistral.html + 2026-04-02T12:06:06.440Z + + + https://docs.axolotl.ai/docs/models/plano.html + 2026-04-02T12:06:06.434Z + + + https://docs.axolotl.ai/docs/models/olmo3.html + 2026-04-02T12:06:06.435Z + + + https://docs.axolotl.ai/docs/models/magistral/vision.html + 2026-04-02T12:06:06.439Z + + + https://docs.axolotl.ai/docs/models/mimo.html + 2026-04-02T12:06:06.434Z + + + https://docs.axolotl.ai/docs/models/index.html + 2026-04-02T12:06:06.446Z + + + https://docs.axolotl.ai/docs/models/trinity.html + 2026-04-02T12:06:06.435Z + + + https://docs.axolotl.ai/docs/models/kimi-linear.html + 2026-04-02T12:06:06.434Z + + + https://docs.axolotl.ai/docs/models/orpheus.html + 2026-04-02T12:06:06.446Z + + + https://docs.axolotl.ai/docs/models/qwen3.html + 2026-04-02T12:06:06.442Z + + + https://docs.axolotl.ai/docs/models/ministral3/think.html + 2026-04-02T12:06:06.437Z + + + https://docs.axolotl.ai/docs/models/apertus.html + 2026-04-02T12:06:06.442Z + + + https://docs.axolotl.ai/docs/models/gpt-oss.html + 2026-04-02T12:06:06.443Z + + + https://docs.axolotl.ai/docs/mixed_precision.html + 2026-04-02T12:02:09.350Z + + + https://docs.axolotl.ai/docs/lora_optims.html + 2026-04-02T12:02:09.350Z + + + https://docs.axolotl.ai/docs/dataset_loading.html + 2026-04-02T12:02:09.348Z + + + https://docs.axolotl.ai/docs/input_output.html + 2026-04-02T12:02:09.350Z + + + https://docs.axolotl.ai/docs/fsdp_qlora.html + 2026-04-02T12:02:09.348Z + + + https://docs.axolotl.ai/docs/agents/preference_tuning.html + 2026-04-02T12:02:09.347Z + + + https://docs.axolotl.ai/docs/agents/reward_modelling.html + 2026-04-02T12:02:09.347Z + + + https://docs.axolotl.ai/docs/optimizations.html + 2026-04-02T12:02:09.351Z + + + https://docs.axolotl.ai/docs/training_stability.html + 2026-04-02T12:02:09.352Z https://docs.axolotl.ai/docs/cli.html - 2026-04-01T22:21:40.930Z + 2026-04-02T12:02:09.347Z https://docs.axolotl.ai/docs/api/utils.callbacks.mlflow_.html - 2026-04-01T22:25:10.404Z + 2026-04-02T12:05:43.145Z https://docs.axolotl.ai/docs/api/models.mamba.modeling_mamba.html - 2026-04-01T22:25:10.298Z + 2026-04-02T12:05:43.040Z https://docs.axolotl.ai/docs/api/core.trainers.dpo.trainer.html - 2026-04-01T22:25:09.150Z + 2026-04-02T12:05:41.895Z https://docs.axolotl.ai/docs/api/cli.utils.fetch.html - 2026-04-01T22:25:09.068Z + 2026-04-02T12:05:41.812Z https://docs.axolotl.ai/docs/api/core.builders.causal.html - 2026-04-01T22:25:08.759Z + 2026-04-02T12:05:41.506Z https://docs.axolotl.ai/docs/api/core.builders.rl.html - 2026-04-01T22:25:08.765Z + 2026-04-02T12:05:41.512Z https://docs.axolotl.ai/docs/api/utils.bench.html - 2026-04-01T22:25:09.786Z + 2026-04-02T12:05:42.533Z https://docs.axolotl.ai/docs/api/prompt_strategies.kto.user_defined.html - 2026-04-01T22:25:09.504Z + 2026-04-02T12:05:42.251Z https://docs.axolotl.ai/docs/api/prompt_strategies.alpaca_instruct.html - 2026-04-01T22:25:09.353Z + 2026-04-02T12:05:42.100Z https://docs.axolotl.ai/docs/api/prompt_strategies.alpaca_chat.html - 2026-04-01T22:25:09.351Z + 2026-04-02T12:05:42.098Z https://docs.axolotl.ai/docs/api/utils.collators.mm_chat.html - 2026-04-01T22:25:10.333Z + 2026-04-02T12:05:43.075Z https://docs.axolotl.ai/docs/api/utils.schedulers.html - 2026-04-01T22:25:09.857Z + 2026-04-02T12:05:42.604Z https://docs.axolotl.ai/docs/api/kernels.utils.html - 2026-04-01T22:25:09.625Z + 2026-04-02T12:05:42.372Z https://docs.axolotl.ai/docs/api/core.chat.format.chatml.html - 2026-04-01T22:25:08.812Z + 2026-04-02T12:05:41.560Z https://docs.axolotl.ai/docs/api/loaders.constants.html - 2026-04-01T22:25:09.241Z + 2026-04-02T12:05:41.988Z https://docs.axolotl.ai/docs/api/utils.schemas.model.html - 2026-04-01T22:25:09.961Z + 2026-04-02T12:05:42.708Z https://docs.axolotl.ai/docs/api/integrations.grokfast.optimizer.html - 2026-04-01T22:25:10.245Z + 2026-04-02T12:05:42.994Z https://docs.axolotl.ai/docs/api/cli.utils.load.html - 2026-04-01T22:25:09.075Z + 2026-04-02T12:05:41.819Z https://docs.axolotl.ai/docs/api/loaders.adapter.html - 2026-04-01T22:25:09.217Z + 2026-04-02T12:05:41.964Z https://docs.axolotl.ai/docs/api/cli.train.html - 2026-04-01T22:25:08.883Z + 2026-04-02T12:05:41.632Z https://docs.axolotl.ai/docs/api/monkeypatch.stablelm_attn_hijack_flash.html - 2026-04-01T22:25:09.694Z + 2026-04-02T12:05:42.441Z https://docs.axolotl.ai/docs/api/cli.checks.html - 2026-04-01T22:25:08.938Z + 2026-04-02T12:05:41.680Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.user_defined.html - 2026-04-01T22:25:09.480Z + 2026-04-02T12:05:42.227Z https://docs.axolotl.ai/docs/api/prompt_strategies.llama2_chat.html - 2026-04-01T22:25:09.394Z + 2026-04-02T12:05:42.141Z https://docs.axolotl.ai/docs/api/core.trainers.trl.html - 2026-04-01T22:25:09.135Z + 2026-04-02T12:05:41.879Z https://docs.axolotl.ai/docs/api/monkeypatch.mistral_attn_hijack_flash.html - 2026-04-01T22:25:09.635Z + 2026-04-02T12:05:42.383Z https://docs.axolotl.ai/docs/api/core.trainers.mixins.scheduler.html - 2026-04-01T22:25:09.262Z + 2026-04-02T12:05:42.008Z https://docs.axolotl.ai/docs/api/core.trainers.grpo.trainer.html - 2026-04-01T22:25:09.168Z + 2026-04-02T12:05:41.915Z https://docs.axolotl.ai/docs/api/cli.merge_lora.html - 2026-04-01T22:25:08.994Z + 2026-04-02T12:05:41.737Z https://docs.axolotl.ai/docs/api/datasets.html - 2026-04-01T22:25:08.664Z + 2026-04-02T12:05:41.409Z https://docs.axolotl.ai/docs/api/utils.schemas.training.html - 2026-04-01T22:25:09.969Z + 2026-04-02T12:05:42.717Z https://docs.axolotl.ai/docs/api/utils.distributed.html - 2026-04-01T22:25:09.882Z + 2026-04-02T12:05:42.629Z https://docs.axolotl.ai/docs/api/cli.cloud.base.html - 2026-04-01T22:25:09.037Z + 2026-04-02T12:05:41.781Z https://docs.axolotl.ai/docs/api/kernels.geglu.html - 2026-04-01T22:25:09.595Z + 2026-04-02T12:05:42.342Z https://docs.axolotl.ai/docs/api/core.trainers.mixins.optimizer.html - 2026-04-01T22:25:09.249Z + 2026-04-02T12:05:41.995Z https://docs.axolotl.ai/docs/api/index.html - 2026-04-01T22:25:08.565Z + 2026-04-02T12:05:41.309Z https://docs.axolotl.ai/docs/api/prompt_strategies.base.html - 2026-04-01T22:25:09.292Z + 2026-04-02T12:05:42.039Z https://docs.axolotl.ai/docs/api/cli.evaluate.html - 2026-04-01T22:25:08.899Z + 2026-04-02T12:05:41.642Z https://docs.axolotl.ai/docs/api/train.html - 2026-04-01T22:25:08.643Z + 2026-04-02T12:05:41.388Z https://docs.axolotl.ai/docs/api/common.architectures.html - 2026-04-01T22:25:10.276Z + 2026-04-02T12:05:43.018Z https://docs.axolotl.ai/docs/api/prompt_strategies.kto.llama3.html - 2026-04-01T22:25:09.492Z + 2026-04-02T12:05:42.239Z https://docs.axolotl.ai/docs/api/utils.callbacks.lisa.html - 2026-04-01T22:25:10.400Z + 2026-04-02T12:05:43.141Z https://docs.axolotl.ai/docs/api/cli.utils.train.html - 2026-04-01T22:25:09.097Z + 2026-04-02T12:05:41.842Z https://docs.axolotl.ai/docs/api/integrations.liger.args.html - 2026-04-01T22:25:10.260Z + 2026-04-02T12:05:43.007Z https://docs.axolotl.ai/docs/api/prompt_tokenizers.html - 2026-04-01T22:25:08.734Z + 2026-04-02T12:05:41.480Z https://docs.axolotl.ai/docs/api/cli.utils.sweeps.html - 2026-04-01T22:25:09.082Z + 2026-04-02T12:05:41.827Z https://docs.axolotl.ai/docs/api/cli.utils.args.html - 2026-04-01T22:25:09.061Z + 2026-04-02T12:05:41.806Z https://docs.axolotl.ai/docs/api/utils.chat_templates.html - 2026-04-01T22:25:09.769Z + 2026-04-02T12:05:42.516Z https://docs.axolotl.ai/docs/api/utils.schemas.config.html - 2026-04-01T22:25:09.952Z + 2026-04-02T12:05:42.699Z https://docs.axolotl.ai/docs/api/prompt_strategies.user_defined.html - 2026-04-01T22:25:09.378Z + 2026-04-02T12:05:42.125Z https://docs.axolotl.ai/docs/api/utils.schemas.datasets.html - 2026-04-01T22:25:09.994Z + 2026-04-02T12:05:42.742Z https://docs.axolotl.ai/docs/api/integrations.base.html - 2026-04-01T22:25:10.240Z + 2026-04-02T12:05:42.988Z https://docs.axolotl.ai/docs/api/utils.tokenization.html - 2026-04-01T22:25:09.767Z + 2026-04-02T12:05:42.514Z https://docs.axolotl.ai/docs/api/monkeypatch.multipack.html - 2026-04-01T22:25:09.637Z + 2026-04-02T12:05:42.384Z https://docs.axolotl.ai/docs/api/integrations.kd.trainer.html - 2026-04-01T22:25:10.255Z + 2026-04-02T12:05:43.003Z https://docs.axolotl.ai/docs/api/monkeypatch.mixtral.html - 2026-04-01T22:25:09.721Z + 2026-04-02T12:05:42.468Z https://docs.axolotl.ai/docs/api/core.trainers.base.html - 2026-04-01T22:25:09.116Z + 2026-04-02T12:05:41.860Z https://docs.axolotl.ai/docs/api/utils.schemas.integrations.html - 2026-04-01T22:25:10.036Z + 2026-04-02T12:05:42.784Z https://docs.axolotl.ai/docs/api/core.trainers.mixins.rng_state_loader.html - 2026-04-01T22:25:09.253Z + 2026-04-02T12:05:41.999Z https://docs.axolotl.ai/docs/api/cli.main.html - 2026-04-01T22:25:08.873Z + 2026-04-02T12:05:41.621Z https://docs.axolotl.ai/docs/api/monkeypatch.trainer_fsdp_optim.html - 2026-04-01T22:25:09.698Z + 2026-04-02T12:05:42.445Z https://docs.axolotl.ai/docs/api/core.datasets.transforms.chat_builder.html - 2026-04-01T22:25:08.832Z + 2026-04-02T12:05:41.579Z https://docs.axolotl.ai/docs/api/prompt_strategies.alpaca_w_system.html - 2026-04-01T22:25:09.368Z + 2026-04-02T12:05:42.114Z https://docs.axolotl.ai/docs/api/integrations.cut_cross_entropy.args.html - 2026-04-01T22:25:10.244Z + 2026-04-02T12:05:42.992Z https://docs.axolotl.ai/docs/api/monkeypatch.transformers_fa_utils.html - 2026-04-01T22:25:09.706Z + 2026-04-02T12:05:42.453Z https://docs.axolotl.ai/docs/api/utils.data.streaming.html - 2026-04-01T22:25:09.900Z + 2026-04-02T12:05:42.648Z https://docs.axolotl.ai/docs/api/utils.collators.batching.html - 2026-04-01T22:25:10.323Z + 2026-04-02T12:05:43.065Z https://docs.axolotl.ai/docs/api/utils.samplers.multipack.html - 2026-04-01T22:25:10.384Z + 2026-04-02T12:05:43.126Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.chatml.html - 2026-04-01T22:25:09.476Z + 2026-04-02T12:05:42.223Z https://docs.axolotl.ai/docs/api/utils.dict.html - 2026-04-01T22:25:09.889Z + 2026-04-02T12:05:42.636Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.zephyr.html - 2026-04-01T22:25:09.478Z + 2026-04-02T12:05:42.225Z https://docs.axolotl.ai/docs/api/utils.optimizers.adopt.html - 2026-04-01T22:25:09.898Z + 2026-04-02T12:05:42.646Z https://docs.axolotl.ai/docs/api/prompt_strategies.metharme.html - 2026-04-01T22:25:09.423Z + 2026-04-02T12:05:42.170Z https://docs.axolotl.ai/docs/api/monkeypatch.gradient_checkpointing.offload_cpu.html - 2026-04-01T22:25:09.725Z + 2026-04-02T12:05:42.472Z https://docs.axolotl.ai/docs/rlhf.html - 2026-04-01T22:21:40.934Z + 2026-04-02T12:02:09.351Z https://docs.axolotl.ai/docs/dataset-formats/inst_tune.html - 2026-04-01T22:21:40.930Z + 2026-04-02T12:02:09.348Z https://docs.axolotl.ai/docs/dataset-formats/stepwise_supervised.html - 2026-04-01T22:21:40.930Z + 2026-04-02T12:02:09.348Z https://docs.axolotl.ai/docs/dataset-formats/tokenized.html - 2026-04-01T22:21:40.930Z + 2026-04-02T12:02:09.348Z https://docs.axolotl.ai/docs/multimodal.html - 2026-04-01T22:21:40.933Z + 2026-04-02T12:02:09.351Z diff --git a/src/axolotl/integrations/LICENSE.html b/src/axolotl/integrations/LICENSE.html index 0039ef087..f24a175b1 100644 --- a/src/axolotl/integrations/LICENSE.html +++ b/src/axolotl/integrations/LICENSE.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +
  • diff --git a/src/axolotl/integrations/cut_cross_entropy/ACKNOWLEDGEMENTS.html b/src/axolotl/integrations/cut_cross_entropy/ACKNOWLEDGEMENTS.html index f4b8fc587..49e40d761 100644 --- a/src/axolotl/integrations/cut_cross_entropy/ACKNOWLEDGEMENTS.html +++ b/src/axolotl/integrations/cut_cross_entropy/ACKNOWLEDGEMENTS.html @@ -141,6 +141,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); Quickstart
    +
    • +
  • +
    + + Which Fine-Tuning Method Should I Use? +
  • @@ -525,6 +531,24 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); RLHF (Beta)
    +
    • +
  • +
    + + GRPO Training +
    +
    • +
  • +
    + + EBFT Training +
    +
    • +
  • +
    + + vLLM Serving for GRPO Training +
  • @@ -696,6 +720,12 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); FAQ
    +
    • +
  • +
    + + Training Stability & Debugging +