From 31851b799183b5c680878285660f2fcc51bb12cb Mon Sep 17 00:00:00 2001 From: Quarto GHA Workflow Runner Date: Tue, 6 Jan 2026 14:26:27 +0000 Subject: [PATCH] Built site for gh-pages --- .nojekyll | 2 +- docs/custom_integrations.html | 1321 ++++++++++++++++++++++++++++++++- search.json | 11 + sitemap.xml | 470 ++++++------ 4 files changed, 1565 insertions(+), 239 deletions(-) diff --git a/.nojekyll b/.nojekyll index be700d70b..0e2f10eb5 100644 --- a/.nojekyll +++ b/.nojekyll @@ -1 +1 @@ -a31e3b61 \ No newline at end of file +534f52b6 \ No newline at end of file diff --git a/docs/custom_integrations.html b/docs/custom_integrations.html index e08c90fb7..922e38556 100644 --- a/docs/custom_integrations.html +++ b/docs/custom_integrations.html @@ -802,6 +802,94 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});
  • Usage
  • Citation
    • +
  • SwanLab Integration for Axolotl +
  • Adding a new integration
    • @@ -1335,6 +1423,1233 @@ By identifying the top n% of layers with the highest SNR, you can optimize train

    Please see reference here

    +
    +

    SwanLab Integration for Axolotl

    +

    SwanLab is an open-source, lightweight AI experiment tracking and visualization tool that provides a platform for tracking, recording, comparing, and collaborating on experiments.

    +

    This integration enables seamless experiment tracking and visualization of Axolotl training runs using SwanLab.

    +
    +

    Features

    +
      +
    • 📊 Automatic Metrics Logging: Training loss, learning rate, and other metrics are automatically logged
      • +
    • 🎯 Hyperparameter Tracking: Model configuration and training parameters are tracked
      • +
    • 📈 Real-time Visualization: Monitor training progress in real-time through SwanLab dashboard
      • +
    • ☁️ Cloud & Local Support: Works in both cloud-synced and offline modes
      • +
    • 🔄 Experiment Comparison: Compare multiple training runs easily
      • +
    • 🤝 Team Collaboration: Share experiments with team members
      • +
    • 🎭 RLHF Completion Logging: Automatically log model outputs during DPO/KTO/ORPO/GRPO training for qualitative analysis
      • +
    • ⚡ Performance Profiling: Built-in profiling decorators to measure and optimize training performance
      • +
    • 🔔 Lark Notifications: Send real-time training updates to team chat (Feishu/Lark integration)
      • +
    +
    +
    +

    Installation

    +
    pip install swanlab
    +
    +
    +

    Quick Start

    +
    +
    +

    1. Register for SwanLab (Optional for cloud mode)

    +

    If you want to use cloud sync features, register at https://swanlab.cn to get your API key.

    +
    +
    +

    2. Configure Axolotl Config File

    +

    Add SwanLab configuration to your Axolotl YAML config:

    +
    plugins:
+  - axolotl.integrations.swanlab.SwanLabPlugin
+
+use_swanlab: true
+swanlab_project: my-llm-project
+swanlab_experiment_name: qwen-finetune-v1
+swanlab_mode: cloud  # Options: cloud, local, offline, disabled
+swanlab_workspace: my-team  # Optional: organization name
+swanlab_api_key: YOUR_API_KEY  # Optional: can also use env var SWANLAB_API_KEY
    +
    +
    +

    3. Run Training

    +
    export SWANLAB_API_KEY=your-api-key-here
+
+swanlab login
+
+accelerate launch -m axolotl.cli.train your-config.yaml
    +
    +
    +

    Configuration Options

    +
    +
    +

    Basic Configuration

    + ++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ParameterTypeDefaultDescription
    use_swanlabboolfalseEnable SwanLab tracking
    swanlab_projectstrNoneProject name (required)
    swanlab_experiment_namestrNoneExperiment name
    swanlab_descriptionstrNoneExperiment description
    swanlab_modestrcloudSync mode: cloud, local, offline, disabled
    +
    +
    +

    Advanced Configuration

    + ++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ParameterTypeDefaultDescription
    swanlab_workspacestrNoneWorkspace/organization name
    swanlab_api_keystrNoneAPI key (prefer env var)
    swanlab_web_hoststrNonePrivate deployment web host
    swanlab_api_hoststrNonePrivate deployment API host
    swanlab_log_modelboolfalseLog model checkpoints (coming soon)
    swanlab_lark_webhook_urlstrNoneLark (Feishu) webhook URL for team notifications
    swanlab_lark_secretstrNoneLark webhook HMAC secret for authentication
    swanlab_log_completionsbooltrueEnable RLHF completion table logging (DPO/KTO/ORPO/GRPO)
    swanlab_completion_log_intervalint100Steps between completion logging
    swanlab_completion_max_bufferint128Max completions to buffer (memory bound)
    +
    +
    +

    Configuration Examples

    +
    +
    +

    Example 1: Basic Cloud Sync

    +
    plugins:
+  - axolotl.integrations.swanlab.SwanLabPlugin
+
+use_swanlab: true
+swanlab_project: llama-finetune
+swanlab_experiment_name: llama-3-8b-instruct-v1
+swanlab_mode: cloud
    +
    +
    +

    Example 2: Offline/Local Mode

    +
    plugins:
+  - axolotl.integrations.swanlab.SwanLabPlugin
+
+use_swanlab: true
+swanlab_project: local-experiments
+swanlab_experiment_name: test-run-1
+swanlab_mode: local  # or 'offline'
    +
    +
    +

    Example 3: Team Workspace

    +
    plugins:
+  - axolotl.integrations.swanlab.SwanLabPlugin
+
+use_swanlab: true
+swanlab_project: research-project
+swanlab_experiment_name: experiment-42
+swanlab_workspace: my-research-team
+swanlab_mode: cloud
    +
    +
    +

    Example 4: Private Deployment

    +
    plugins:
+  - axolotl.integrations.swanlab.SwanLabPlugin
+
+use_swanlab: true
+swanlab_project: internal-project
+swanlab_experiment_name: secure-training
+swanlab_mode: cloud
+swanlab_web_host: https://swanlab.yourcompany.com
+swanlab_api_host: https://api.swanlab.yourcompany.com
    +
    +
    +

    Team Notifications with Lark (Feishu)

    +

    SwanLab supports sending real-time training notifications to your team chat via Lark (Feishu), ByteDance’s enterprise collaboration platform. This is especially useful for: +- Production training monitoring: Get alerts when training starts, completes, or encounters errors +- Team collaboration: Keep your ML team informed about long-running experiments +- Multi-timezone teams: Team members can check training progress without being online

    +
    +
    +

    Prerequisites

    +
      +
    1. Lark Bot Setup: Create a custom bot in your Lark group chat
      2. +
    2. Webhook URL: Get the webhook URL from your Lark bot settings
      3. +
    3. HMAC Secret (recommended): Enable signature verification in your Lark bot for security
      4. +
    +

    For detailed Lark bot setup instructions, see Lark Custom Bot Documentation.

    +
    +
    +

    Example 5: Basic Lark Notifications

    +

    Send training notifications to a Lark group chat:

    +
    plugins:
+  - axolotl.integrations.swanlab.SwanLabPlugin
+
+use_swanlab: true
+swanlab_project: production-training
+swanlab_experiment_name: llama-3-finetune-v2
+swanlab_mode: cloud
+
+swanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxx
    +

    Note: This configuration will work, but you’ll see a security warning recommending HMAC secret configuration.

    +
    + +
    +

    Example 7: Team Workspace + Lark Notifications

    +

    Combine team workspace collaboration with Lark notifications:

    +
    plugins:
+  - axolotl.integrations.swanlab.SwanLabPlugin
+
+use_swanlab: true
+swanlab_project: research-project
+swanlab_experiment_name: multimodal-experiment-42
+swanlab_workspace: ml-research-team
+swanlab_mode: cloud
+
+swanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxx
+swanlab_lark_secret: your-webhook-secret-key
    +
    +
    +

    What Notifications Are Sent?

    +

    SwanLab’s Lark integration sends notifications for key training events: +- Training Start: When your experiment begins +- Training Complete: When training finishes successfully +- Training Errors: If training crashes or encounters critical errors +- Metric Milestones: Configurable alerts for metric thresholds (if configured in SwanLab)

    +

    Each notification includes: +- Experiment name and project +- Training status +- Key metrics (loss, learning rate) +- Direct link to SwanLab dashboard

    +
    +
    +

    Lark Configuration Validation

    +

    The plugin validates your Lark configuration at startup:

    +
    +

    ✅ Valid Configurations

    +
    use_swanlab: true
+swanlab_project: my-project
+
+use_swanlab: true
+swanlab_project: my-project
+swanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxx
+swanlab_lark_secret: your-secret
+
+use_swanlab: true
+swanlab_project: my-project
+swanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxx
    +
    +
    +
    +

    Security Best Practices

    +
      +

    1. Always use HMAC secret in production:

      +
      swanlab_lark_webhook_url: https://open.feishu.cn/...
+swanlab_lark_secret: your-secret-key  # ✅ Add this!
      2. +

    2. Store secrets in environment variables (even better):

      +
      # In your training script/environment
+export SWANLAB_LARK_WEBHOOK_URL="https://open.feishu.cn/..."
+export SWANLAB_LARK_SECRET="your-secret-key"
      +

      Then in config:

      +
      # SwanLab plugin will auto-detect environment variables
+use_swanlab: true
+swanlab_project: my-project
+# Lark URL and secret read from env vars
      3. +

    3. Rotate webhook secrets periodically: Update your Lark bot’s secret every 90 days

      4. +

    4. Use separate webhooks for dev/prod: Don’t mix development and production notifications

      5. +
    +
    +
    +

    Distributed Training

    +

    Lark notifications are automatically deduplicated in distributed training: +- Only rank 0 sends notifications +- Other GPU ranks skip Lark registration +- Prevents duplicate messages in multi-GPU training

    +
    torchrun --nproc_per_node=4 -m axolotl.cli.train config.yml
    +
    +
    +

    RLHF Completion Table Logging

    +

    For RLHF (Reinforcement Learning from Human Feedback) training methods like DPO, KTO, ORPO, and GRPO, SwanLab can log model completions (prompts, chosen/rejected responses, rewards) to a visual table for qualitative analysis. This helps you:

    +
      +
    • Inspect model behavior: See actual model outputs during training
      • +
    • Debug preference learning: Compare chosen vs rejected responses
      • +
    • Track reward patterns: Monitor how rewards evolve over training
      • +
    • Share examples with team: Visual tables in SwanLab dashboard
      • +
    +
    +
    +

    Features

    +
      +
    • ✅ Automatic detection: Works with DPO, KTO, ORPO, GRPO trainers
      • +
    • ✅ Memory-safe buffering: Bounded buffer prevents memory leaks in long training runs
      • +
    • ✅ Periodic logging: Configurable logging interval to reduce overhead
      • +
    • ✅ Rich visualization: SwanLab tables show prompts, responses, and metrics side-by-side
      • +
    +
    +
    +

    Configuration

    + ++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ParameterTypeDefaultDescription
    swanlab_log_completionsbooltrueEnable completion logging for RLHF trainers
    swanlab_completion_log_intervalint100Log completions to SwanLab every N training steps
    swanlab_completion_max_bufferint128Maximum completions to buffer (memory bound)
    +
    +
    +

    Example: DPO Training with Completion Logging

    +
    plugins:
+  - axolotl.integrations.swanlab.SwanLabPlugin
+
+use_swanlab: true
+swanlab_project: dpo-training
+swanlab_experiment_name: llama-3-dpo-v1
+swanlab_mode: cloud
+
+swanlab_log_completions: true
+swanlab_completion_log_interval: 100  # Log every 100 steps
+swanlab_completion_max_buffer: 128    # Keep last 128 completions
+
+rl: dpo
+datasets:
+  - path: /path/to/preference_dataset
+    type: chatml.intel
    +
    +
    +

    Example: Disable Completion Logging

    +

    If you’re doing a quick test run or don’t need completion tables:

    +
    plugins:
+  - axolotl.integrations.swanlab.SwanLabPlugin
+
+use_swanlab: true
+swanlab_project: dpo-training
+
+swanlab_log_completions: false
    +
    +
    +

    Supported RLHF Trainers

    +

    The completion logging callback automatically activates for these trainer types:

    +
      +
    • DPO (Direct Preference Optimization): Logs prompts, chosen, rejected, reward_diff
      • +
    • KTO (Kahneman-Tversky Optimization): Logs prompts, completions, labels, rewards
      • +
    • ORPO (Odds Ratio Preference Optimization): Logs prompts, chosen, rejected, log_odds_ratio
      • +
    • GRPO (Group Relative Policy Optimization): Logs prompts, completions, rewards, advantages
      • +
    • CPO (Constrained Policy Optimization): Logs prompts, chosen, rejected
      • +
    +

    For non-RLHF trainers (standard supervised fine-tuning), the completion callback is automatically skipped.

    +
    +
    +

    How It Works

    +
      +
    1. Auto-detection: Plugin detects trainer type at initialization
      2. +
    2. Buffering: Completions are buffered in memory (up to swanlab_completion_max_buffer)
      3. +
    3. Periodic logging: Every swanlab_completion_log_interval steps, buffer is logged to SwanLab
      4. +
    4. Memory safety: Old completions are automatically dropped when buffer is full (uses collections.deque)
      5. +
    5. Final flush: Remaining completions are logged when training completes
      6. +
    +
    +
    +

    Viewing Completion Tables

    +

    After training starts, you can view completion tables in your SwanLab dashboard:

    +
      +
    1. Navigate to your experiment in SwanLab
      2. +
    2. Look for the “rlhf_completions” table in the metrics panel
      3. +
    3. The table shows: +
        +
      • step: Training step when completion was generated
        • +
      • prompt: Input prompt
        • +
      • chosen: Preferred response (DPO/ORPO)
        • +
      • rejected: Non-preferred response (DPO/ORPO)
        • +
      • completion: Model output (KTO/GRPO)
        • +
      • reward_diff/reward: Reward metrics
        • +
      • Trainer-specific metrics (e.g., log_odds_ratio for ORPO)
        • +
      4. +
    +
    +
    +

    Memory Management

    +

    The completion buffer is memory-bounded to prevent memory leaks:

    +
    from collections import deque
+
+buffer = deque(maxlen=128)  # Old completions automatically dropped
    +

    Memory usage estimate: +- Average completion: ~500 characters (prompt + responses) +- Buffer size 128: ~64 KB (negligible) +- Buffer size 1024: ~512 KB (still small)

    +

    Recommendation: Default buffer size (128) works well for most cases. Increase to 512-1024 only if you need to review more historical completions.

    +
    +
    +

    Performance Impact

    +

    Completion logging has minimal overhead:

    +
      +
    • Buffering: O(1) append operation, negligible CPU/memory
      • +
    • Logging: Only happens every N steps (default: 100)
      • +
    • Network: SwanLab batches table uploads efficiently
      • +
    +

    Expected overhead: < 0.5% per training step

    +
    +
    +

    Troubleshooting

    +
    +

    Completions not appearing in SwanLab

    +

    Cause: Trainer may not be logging completion data in the expected format.

    +

    Diagnostic steps: +1. Check trainer type detection in logs: +text INFO: SwanLab RLHF completion logging enabled for DPOTrainer (type: dpo) +2. Verify your trainer is an RLHF trainer (DPO/KTO/ORPO/GRPO) +3. Check if trainer logs completion data (this depends on TRL version)

    +

    Note: The current implementation expects trainers to log completion data in the logs dict during on_log() callback. Some TRL trainers may not expose this data by default. You may need to patch the trainer to expose completions.

    +
    +
    +

    Buffer fills up too quickly

    +

    Cause: High logging frequency with small buffer size.

    +

    Solution: Increase buffer size or logging interval:

    +
    swanlab_completion_log_interval: 200  # Log less frequently
+swanlab_completion_max_buffer: 512    # Larger buffer
    +
    +
    +

    Memory usage growing over time

    +

    Cause: Buffer should be bounded, so this indicates a bug.

    +

    Solution: +1. Verify swanlab_completion_max_buffer is set +2. Check SwanLab version is up to date +3. Report issue with memory profiling data

    +
    +
    +
    +

    Performance Profiling

    +

    SwanLab integration includes profiling utilities to measure and log execution time of trainer methods. This helps you:

    +
      +
    • Identify bottlenecks: Find slow operations in your training loop
      • +
    • Optimize performance: Track improvements after optimization changes
      • +
    • Monitor distributed training: See per-rank timing differences
      • +
    • Debug hangs: Detect methods that take unexpectedly long
      • +
    +
    +
    +

    Features

    +
      +
    • ✅ Zero-config profiling: Automatic timing of key trainer methods
      • +
    • ✅ Decorator-based: Easy to add profiling to custom methods with @swanlab_profile
      • +
    • ✅ Context manager: Fine-grained profiling with swanlab_profiling_context()
      • +
    • ✅ Advanced filtering: ProfilingConfig for throttling and minimum duration thresholds
      • +
    • ✅ Exception-safe: Logs duration even if function raises an exception
      • +
    +
    +
    +

    Basic Usage: Decorator

    +

    Add profiling to any trainer method with the @swanlab_profile decorator:

    +
    from axolotl.integrations.swanlab.profiling import swanlab_profile
+
+class MyCustomTrainer(AxolotlTrainer):
+    @swanlab_profile
+    def training_step(self, model, inputs):
+        # Your training step logic
+        return super().training_step(model, inputs)
+
+    @swanlab_profile
+    def prediction_step(self, model, inputs, prediction_loss_only):
+        # Your prediction logic
+        return super().prediction_step(model, inputs, prediction_loss_only)
    +

    The decorator automatically: +1. Measures execution time with high-precision timer +2. Logs to SwanLab as profiling/Time taken: ClassName.method_name +3. Only logs if SwanLab is enabled (use_swanlab: true) +4. Gracefully handles exceptions (logs duration, then re-raises)

    +
    +
    +

    Advanced Usage: Context Manager

    +

    For fine-grained profiling within a method:

    +
    from axolotl.integrations.swanlab.profiling import swanlab_profiling_context
+
+class MyTrainer(AxolotlTrainer):
+    def complex_training_step(self, model, inputs):
+        # Profile just the forward pass
+        with swanlab_profiling_context(self, "forward_pass"):
+            outputs = model(**inputs)
+
+        # Profile just the backward pass
+        with swanlab_profiling_context(self, "backward_pass"):
+            loss = outputs.loss
+            loss.backward()
+
+        return outputs
    +
    +
    +

    Advanced Usage: ProfilingConfig

    +

    Filter and throttle profiling logs with ProfilingConfig:

    +
    from axolotl.integrations.swanlab.profiling import (
+    swanlab_profiling_context_advanced,
+    ProfilingConfig,
+)
+
+profiling_config = ProfilingConfig(
+    enabled=True,
+    min_duration_ms=1.0,    # Only log if duration > 1ms
+    log_interval=10,        # Log every 10th call
+)
+
+class MyTrainer(AxolotlTrainer):
+    def frequently_called_method(self, data):
+        with swanlab_profiling_context_advanced(
+            self,
+            "frequent_op",
+            config=profiling_config
+        ):
+            # This only logs every 10th call, and only if it takes > 1ms
+            result = expensive_computation(data)
+        return result
    +

    ProfilingConfig Parameters: +- enabled: Enable/disable profiling globally (default: True) +- min_duration_ms: Minimum duration to log in milliseconds (default: 0.1) +- log_interval: Log every Nth function call (default: 1 = log all)

    +

    Use cases: +- High-frequency methods: Use log_interval=100 to reduce logging overhead +- Filter noise: Use min_duration_ms=1.0 to skip very fast operations +- Debugging: Use log_interval=1, min_duration_ms=0.0 to log everything

    +
    +
    +

    Viewing Profiling Metrics

    +

    In your SwanLab dashboard, profiling metrics appear under the “profiling” namespace:

    +
    profiling/Time taken: AxolotlTrainer.training_step
+profiling/Time taken: AxolotlTrainer.prediction_step
+profiling/Time taken: MyTrainer.forward_pass
+profiling/Time taken: MyTrainer.backward_pass
    +

    You can: +- Track over time: See if methods get faster/slower during training +- Compare runs: Compare profiling metrics across experiments +- Identify regressions: Detect if a code change slowed down training

    +
    +
    +

    Configuration in Axolotl Config

    +

    Profiling is automatically enabled when SwanLab is enabled. No additional config needed:

    +
    plugins:
+  - axolotl.integrations.swanlab.SwanLabPlugin
+
+use_swanlab: true
+swanlab_project: my-project
    +

    To disable profiling while keeping SwanLab enabled:

    +
    from axolotl.integrations.swanlab.profiling import DEFAULT_PROFILING_CONFIG
+
+DEFAULT_PROFILING_CONFIG.enabled = False
    +
    +
    +

    Performance Impact

    +
      +
    • Decorator overhead: ~2-5 microseconds per call (negligible)
      • +
    • Context manager overhead: ~1-3 microseconds (negligible)
      • +
    • Logging overhead: Only when SwanLab is enabled and method duration exceeds threshold
      • +
    • Network overhead: SwanLab batches metrics efficiently
      • +
    +

    Expected overhead: < 0.1% per training step (effectively zero)

    +
    +
    +

    Best Practices

    +
      +
    1. Profile bottlenecks first: Start by profiling suspected slow operations
      2. +
    2. Use min_duration_ms: Filter out fast operations (< 1ms) to reduce noise
      3. +
    3. Throttle high-frequency calls: Use log_interval for methods called > 100 times/step
      4. +
    4. Profile across runs: Compare profiling metrics before/after optimization
      5. +
    5. Monitor distributed training: Check for rank-specific slowdowns
      6. +
    +
    +
    +

    Example: Complete Profiling Setup

    +
    from axolotl.integrations.swanlab.profiling import (
+    swanlab_profile,
+    swanlab_profiling_context,
+    ProfilingConfig,
+)
+
+class OptimizedTrainer(AxolotlTrainer):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # Custom profiling config for high-frequency operations
+        self.fast_op_config = ProfilingConfig(
+            enabled=True,
+            min_duration_ms=0.5,
+            log_interval=50,
+        )
+
+    @swanlab_profile
+    def training_step(self, model, inputs):
+        """Main training step - always profile."""
+        return super().training_step(model, inputs)
+
+    @swanlab_profile
+    def compute_loss(self, model, inputs, return_outputs=False):
+        """Loss computation - always profile."""
+        return super().compute_loss(model, inputs, return_outputs)
+
+    def _prepare_inputs(self, inputs):
+        """High-frequency operation - throttled profiling."""
+        with swanlab_profiling_context_advanced(
+            self,
+            "prepare_inputs",
+            config=self.fast_op_config,
+        ):
+            return super()._prepare_inputs(inputs)
    +
    +
    +

    Troubleshooting

    +
    +

    Profiling metrics not appearing in SwanLab

    +

    Cause: SwanLab is not enabled or not initialized.

    +

    Solution:

    +
    use_swanlab: true
+swanlab_project: my-project
    +

    Check logs for:

    +
    INFO: SwanLab initialized for project: my-project
    +
    +
    +

    Too many profiling metrics cluttering dashboard

    +

    Cause: Profiling every function call for high-frequency operations.

    +

    Solution: Use ProfilingConfig with throttling:

    +
    config = ProfilingConfig(
+    min_duration_ms=1.0,    # Skip fast ops
+    log_interval=100,       # Log every 100th call
+)
    +
    +
    +

    Profiling overhead impacting training speed

    +

    Cause: Profiling itself should have negligible overhead (< 0.1%). If you see > 1% slowdown, this indicates a bug.

    +

    Solution: +1. Disable profiling temporarily to confirm: +python DEFAULT_PROFILING_CONFIG.enabled = False +2. Report issue with profiling data and trainer details

    +
    +
    +

    Profiling shows inconsistent timing

    +

    Cause: Normal variation due to GPU warmup, data loading, or system load.

    +

    Solution: +- Ignore first few steps (warmup period) +- Look at average/median timing over many steps +- Use log_interval to reduce noise from individual outliers

    +
    +
    +
    +

    Complete Config Example

    +

    Here’s a complete example integrating SwanLab with your RVQ-Alpha training:

    +
    base_model: /path/to/your/model
+model_type: Qwen2ForCausalLM
+
+plugins:
+  - axolotl.integrations.swanlab.SwanLabPlugin
+  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
+
+use_swanlab: true
+swanlab_project: RVQ-Alpha-Training
+swanlab_experiment_name: Qwen2.5-7B-MetaQA-Perturb-P020
+swanlab_description: "Training on MetaQA and Perturbation datasets with NEW-RVQ encoding"
+swanlab_mode: cloud
+swanlab_workspace: single-cell-genomics
+
+sequence_len: 32768
+micro_batch_size: 1
+gradient_accumulation_steps: 1
+num_epochs: 2
+learning_rate: 2e-5
+optimizer: adamw_torch_fused
+
+datasets:
+  - path: /path/to/dataset
+    type: chat_template
+
+output_dir: ./outputs
    +
    +
    +

    Modes Explained

    +
    +
    +

    cloud Mode (Default)

    +
      +
    • Syncs experiments to SwanLab cloud in real-time
      • +
    • Requires API key and internet connection
      • +
    • Best for: Team collaboration, remote monitoring
      • +
    +
    +
    +

    local Mode

    +
      +
    • Saves experiments locally only
      • +
    • No cloud sync
      • +
    • Best for: Local development, air-gapped environments
      • +
    +
    +
    +

    offline Mode

    +
      +
    • Saves metadata locally
      • +
    • Can sync to cloud later using swanlab sync
      • +
    • Best for: Unstable internet, sync later
      • +
    +
    +
    +

    disabled Mode

    +
      +
    • Turns off SwanLab completely
      • +
    • No logging or tracking
      • +
    • Best for: Debugging, testing
      • +
    +
    +
    +

    Configuration Validation & Conflict Detection

    +

    SwanLab integration includes comprehensive validation and conflict detection to help you catch configuration errors early and avoid performance issues.

    +
    +
    +

    Required Fields Validation

    +

    The plugin validates your configuration at startup and provides clear error messages with solutions:

    +
    +

    Missing Project Name

    +
    use_swanlab: true
    +

    Solution:

    +
    use_swanlab: true
+swanlab_project: my-project
    +
    +
    +

    Invalid Mode

    +
    use_swanlab: true
+swanlab_project: my-project
+swanlab_mode: invalid-mode
    +

    Solution:

    +
    use_swanlab: true
+swanlab_project: my-project
+swanlab_mode: cloud  # or: local, offline, disabled
    +
    +
    +

    Empty Project Name

    +
    use_swanlab: true
+swanlab_project: ""
    +

    Solution:

    +
    use_swanlab: true
+swanlab_project: my-project
    +
    +
    +
    +

    Cloud Mode API Key Warning

    +

    When using cloud mode without an API key, you’ll receive a warning with multiple solutions:

    +
    use_swanlab: true
+swanlab_project: my-project
+swanlab_mode: cloud
    +

    Solutions: +1. Set environment variable: export SWANLAB_API_KEY=your-api-key +2. Add to config (less secure): swanlab_api_key: your-api-key +3. Run swanlab login before training +4. Use swanlab_mode: local for offline tracking

    +
    +
    +

    Multi-Logger Performance Warnings

    +

    Using multiple logging tools simultaneously (SwanLab + WandB + MLflow + Comet) can impact training performance:

    +
    +

    Two Loggers - Warning

    +
    use_swanlab: true
+swanlab_project: my-project
+
+use_wandb: true
+wandb_project: my-project
    +

    Impact: +- Performance overhead: ~1-2% per logger (cumulative) +- Increased memory usage +- Longer training time per step +- Potential config/callback conflicts

    +

    Recommendations: +- Choose ONE primary logging tool for production training +- Use multiple loggers only for: +- Migration period (transitioning between tools) +- Short comparison runs +- Debugging specific tool issues +- Monitor system resources (CPU, memory) during training

    +
    +
    +

    Three+ Loggers - Error-Level Warning

    +
    use_swanlab: true
+swanlab_project: my-project
+
+use_wandb: true
+wandb_project: my-project
+
+use_mlflow: true
+mlflow_tracking_uri: http://localhost:5000
    +

    Why This Matters: +- With 3 loggers: ~4-5% overhead per step → significant slowdown over long training +- Example: 10,000 steps at 2s/step → ~400-500 seconds extra (6-8 minutes) +- Memory overhead scales with number of loggers +- Rare edge cases with callback ordering conflicts

    +
    +
    +
    +

    Auto-Enable Logic

    +

    For convenience, SwanLab will auto-enable if you specify a project without setting use_swanlab:

    +
    swanlab_project: my-project
+
+use_swanlab: true
+swanlab_project: my-project
    +
    +
    +

    Distributed Training Detection

    +

    In distributed training scenarios (multi-GPU), the plugin automatically detects and reports:

    +
    use_swanlab: true
+swanlab_project: my-project
+swanlab_mode: cloud
    +

    Why Only Rank 0: +- Avoids duplicate experiment runs +- Reduces network/cloud API overhead on worker ranks +- Prevents race conditions in metric logging

    +
    +
    +

    Authentication

    +
    + +
    +

    Method 2: Login Command

    +
    swanlab login
    +
    +
    +

    Method 3: Config File

    +
    swanlab_api_key: your-api-key-here
    +
    +
    +

    What Gets Logged?

    +
    +
    +

    Automatically Logged Metrics

    +
      +
    • Training loss
      • +
    • Learning rate
      • +
    • Gradient norm
      • +
    • Training steps
      • +
    • Epoch progress
      • +
    +
    +
    +

    Automatically Logged Config

    +
      +
    • Model configuration (base_model, model_type)
      • +
    • Training hyperparameters (learning_rate, batch_size, etc.)
      • +
    • Optimizer settings
      • +
    • Parallelization settings (FSDP, DeepSpeed, Context Parallel)
      • +
    • Axolotl configuration file
      • +
    • DeepSpeed configuration (if used)
      • +
    +
    +
    +

    Viewing Your Experiments

    +
    +
    +

    Cloud Mode

    +

    Visit https://swanlab.cn and navigate to your project to view: +- Real-time training metrics +- Hyperparameter comparison +- System resource usage +- Configuration files

    +
    +
    +

    Local Mode

    +
    swanlab watch ./swanlog
    +
    +
    +

    Integration with Existing Tools

    +

    SwanLab can work alongside other tracking tools:

    +
    plugins:
+  - axolotl.integrations.swanlab.SwanLabPlugin
+
+use_swanlab: true
+swanlab_project: my-project
+
+use_wandb: true
+wandb_project: my-project
    +
    +
    +

    Troubleshooting

    +
    +
    +

    Configuration Errors

    +
    +

    Error: “SwanLab enabled but ‘swanlab_project’ is not set”

    +

    Cause: You enabled SwanLab (use_swanlab: true) but forgot to specify a project name.

    +

    Solution:

    +
    use_swanlab: true
+swanlab_project: my-project  # Add this line
    +
    +
    +

    Error: “Invalid swanlab_mode: ‘xxx’”

    +

    Cause: You provided an invalid mode value.

    +

    Solution: Use one of the valid modes:

    +
    swanlab_mode: cloud     # or: local, offline, disabled
    +
    +
    +

    Error: “swanlab_project cannot be an empty string”

    +

    Cause: You set swanlab_project: "" (empty string).

    +

    Solution: Either provide a valid name or remove the field:

    +
    swanlab_project: my-project
    +
    +
    +
    +

    Import Errors

    +
    +

    Error: “SwanLab is not installed”

    +

    Cause: SwanLab package is not installed in your environment.

    +

    Solution:

    +
    pip install swanlab
+pip install swanlab>=0.3.0
    +
    +
    +
    +

    Performance Issues

    +
    +

    Warning: “Multiple logging tools enabled”

    +

    Cause: You have multiple experiment tracking tools enabled (e.g., SwanLab + WandB + MLflow).

    +

    Impact: ~1-2% performance overhead per logger, cumulative.

    +

    Solution: For production training, disable all but one logger:

    +
    use_swanlab: true
+swanlab_project: my-project
+use_wandb: false      # Disable others
+use_mlflow: false
+
+use_swanlab: false
+use_wandb: true
+wandb_project: my-project
    +

    Exception: Multiple loggers are acceptable for: +- Short comparison runs (< 100 steps) +- Migration testing between logging tools +- Debugging logger-specific issues

    +
    +
    +
    +

    Distributed Training Issues

    +
    +

    SwanLab creates duplicate runs in multi-GPU training

    +

    Cause: All ranks are initializing SwanLab instead of just rank 0.

    +

    Expected Behavior: The plugin automatically ensures only rank 0 initializes SwanLab. You should see:

    +
    Info: Distributed training detected (world_size=4)
+Info: Only rank 0 will initialize SwanLab
+Info: Other ranks will skip SwanLab to avoid conflicts
    +

    If you see duplicates: +1. Check your plugin is loaded correctly +2. Verify you’re using the latest SwanLab integration code +3. Check logs for initialization messages on all ranks

    +
    +
    +
    +

    SwanLab not logging metrics

    +

    Solution: Ensure SwanLab is initialized before training starts. The plugin automatically handles this in pre_model_load.

    +
    +
    +

    API Key errors

    +

    Solution:

    +
    echo $SWANLAB_API_KEY
+
+swanlab login
    +
    +
    +

    Cloud sync issues

    +

    Solution: Use offline mode and sync later:

    +
    swanlab_mode: offline
    +

    Then sync when ready:

    +
    swanlab sync ./swanlog
    +
    +
    +

    Plugin not loaded

    +

    Solution: Verify plugin path in config:

    +
    plugins:
+  - axolotl.integrations.swanlab.SwanLabPlugin  # Correct path
    +
    +
    +

    Lark Notification Issues

    +
    +

    Error: “Failed to import SwanLab Lark plugin”

    +

    Cause: Your SwanLab version doesn’t include the Lark plugin (requires SwanLab >= 0.3.0).

    +

    Solution:

    +
    pip install --upgrade swanlab
+
+pip install 'swanlab>=0.3.0'
    +
    +
    +

    Warning: “Lark webhook has no secret configured”

    +

    Cause: You provided swanlab_lark_webhook_url but no swanlab_lark_secret.

    +

    Impact: Lark notifications will work, but without HMAC authentication (security risk).

    +

    Solution: Add HMAC secret for production use:

    +
    swanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxx
+swanlab_lark_secret: your-webhook-secret  # Add this line
    +

    When it’s OK to skip secret: +- Local development and testing +- Internal networks with restricted access +- Non-sensitive training experiments

    +

    When secret is required: +- Production training jobs +- Training with proprietary data +- Multi-team shared Lark groups

    +
    +
    +

    Error: “Failed to register Lark callback”

    +

    Cause: Invalid webhook URL or network connectivity issues.

    +

    Diagnostic steps:

    +
    curl -X POST "YOUR_WEBHOOK_URL" \
+  -H 'Content-Type: application/json' \
+  -d '{"msg_type":"text","content":{"text":"Test from Axolotl"}}'
+
+pip show swanlab
    +

    Solution: +1. Verify webhook URL is correct (copy from Lark bot settings) +2. Check network connectivity to Lark API +3. Ensure webhook is not expired (Lark webhooks can expire) +4. Regenerate webhook URL in Lark bot settings if needed

    +
    +
    +

    Lark notifications not received

    +

    Cause: Multiple possible causes.

    +

    Diagnostic checklist:

    +
      +

    1. Check training logs for Lark registration confirmation:

      +
      # Expected log message (rank 0 only):
+INFO: Registered Lark notification callback with HMAC authentication
      2. +

    2. Verify webhook in Lark: Test webhook manually (see above)

      3. +

    3. Check distributed training: Only rank 0 sends notifications

      +
      # If running multi-GPU, check rank 0 logs specifically
+grep "Registered Lark" logs/rank_0.log
      4. +

    4. Verify SwanLab is initialized: Lark callback needs SwanLab to be running

      +
      use_swanlab: true  # Must be enabled
+swanlab_project: my-project  # Must be set
      5. +

    5. Check Lark bot permissions: Ensure bot is added to the target group chat

      6. +
    +
    +
    +

    Duplicate Lark notifications in multi-GPU training

    +

    Expected Behavior: Should NOT happen - only rank 0 sends notifications.

    +

    If you see duplicates: +1. Check that all GPUs are using the same config file +2. Verify plugin is loaded correctly on all ranks +3. Check logs for unexpected Lark initialization on non-zero ranks +4. Ensure RANK or LOCAL_RANK environment variables are set correctly

    +

    Solution: This is a bug if it occurs. Report with: +- Full training command +- Logs from all ranks +- Config file

    +
    +
    +
    +

    Comparison: SwanLab vs WandB

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    FeatureSwanLabWandB
    Open Source✅ Yes❌ No
    Self-Hosting✅ Easy⚠️ Complex
    Free Tier✅ Generous⚠️ Limited
    Chinese Support✅ Native⚠️ Limited
    Offline Mode✅ Full support✅ Supported
    Integration🆕 New✅ Mature
    +
    +
    +

    Advanced Usage

    +
    +
    +

    Custom Logging

    +

    You can add custom metrics in your callbacks:

    +
    import swanlab
+
+swanlab.log({
+    "custom_metric": value,
+    "epoch": epoch_num
+})
    +
    +
    +

    Experiment Comparison

    +
    swanlab compare run1 run2 run3
    +
    +
    +

    Support

    + +
    +
    +

    License

    +

    This integration follows the Axolotl Community License Agreement.

    +
    +
    +

    Acknowledgements

    +

    This integration is built on top of: +- SwanLab - Experiment tracking tool +- Transformers - SwanLabCallback +- Axolotl - Training framework

    +

    Please see reference here

    +
    +

    Adding a new integration

    Plugins can be used to customize the behavior of the training pipeline through hooks. See axolotl.integrations.BasePlugin for the possible hooks.

    @@ -1375,10 +2690,10 @@ Warning

    If you could not load your integration, please ensure you are pip installing in editable mode.

    -
    pip install -e .
    +
    pip install -e .

    and correctly spelled the integration name in the config file.

    -
    plugins:
-  - axolotl.integrations.your_integration_name.YourIntegrationPlugin
    +
    plugins:
+  - axolotl.integrations.your_integration_name.YourIntegrationPlugin
    diff --git a/search.json b/search.json index 773ed0065..7297b6dce 100644 --- a/search.json +++ b/search.json @@ -3311,6 +3311,17 @@ "Custom Integrations" ] }, + { + "objectID": "docs/custom_integrations.html#swanlab-integration-for-axolotl", + "href": "docs/custom_integrations.html#swanlab-integration-for-axolotl", + "title": "Custom Integrations", + "section": "SwanLab Integration for Axolotl", + "text": "SwanLab Integration for Axolotl\nSwanLab is an open-source, lightweight AI experiment tracking and visualization tool that provides a platform for tracking, recording, comparing, and collaborating on experiments.\nThis integration enables seamless experiment tracking and visualization of Axolotl training runs using SwanLab.\n\nFeatures\n\n📊 Automatic Metrics Logging: Training loss, learning rate, and other metrics are automatically logged\n🎯 Hyperparameter Tracking: Model configuration and training parameters are tracked\n📈 Real-time Visualization: Monitor training progress in real-time through SwanLab dashboard\n☁️ Cloud & Local Support: Works in both cloud-synced and offline modes\n🔄 Experiment Comparison: Compare multiple training runs easily\n🤝 Team Collaboration: Share experiments with team members\n🎭 RLHF Completion Logging: Automatically log model outputs during DPO/KTO/ORPO/GRPO training for qualitative analysis\n⚡ Performance Profiling: Built-in profiling decorators to measure and optimize training performance\n🔔 Lark Notifications: Send real-time training updates to team chat (Feishu/Lark integration)\n\n\n\nInstallation\npip install swanlab\n\n\nQuick Start\n\n\n1. Register for SwanLab (Optional for cloud mode)\nIf you want to use cloud sync features, register at https://swanlab.cn to get your API key.\n\n\n2. Configure Axolotl Config File\nAdd SwanLab configuration to your Axolotl YAML config:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: my-llm-project\nswanlab_experiment_name: qwen-finetune-v1\nswanlab_mode: cloud # Options: cloud, local, offline, disabled\nswanlab_workspace: my-team # Optional: organization name\nswanlab_api_key: YOUR_API_KEY # Optional: can also use env var SWANLAB_API_KEY\n\n\n3. Run Training\nexport SWANLAB_API_KEY=your-api-key-here\n\nswanlab login\n\naccelerate launch -m axolotl.cli.train your-config.yaml\n\n\nConfiguration Options\n\n\nBasic Configuration\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nuse_swanlab\nbool\nfalse\nEnable SwanLab tracking\n\n\nswanlab_project\nstr\nNone\nProject name (required)\n\n\nswanlab_experiment_name\nstr\nNone\nExperiment name\n\n\nswanlab_description\nstr\nNone\nExperiment description\n\n\nswanlab_mode\nstr\ncloud\nSync mode: cloud, local, offline, disabled\n\n\n\n\n\nAdvanced Configuration\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nswanlab_workspace\nstr\nNone\nWorkspace/organization name\n\n\nswanlab_api_key\nstr\nNone\nAPI key (prefer env var)\n\n\nswanlab_web_host\nstr\nNone\nPrivate deployment web host\n\n\nswanlab_api_host\nstr\nNone\nPrivate deployment API host\n\n\nswanlab_log_model\nbool\nfalse\nLog model checkpoints (coming soon)\n\n\nswanlab_lark_webhook_url\nstr\nNone\nLark (Feishu) webhook URL for team notifications\n\n\nswanlab_lark_secret\nstr\nNone\nLark webhook HMAC secret for authentication\n\n\nswanlab_log_completions\nbool\ntrue\nEnable RLHF completion table logging (DPO/KTO/ORPO/GRPO)\n\n\nswanlab_completion_log_interval\nint\n100\nSteps between completion logging\n\n\nswanlab_completion_max_buffer\nint\n128\nMax completions to buffer (memory bound)\n\n\n\n\n\nConfiguration Examples\n\n\nExample 1: Basic Cloud Sync\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: llama-finetune\nswanlab_experiment_name: llama-3-8b-instruct-v1\nswanlab_mode: cloud\n\n\nExample 2: Offline/Local Mode\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: local-experiments\nswanlab_experiment_name: test-run-1\nswanlab_mode: local # or 'offline'\n\n\nExample 3: Team Workspace\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: research-project\nswanlab_experiment_name: experiment-42\nswanlab_workspace: my-research-team\nswanlab_mode: cloud\n\n\nExample 4: Private Deployment\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: internal-project\nswanlab_experiment_name: secure-training\nswanlab_mode: cloud\nswanlab_web_host: https://swanlab.yourcompany.com\nswanlab_api_host: https://api.swanlab.yourcompany.com\n\n\nTeam Notifications with Lark (Feishu)\nSwanLab supports sending real-time training notifications to your team chat via Lark (Feishu), ByteDance’s enterprise collaboration platform. This is especially useful for:\n- Production training monitoring: Get alerts when training starts, completes, or encounters errors\n- Team collaboration: Keep your ML team informed about long-running experiments\n- Multi-timezone teams: Team members can check training progress without being online\n\n\nPrerequisites\n\nLark Bot Setup: Create a custom bot in your Lark group chat\nWebhook URL: Get the webhook URL from your Lark bot settings\nHMAC Secret (recommended): Enable signature verification in your Lark bot for security\n\nFor detailed Lark bot setup instructions, see Lark Custom Bot Documentation.\n\n\nExample 5: Basic Lark Notifications\nSend training notifications to a Lark group chat:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: production-training\nswanlab_experiment_name: llama-3-finetune-v2\nswanlab_mode: cloud\n\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxx\nNote: This configuration will work, but you’ll see a security warning recommending HMAC secret configuration.\n\n\nExample 6: Lark Notifications with HMAC Security (Recommended)\nFor production use, enable HMAC signature verification:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: production-training\nswanlab_experiment_name: llama-3-finetune-v2\nswanlab_mode: cloud\n\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxx\nswanlab_lark_secret: your-webhook-secret-key\nWhy HMAC secret matters:\n- Prevents unauthorized parties from sending fake notifications to your Lark group\n- Ensures notifications genuinely come from your training jobs\n- Required for production deployments with sensitive training data\n\n\nExample 7: Team Workspace + Lark Notifications\nCombine team workspace collaboration with Lark notifications:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: research-project\nswanlab_experiment_name: multimodal-experiment-42\nswanlab_workspace: ml-research-team\nswanlab_mode: cloud\n\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxx\nswanlab_lark_secret: your-webhook-secret-key\n\n\nWhat Notifications Are Sent?\nSwanLab’s Lark integration sends notifications for key training events:\n- Training Start: When your experiment begins\n- Training Complete: When training finishes successfully\n- Training Errors: If training crashes or encounters critical errors\n- Metric Milestones: Configurable alerts for metric thresholds (if configured in SwanLab)\nEach notification includes:\n- Experiment name and project\n- Training status\n- Key metrics (loss, learning rate)\n- Direct link to SwanLab dashboard\n\n\nLark Configuration Validation\nThe plugin validates your Lark configuration at startup:\n\n✅ Valid Configurations\nuse_swanlab: true\nswanlab_project: my-project\n\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxx\nswanlab_lark_secret: your-secret\n\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxx\n\n\n\nSecurity Best Practices\n\nAlways use HMAC secret in production:\nswanlab_lark_webhook_url: https://open.feishu.cn/...\nswanlab_lark_secret: your-secret-key # ✅ Add this!\nStore secrets in environment variables (even better):\n# In your training script/environment\nexport SWANLAB_LARK_WEBHOOK_URL=\"https://open.feishu.cn/...\"\nexport SWANLAB_LARK_SECRET=\"your-secret-key\"\nThen in config:\n# SwanLab plugin will auto-detect environment variables\nuse_swanlab: true\nswanlab_project: my-project\n# Lark URL and secret read from env vars\nRotate webhook secrets periodically: Update your Lark bot’s secret every 90 days\nUse separate webhooks for dev/prod: Don’t mix development and production notifications\n\n\n\nDistributed Training\nLark notifications are automatically deduplicated in distributed training:\n- Only rank 0 sends notifications\n- Other GPU ranks skip Lark registration\n- Prevents duplicate messages in multi-GPU training\ntorchrun --nproc_per_node=4 -m axolotl.cli.train config.yml\n\n\nRLHF Completion Table Logging\nFor RLHF (Reinforcement Learning from Human Feedback) training methods like DPO, KTO, ORPO, and GRPO, SwanLab can log model completions (prompts, chosen/rejected responses, rewards) to a visual table for qualitative analysis. This helps you:\n\nInspect model behavior: See actual model outputs during training\nDebug preference learning: Compare chosen vs rejected responses\nTrack reward patterns: Monitor how rewards evolve over training\nShare examples with team: Visual tables in SwanLab dashboard\n\n\n\nFeatures\n\n✅ Automatic detection: Works with DPO, KTO, ORPO, GRPO trainers\n✅ Memory-safe buffering: Bounded buffer prevents memory leaks in long training runs\n✅ Periodic logging: Configurable logging interval to reduce overhead\n✅ Rich visualization: SwanLab tables show prompts, responses, and metrics side-by-side\n\n\n\nConfiguration\n\n\n\n\n\n\n\n\n\nParameter\nType\nDefault\nDescription\n\n\n\n\nswanlab_log_completions\nbool\ntrue\nEnable completion logging for RLHF trainers\n\n\nswanlab_completion_log_interval\nint\n100\nLog completions to SwanLab every N training steps\n\n\nswanlab_completion_max_buffer\nint\n128\nMaximum completions to buffer (memory bound)\n\n\n\n\n\nExample: DPO Training with Completion Logging\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: dpo-training\nswanlab_experiment_name: llama-3-dpo-v1\nswanlab_mode: cloud\n\nswanlab_log_completions: true\nswanlab_completion_log_interval: 100 # Log every 100 steps\nswanlab_completion_max_buffer: 128 # Keep last 128 completions\n\nrl: dpo\ndatasets:\n - path: /path/to/preference_dataset\n type: chatml.intel\n\n\nExample: Disable Completion Logging\nIf you’re doing a quick test run or don’t need completion tables:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: dpo-training\n\nswanlab_log_completions: false\n\n\nSupported RLHF Trainers\nThe completion logging callback automatically activates for these trainer types:\n\nDPO (Direct Preference Optimization): Logs prompts, chosen, rejected, reward_diff\nKTO (Kahneman-Tversky Optimization): Logs prompts, completions, labels, rewards\nORPO (Odds Ratio Preference Optimization): Logs prompts, chosen, rejected, log_odds_ratio\nGRPO (Group Relative Policy Optimization): Logs prompts, completions, rewards, advantages\nCPO (Constrained Policy Optimization): Logs prompts, chosen, rejected\n\nFor non-RLHF trainers (standard supervised fine-tuning), the completion callback is automatically skipped.\n\n\nHow It Works\n\nAuto-detection: Plugin detects trainer type at initialization\nBuffering: Completions are buffered in memory (up to swanlab_completion_max_buffer)\nPeriodic logging: Every swanlab_completion_log_interval steps, buffer is logged to SwanLab\nMemory safety: Old completions are automatically dropped when buffer is full (uses collections.deque)\nFinal flush: Remaining completions are logged when training completes\n\n\n\nViewing Completion Tables\nAfter training starts, you can view completion tables in your SwanLab dashboard:\n\nNavigate to your experiment in SwanLab\nLook for the “rlhf_completions” table in the metrics panel\nThe table shows:\n\nstep: Training step when completion was generated\nprompt: Input prompt\nchosen: Preferred response (DPO/ORPO)\nrejected: Non-preferred response (DPO/ORPO)\ncompletion: Model output (KTO/GRPO)\nreward_diff/reward: Reward metrics\nTrainer-specific metrics (e.g., log_odds_ratio for ORPO)\n\n\n\n\nMemory Management\nThe completion buffer is memory-bounded to prevent memory leaks:\nfrom collections import deque\n\nbuffer = deque(maxlen=128) # Old completions automatically dropped\nMemory usage estimate:\n- Average completion: ~500 characters (prompt + responses)\n- Buffer size 128: ~64 KB (negligible)\n- Buffer size 1024: ~512 KB (still small)\nRecommendation: Default buffer size (128) works well for most cases. Increase to 512-1024 only if you need to review more historical completions.\n\n\nPerformance Impact\nCompletion logging has minimal overhead:\n\nBuffering: O(1) append operation, negligible CPU/memory\nLogging: Only happens every N steps (default: 100)\nNetwork: SwanLab batches table uploads efficiently\n\nExpected overhead: < 0.5% per training step\n\n\nTroubleshooting\n\nCompletions not appearing in SwanLab\nCause: Trainer may not be logging completion data in the expected format.\nDiagnostic steps:\n1. Check trainer type detection in logs:\ntext INFO: SwanLab RLHF completion logging enabled for DPOTrainer (type: dpo)\n2. Verify your trainer is an RLHF trainer (DPO/KTO/ORPO/GRPO)\n3. Check if trainer logs completion data (this depends on TRL version)\nNote: The current implementation expects trainers to log completion data in the logs dict during on_log() callback. Some TRL trainers may not expose this data by default. You may need to patch the trainer to expose completions.\n\n\nBuffer fills up too quickly\nCause: High logging frequency with small buffer size.\nSolution: Increase buffer size or logging interval:\nswanlab_completion_log_interval: 200 # Log less frequently\nswanlab_completion_max_buffer: 512 # Larger buffer\n\n\nMemory usage growing over time\nCause: Buffer should be bounded, so this indicates a bug.\nSolution:\n1. Verify swanlab_completion_max_buffer is set\n2. Check SwanLab version is up to date\n3. Report issue with memory profiling data\n\n\n\nPerformance Profiling\nSwanLab integration includes profiling utilities to measure and log execution time of trainer methods. This helps you:\n\nIdentify bottlenecks: Find slow operations in your training loop\nOptimize performance: Track improvements after optimization changes\nMonitor distributed training: See per-rank timing differences\nDebug hangs: Detect methods that take unexpectedly long\n\n\n\nFeatures\n\n✅ Zero-config profiling: Automatic timing of key trainer methods\n✅ Decorator-based: Easy to add profiling to custom methods with @swanlab_profile\n✅ Context manager: Fine-grained profiling with swanlab_profiling_context()\n✅ Advanced filtering: ProfilingConfig for throttling and minimum duration thresholds\n✅ Exception-safe: Logs duration even if function raises an exception\n\n\n\nBasic Usage: Decorator\nAdd profiling to any trainer method with the @swanlab_profile decorator:\nfrom axolotl.integrations.swanlab.profiling import swanlab_profile\n\nclass MyCustomTrainer(AxolotlTrainer):\n @swanlab_profile\n def training_step(self, model, inputs):\n # Your training step logic\n return super().training_step(model, inputs)\n\n @swanlab_profile\n def prediction_step(self, model, inputs, prediction_loss_only):\n # Your prediction logic\n return super().prediction_step(model, inputs, prediction_loss_only)\nThe decorator automatically:\n1. Measures execution time with high-precision timer\n2. Logs to SwanLab as profiling/Time taken: ClassName.method_name\n3. Only logs if SwanLab is enabled (use_swanlab: true)\n4. Gracefully handles exceptions (logs duration, then re-raises)\n\n\nAdvanced Usage: Context Manager\nFor fine-grained profiling within a method:\nfrom axolotl.integrations.swanlab.profiling import swanlab_profiling_context\n\nclass MyTrainer(AxolotlTrainer):\n def complex_training_step(self, model, inputs):\n # Profile just the forward pass\n with swanlab_profiling_context(self, \"forward_pass\"):\n outputs = model(**inputs)\n\n # Profile just the backward pass\n with swanlab_profiling_context(self, \"backward_pass\"):\n loss = outputs.loss\n loss.backward()\n\n return outputs\n\n\nAdvanced Usage: ProfilingConfig\nFilter and throttle profiling logs with ProfilingConfig:\nfrom axolotl.integrations.swanlab.profiling import (\n swanlab_profiling_context_advanced,\n ProfilingConfig,\n)\n\nprofiling_config = ProfilingConfig(\n enabled=True,\n min_duration_ms=1.0, # Only log if duration > 1ms\n log_interval=10, # Log every 10th call\n)\n\nclass MyTrainer(AxolotlTrainer):\n def frequently_called_method(self, data):\n with swanlab_profiling_context_advanced(\n self,\n \"frequent_op\",\n config=profiling_config\n ):\n # This only logs every 10th call, and only if it takes > 1ms\n result = expensive_computation(data)\n return result\nProfilingConfig Parameters:\n- enabled: Enable/disable profiling globally (default: True)\n- min_duration_ms: Minimum duration to log in milliseconds (default: 0.1)\n- log_interval: Log every Nth function call (default: 1 = log all)\nUse cases:\n- High-frequency methods: Use log_interval=100 to reduce logging overhead\n- Filter noise: Use min_duration_ms=1.0 to skip very fast operations\n- Debugging: Use log_interval=1, min_duration_ms=0.0 to log everything\n\n\nViewing Profiling Metrics\nIn your SwanLab dashboard, profiling metrics appear under the “profiling” namespace:\nprofiling/Time taken: AxolotlTrainer.training_step\nprofiling/Time taken: AxolotlTrainer.prediction_step\nprofiling/Time taken: MyTrainer.forward_pass\nprofiling/Time taken: MyTrainer.backward_pass\nYou can:\n- Track over time: See if methods get faster/slower during training\n- Compare runs: Compare profiling metrics across experiments\n- Identify regressions: Detect if a code change slowed down training\n\n\nConfiguration in Axolotl Config\nProfiling is automatically enabled when SwanLab is enabled. No additional config needed:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: my-project\nTo disable profiling while keeping SwanLab enabled:\nfrom axolotl.integrations.swanlab.profiling import DEFAULT_PROFILING_CONFIG\n\nDEFAULT_PROFILING_CONFIG.enabled = False\n\n\nPerformance Impact\n\nDecorator overhead: ~2-5 microseconds per call (negligible)\nContext manager overhead: ~1-3 microseconds (negligible)\nLogging overhead: Only when SwanLab is enabled and method duration exceeds threshold\nNetwork overhead: SwanLab batches metrics efficiently\n\nExpected overhead: < 0.1% per training step (effectively zero)\n\n\nBest Practices\n\nProfile bottlenecks first: Start by profiling suspected slow operations\nUse min_duration_ms: Filter out fast operations (< 1ms) to reduce noise\nThrottle high-frequency calls: Use log_interval for methods called > 100 times/step\nProfile across runs: Compare profiling metrics before/after optimization\nMonitor distributed training: Check for rank-specific slowdowns\n\n\n\nExample: Complete Profiling Setup\nfrom axolotl.integrations.swanlab.profiling import (\n swanlab_profile,\n swanlab_profiling_context,\n ProfilingConfig,\n)\n\nclass OptimizedTrainer(AxolotlTrainer):\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n\n # Custom profiling config for high-frequency operations\n self.fast_op_config = ProfilingConfig(\n enabled=True,\n min_duration_ms=0.5,\n log_interval=50,\n )\n\n @swanlab_profile\n def training_step(self, model, inputs):\n \"\"\"Main training step - always profile.\"\"\"\n return super().training_step(model, inputs)\n\n @swanlab_profile\n def compute_loss(self, model, inputs, return_outputs=False):\n \"\"\"Loss computation - always profile.\"\"\"\n return super().compute_loss(model, inputs, return_outputs)\n\n def _prepare_inputs(self, inputs):\n \"\"\"High-frequency operation - throttled profiling.\"\"\"\n with swanlab_profiling_context_advanced(\n self,\n \"prepare_inputs\",\n config=self.fast_op_config,\n ):\n return super()._prepare_inputs(inputs)\n\n\nTroubleshooting\n\nProfiling metrics not appearing in SwanLab\nCause: SwanLab is not enabled or not initialized.\nSolution:\nuse_swanlab: true\nswanlab_project: my-project\nCheck logs for:\nINFO: SwanLab initialized for project: my-project\n\n\nToo many profiling metrics cluttering dashboard\nCause: Profiling every function call for high-frequency operations.\nSolution: Use ProfilingConfig with throttling:\nconfig = ProfilingConfig(\n min_duration_ms=1.0, # Skip fast ops\n log_interval=100, # Log every 100th call\n)\n\n\nProfiling overhead impacting training speed\nCause: Profiling itself should have negligible overhead (< 0.1%). If you see > 1% slowdown, this indicates a bug.\nSolution:\n1. Disable profiling temporarily to confirm:\npython DEFAULT_PROFILING_CONFIG.enabled = False\n2. Report issue with profiling data and trainer details\n\n\nProfiling shows inconsistent timing\nCause: Normal variation due to GPU warmup, data loading, or system load.\nSolution:\n- Ignore first few steps (warmup period)\n- Look at average/median timing over many steps\n- Use log_interval to reduce noise from individual outliers\n\n\n\nComplete Config Example\nHere’s a complete example integrating SwanLab with your RVQ-Alpha training:\nbase_model: /path/to/your/model\nmodel_type: Qwen2ForCausalLM\n\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin\n\nuse_swanlab: true\nswanlab_project: RVQ-Alpha-Training\nswanlab_experiment_name: Qwen2.5-7B-MetaQA-Perturb-P020\nswanlab_description: \"Training on MetaQA and Perturbation datasets with NEW-RVQ encoding\"\nswanlab_mode: cloud\nswanlab_workspace: single-cell-genomics\n\nsequence_len: 32768\nmicro_batch_size: 1\ngradient_accumulation_steps: 1\nnum_epochs: 2\nlearning_rate: 2e-5\noptimizer: adamw_torch_fused\n\ndatasets:\n - path: /path/to/dataset\n type: chat_template\n\noutput_dir: ./outputs\n\n\nModes Explained\n\n\ncloud Mode (Default)\n\nSyncs experiments to SwanLab cloud in real-time\nRequires API key and internet connection\nBest for: Team collaboration, remote monitoring\n\n\n\nlocal Mode\n\nSaves experiments locally only\nNo cloud sync\nBest for: Local development, air-gapped environments\n\n\n\noffline Mode\n\nSaves metadata locally\nCan sync to cloud later using swanlab sync\nBest for: Unstable internet, sync later\n\n\n\ndisabled Mode\n\nTurns off SwanLab completely\nNo logging or tracking\nBest for: Debugging, testing\n\n\n\nConfiguration Validation & Conflict Detection\nSwanLab integration includes comprehensive validation and conflict detection to help you catch configuration errors early and avoid performance issues.\n\n\nRequired Fields Validation\nThe plugin validates your configuration at startup and provides clear error messages with solutions:\n\nMissing Project Name\nuse_swanlab: true\nSolution:\nuse_swanlab: true\nswanlab_project: my-project\n\n\nInvalid Mode\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_mode: invalid-mode\nSolution:\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_mode: cloud # or: local, offline, disabled\n\n\nEmpty Project Name\nuse_swanlab: true\nswanlab_project: \"\"\nSolution:\nuse_swanlab: true\nswanlab_project: my-project\n\n\n\nCloud Mode API Key Warning\nWhen using cloud mode without an API key, you’ll receive a warning with multiple solutions:\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_mode: cloud\nSolutions:\n1. Set environment variable: export SWANLAB_API_KEY=your-api-key\n2. Add to config (less secure): swanlab_api_key: your-api-key\n3. Run swanlab login before training\n4. Use swanlab_mode: local for offline tracking\n\n\nMulti-Logger Performance Warnings\nUsing multiple logging tools simultaneously (SwanLab + WandB + MLflow + Comet) can impact training performance:\n\nTwo Loggers - Warning\nuse_swanlab: true\nswanlab_project: my-project\n\nuse_wandb: true\nwandb_project: my-project\nImpact:\n- Performance overhead: ~1-2% per logger (cumulative)\n- Increased memory usage\n- Longer training time per step\n- Potential config/callback conflicts\nRecommendations:\n- Choose ONE primary logging tool for production training\n- Use multiple loggers only for:\n- Migration period (transitioning between tools)\n- Short comparison runs\n- Debugging specific tool issues\n- Monitor system resources (CPU, memory) during training\n\n\nThree+ Loggers - Error-Level Warning\nuse_swanlab: true\nswanlab_project: my-project\n\nuse_wandb: true\nwandb_project: my-project\n\nuse_mlflow: true\nmlflow_tracking_uri: http://localhost:5000\nWhy This Matters:\n- With 3 loggers: ~4-5% overhead per step → significant slowdown over long training\n- Example: 10,000 steps at 2s/step → ~400-500 seconds extra (6-8 minutes)\n- Memory overhead scales with number of loggers\n- Rare edge cases with callback ordering conflicts\n\n\n\nAuto-Enable Logic\nFor convenience, SwanLab will auto-enable if you specify a project without setting use_swanlab:\nswanlab_project: my-project\n\nuse_swanlab: true\nswanlab_project: my-project\n\n\nDistributed Training Detection\nIn distributed training scenarios (multi-GPU), the plugin automatically detects and reports:\nuse_swanlab: true\nswanlab_project: my-project\nswanlab_mode: cloud\nWhy Only Rank 0:\n- Avoids duplicate experiment runs\n- Reduces network/cloud API overhead on worker ranks\n- Prevents race conditions in metric logging\n\n\nAuthentication\n\n\nMethod 1: Environment Variable (Recommended)\nexport SWANLAB_API_KEY=your-api-key-here\n\n\nMethod 2: Login Command\nswanlab login\n\n\nMethod 3: Config File\nswanlab_api_key: your-api-key-here\n\n\nWhat Gets Logged?\n\n\nAutomatically Logged Metrics\n\nTraining loss\nLearning rate\nGradient norm\nTraining steps\nEpoch progress\n\n\n\nAutomatically Logged Config\n\nModel configuration (base_model, model_type)\nTraining hyperparameters (learning_rate, batch_size, etc.)\nOptimizer settings\nParallelization settings (FSDP, DeepSpeed, Context Parallel)\nAxolotl configuration file\nDeepSpeed configuration (if used)\n\n\n\nViewing Your Experiments\n\n\nCloud Mode\nVisit https://swanlab.cn and navigate to your project to view:\n- Real-time training metrics\n- Hyperparameter comparison\n- System resource usage\n- Configuration files\n\n\nLocal Mode\nswanlab watch ./swanlog\n\n\nIntegration with Existing Tools\nSwanLab can work alongside other tracking tools:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin\n\nuse_swanlab: true\nswanlab_project: my-project\n\nuse_wandb: true\nwandb_project: my-project\n\n\nTroubleshooting\n\n\nConfiguration Errors\n\nError: “SwanLab enabled but ‘swanlab_project’ is not set”\nCause: You enabled SwanLab (use_swanlab: true) but forgot to specify a project name.\nSolution:\nuse_swanlab: true\nswanlab_project: my-project # Add this line\n\n\nError: “Invalid swanlab_mode: ‘xxx’”\nCause: You provided an invalid mode value.\nSolution: Use one of the valid modes:\nswanlab_mode: cloud # or: local, offline, disabled\n\n\nError: “swanlab_project cannot be an empty string”\nCause: You set swanlab_project: \"\" (empty string).\nSolution: Either provide a valid name or remove the field:\nswanlab_project: my-project\n\n\n\nImport Errors\n\nError: “SwanLab is not installed”\nCause: SwanLab package is not installed in your environment.\nSolution:\npip install swanlab\npip install swanlab>=0.3.0\n\n\n\nPerformance Issues\n\nWarning: “Multiple logging tools enabled”\nCause: You have multiple experiment tracking tools enabled (e.g., SwanLab + WandB + MLflow).\nImpact: ~1-2% performance overhead per logger, cumulative.\nSolution: For production training, disable all but one logger:\nuse_swanlab: true\nswanlab_project: my-project\nuse_wandb: false # Disable others\nuse_mlflow: false\n\nuse_swanlab: false\nuse_wandb: true\nwandb_project: my-project\nException: Multiple loggers are acceptable for:\n- Short comparison runs (< 100 steps)\n- Migration testing between logging tools\n- Debugging logger-specific issues\n\n\n\nDistributed Training Issues\n\nSwanLab creates duplicate runs in multi-GPU training\nCause: All ranks are initializing SwanLab instead of just rank 0.\nExpected Behavior: The plugin automatically ensures only rank 0 initializes SwanLab. You should see:\nInfo: Distributed training detected (world_size=4)\nInfo: Only rank 0 will initialize SwanLab\nInfo: Other ranks will skip SwanLab to avoid conflicts\nIf you see duplicates:\n1. Check your plugin is loaded correctly\n2. Verify you’re using the latest SwanLab integration code\n3. Check logs for initialization messages on all ranks\n\n\n\nSwanLab not logging metrics\nSolution: Ensure SwanLab is initialized before training starts. The plugin automatically handles this in pre_model_load.\n\n\nAPI Key errors\nSolution:\necho $SWANLAB_API_KEY\n\nswanlab login\n\n\nCloud sync issues\nSolution: Use offline mode and sync later:\nswanlab_mode: offline\nThen sync when ready:\nswanlab sync ./swanlog\n\n\nPlugin not loaded\nSolution: Verify plugin path in config:\nplugins:\n - axolotl.integrations.swanlab.SwanLabPlugin # Correct path\n\n\nLark Notification Issues\n\nError: “Failed to import SwanLab Lark plugin”\nCause: Your SwanLab version doesn’t include the Lark plugin (requires SwanLab >= 0.3.0).\nSolution:\npip install --upgrade swanlab\n\npip install 'swanlab>=0.3.0'\n\n\nWarning: “Lark webhook has no secret configured”\nCause: You provided swanlab_lark_webhook_url but no swanlab_lark_secret.\nImpact: Lark notifications will work, but without HMAC authentication (security risk).\nSolution: Add HMAC secret for production use:\nswanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxx\nswanlab_lark_secret: your-webhook-secret # Add this line\nWhen it’s OK to skip secret:\n- Local development and testing\n- Internal networks with restricted access\n- Non-sensitive training experiments\nWhen secret is required:\n- Production training jobs\n- Training with proprietary data\n- Multi-team shared Lark groups\n\n\nError: “Failed to register Lark callback”\nCause: Invalid webhook URL or network connectivity issues.\nDiagnostic steps:\ncurl -X POST \"YOUR_WEBHOOK_URL\" \\\n -H 'Content-Type: application/json' \\\n -d '{\"msg_type\":\"text\",\"content\":{\"text\":\"Test from Axolotl\"}}'\n\npip show swanlab\nSolution:\n1. Verify webhook URL is correct (copy from Lark bot settings)\n2. Check network connectivity to Lark API\n3. Ensure webhook is not expired (Lark webhooks can expire)\n4. Regenerate webhook URL in Lark bot settings if needed\n\n\nLark notifications not received\nCause: Multiple possible causes.\nDiagnostic checklist:\n\nCheck training logs for Lark registration confirmation:\n# Expected log message (rank 0 only):\nINFO: Registered Lark notification callback with HMAC authentication\nVerify webhook in Lark: Test webhook manually (see above)\nCheck distributed training: Only rank 0 sends notifications\n# If running multi-GPU, check rank 0 logs specifically\ngrep \"Registered Lark\" logs/rank_0.log\nVerify SwanLab is initialized: Lark callback needs SwanLab to be running\nuse_swanlab: true # Must be enabled\nswanlab_project: my-project # Must be set\nCheck Lark bot permissions: Ensure bot is added to the target group chat\n\n\n\nDuplicate Lark notifications in multi-GPU training\nExpected Behavior: Should NOT happen - only rank 0 sends notifications.\nIf you see duplicates:\n1. Check that all GPUs are using the same config file\n2. Verify plugin is loaded correctly on all ranks\n3. Check logs for unexpected Lark initialization on non-zero ranks\n4. Ensure RANK or LOCAL_RANK environment variables are set correctly\nSolution: This is a bug if it occurs. Report with:\n- Full training command\n- Logs from all ranks\n- Config file\n\n\n\nComparison: SwanLab vs WandB\n\n\n\nFeature\nSwanLab\nWandB\n\n\n\n\nOpen Source\n✅ Yes\n❌ No\n\n\nSelf-Hosting\n✅ Easy\n⚠️ Complex\n\n\nFree Tier\n✅ Generous\n⚠️ Limited\n\n\nChinese Support\n✅ Native\n⚠️ Limited\n\n\nOffline Mode\n✅ Full support\n✅ Supported\n\n\nIntegration\n🆕 New\n✅ Mature\n\n\n\n\n\nAdvanced Usage\n\n\nCustom Logging\nYou can add custom metrics in your callbacks:\nimport swanlab\n\nswanlab.log({\n \"custom_metric\": value,\n \"epoch\": epoch_num\n})\n\n\nExperiment Comparison\nswanlab compare run1 run2 run3\n\n\nSupport\n\nDocumentation: https://docs.swanlab.cn\nGitHub: https://github.com/SwanHubX/SwanLab\nIssues: Report bugs at GitHub Issues\n\n\n\nLicense\nThis integration follows the Axolotl Community License Agreement.\n\n\nAcknowledgements\nThis integration is built on top of:\n- SwanLab - Experiment tracking tool\n- Transformers - SwanLabCallback\n- Axolotl - Training framework\nPlease see reference here", + "crumbs": [ + "Advanced Features", + "Custom Integrations" + ] + }, { "objectID": "docs/custom_integrations.html#adding-a-new-integration", "href": "docs/custom_integrations.html#adding-a-new-integration", diff --git a/sitemap.xml b/sitemap.xml index d481490c6..713f4a799 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -2,942 +2,942 @@ https://docs.axolotl.ai/src/axolotl/integrations/cut_cross_entropy/ACKNOWLEDGEMENTS.html - 2026-01-05T20:25:02.718Z + 2026-01-06T14:20:21.726Z https://docs.axolotl.ai/docs/mac.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.701Z https://docs.axolotl.ai/docs/cli.html - 2026-01-05T20:25:02.687Z + 2026-01-06T14:20:21.697Z https://docs.axolotl.ai/docs/mixed_precision.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.701Z https://docs.axolotl.ai/docs/installation.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.701Z https://docs.axolotl.ai/docs/dataset_loading.html - 2026-01-05T20:25:02.687Z + 2026-01-06T14:20:21.698Z https://docs.axolotl.ai/docs/sequence_parallelism.html - 2026-01-05T20:25:02.692Z + 2026-01-06T14:20:21.702Z https://docs.axolotl.ai/docs/optimizations.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.702Z https://docs.axolotl.ai/docs/gradient_checkpointing.html - 2026-01-05T20:25:02.688Z + 2026-01-06T14:20:21.698Z https://docs.axolotl.ai/docs/streaming.html - 2026-01-05T20:25:02.692Z + 2026-01-06T14:20:21.702Z https://docs.axolotl.ai/docs/lora_optims.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.701Z https://docs.axolotl.ai/docs/amd_hpc.html - 2026-01-05T20:25:02.687Z + 2026-01-06T14:20:21.697Z https://docs.axolotl.ai/docs/debugging.html - 2026-01-05T20:25:02.688Z + 2026-01-06T14:20:21.698Z https://docs.axolotl.ai/docs/dataset-formats/conversation.html - 2026-01-05T20:25:02.687Z + 2026-01-06T14:20:21.698Z https://docs.axolotl.ai/docs/dataset-formats/inst_tune.html - 2026-01-05T20:25:02.687Z + 2026-01-06T14:20:21.698Z https://docs.axolotl.ai/docs/dataset-formats/index.html - 2026-01-05T20:25:02.687Z + 2026-01-06T14:20:21.698Z https://docs.axolotl.ai/docs/config-reference.html - 2026-01-05T20:28:43.787Z + 2026-01-06T14:24:05.745Z https://docs.axolotl.ai/docs/multimodal.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.701Z https://docs.axolotl.ai/docs/ray-integration.html - 2026-01-05T20:25:02.692Z + 2026-01-06T14:20:21.702Z https://docs.axolotl.ai/docs/faq.html - 2026-01-05T20:25:02.688Z + 2026-01-06T14:20:21.698Z https://docs.axolotl.ai/docs/dataset_preprocessing.html - 2026-01-05T20:25:02.687Z + 2026-01-06T14:20:21.698Z https://docs.axolotl.ai/docs/torchao.html - 2026-01-05T20:25:02.692Z + 2026-01-06T14:20:21.702Z https://docs.axolotl.ai/docs/optimizers.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.702Z https://docs.axolotl.ai/docs/models/apertus.html - 2026-01-05T20:28:44.241Z + 2026-01-06T14:24:06.292Z https://docs.axolotl.ai/docs/models/ministral3/think.html - 2026-01-05T20:28:44.236Z + 2026-01-06T14:24:06.286Z https://docs.axolotl.ai/docs/models/gpt-oss.html - 2026-01-05T20:28:44.242Z + 2026-01-06T14:24:06.292Z https://docs.axolotl.ai/docs/models/phi.html - 2026-01-05T20:28:44.243Z + 2026-01-06T14:24:06.292Z https://docs.axolotl.ai/docs/models/olmo3.html - 2026-01-05T20:28:44.234Z + 2026-01-06T14:24:06.285Z https://docs.axolotl.ai/docs/models/granite4.html - 2026-01-05T20:28:44.243Z + 2026-01-06T14:24:06.293Z https://docs.axolotl.ai/docs/models/seed-oss.html - 2026-01-05T20:28:44.242Z + 2026-01-06T14:24:06.292Z https://docs.axolotl.ai/docs/models/qwen3.html - 2026-01-05T20:28:44.241Z + 2026-01-06T14:24:06.291Z https://docs.axolotl.ai/docs/models/orpheus.html - 2026-01-05T20:28:44.245Z + 2026-01-06T14:24:06.294Z https://docs.axolotl.ai/docs/models/hunyuan.html - 2026-01-05T20:28:44.244Z + 2026-01-06T14:24:06.294Z https://docs.axolotl.ai/docs/models/mistral.html - 2026-01-05T20:28:44.239Z + 2026-01-06T14:24:06.289Z https://docs.axolotl.ai/docs/models/mistral-small.html - 2026-01-05T20:28:44.238Z + 2026-01-06T14:24:06.288Z https://docs.axolotl.ai/docs/models/smolvlm2.html - 2026-01-05T20:28:44.243Z + 2026-01-06T14:24:06.293Z https://docs.axolotl.ai/docs/models/llama-2.html - 2026-01-05T20:28:44.240Z + 2026-01-06T14:24:06.290Z https://docs.axolotl.ai/docs/models/magistral/vision.html - 2026-01-05T20:28:44.238Z + 2026-01-06T14:24:06.288Z https://docs.axolotl.ai/docs/models/jamba.html - 2026-01-05T20:28:44.245Z + 2026-01-06T14:24:06.294Z https://docs.axolotl.ai/docs/models/mimo.html - 2026-01-05T20:28:44.233Z + 2026-01-06T14:24:06.284Z https://docs.axolotl.ai/docs/api/utils.schedulers.html - 2026-01-05T20:28:28.364Z + 2026-01-06T14:23:49.699Z https://docs.axolotl.ai/docs/api/cli.utils.sweeps.html - 2026-01-05T20:28:27.549Z + 2026-01-06T14:23:48.878Z https://docs.axolotl.ai/docs/api/datasets.html - 2026-01-05T20:28:27.139Z + 2026-01-06T14:23:48.449Z https://docs.axolotl.ai/docs/api/utils.tokenization.html - 2026-01-05T20:28:28.280Z + 2026-01-06T14:23:49.614Z https://docs.axolotl.ai/docs/api/loaders.tokenizer.html - 2026-01-05T20:28:27.667Z + 2026-01-06T14:23:48.998Z https://docs.axolotl.ai/docs/api/monkeypatch.llama_expand_mask.html - 2026-01-05T20:28:28.151Z + 2026-01-06T14:23:49.484Z https://docs.axolotl.ai/docs/api/monkeypatch.gradient_checkpointing.offload_cpu.html - 2026-01-05T20:28:28.239Z + 2026-01-06T14:23:49.573Z https://docs.axolotl.ai/docs/api/utils.data.sft.html - 2026-01-05T20:28:28.415Z + 2026-01-06T14:23:49.751Z https://docs.axolotl.ai/docs/api/monkeypatch.transformers_fa_utils.html - 2026-01-05T20:28:28.219Z + 2026-01-06T14:23:49.553Z https://docs.axolotl.ai/docs/api/loaders.patch_manager.html - 2026-01-05T20:28:27.695Z + 2026-01-06T14:23:49.026Z https://docs.axolotl.ai/docs/api/integrations.liger.args.html - 2026-01-05T20:28:28.745Z + 2026-01-06T14:23:50.084Z https://docs.axolotl.ai/docs/api/utils.schemas.peft.html - 2026-01-05T20:28:28.506Z + 2026-01-06T14:23:49.844Z https://docs.axolotl.ai/docs/api/prompt_strategies.pygmalion.html - 2026-01-05T20:28:27.890Z + 2026-01-06T14:23:49.223Z https://docs.axolotl.ai/docs/api/prompt_strategies.alpaca_instruct.html - 2026-01-05T20:28:27.808Z + 2026-01-06T14:23:49.140Z https://docs.axolotl.ai/docs/api/cli.cloud.base.html - 2026-01-05T20:28:27.504Z + 2026-01-06T14:23:48.832Z https://docs.axolotl.ai/docs/api/monkeypatch.gradient_checkpointing.offload_disk.html - 2026-01-05T20:28:28.271Z + 2026-01-06T14:23:49.605Z https://docs.axolotl.ai/docs/api/kernels.swiglu.html - 2026-01-05T20:28:28.121Z + 2026-01-06T14:23:49.454Z https://docs.axolotl.ai/docs/api/integrations.cut_cross_entropy.args.html - 2026-01-05T20:28:28.731Z + 2026-01-06T14:23:50.069Z https://docs.axolotl.ai/docs/api/prompt_strategies.kto.user_defined.html - 2026-01-05T20:28:27.958Z + 2026-01-06T14:23:49.291Z https://docs.axolotl.ai/docs/api/monkeypatch.utils.html - 2026-01-05T20:28:28.196Z + 2026-01-06T14:23:49.530Z https://docs.axolotl.ai/docs/api/core.builders.rl.html - 2026-01-05T20:28:27.239Z + 2026-01-06T14:23:48.553Z https://docs.axolotl.ai/docs/api/loaders.processor.html - 2026-01-05T20:28:27.669Z + 2026-01-06T14:23:48.999Z https://docs.axolotl.ai/docs/api/utils.callbacks.lisa.html - 2026-01-05T20:28:28.876Z + 2026-01-06T14:23:50.218Z https://docs.axolotl.ai/docs/api/core.training_args.html - 2026-01-05T20:28:27.255Z + 2026-01-06T14:23:48.569Z https://docs.axolotl.ai/docs/api/loaders.adapter.html - 2026-01-05T20:28:27.676Z + 2026-01-06T14:23:49.006Z https://docs.axolotl.ai/docs/api/cli.merge_sharded_fsdp_weights.html - 2026-01-05T20:28:27.475Z + 2026-01-06T14:23:48.802Z https://docs.axolotl.ai/docs/api/cli.train.html - 2026-01-05T20:28:27.358Z + 2026-01-06T14:23:48.675Z https://docs.axolotl.ai/docs/api/core.trainers.mixins.rng_state_loader.html - 2026-01-05T20:28:27.708Z + 2026-01-06T14:23:49.039Z https://docs.axolotl.ai/docs/api/prompt_strategies.completion.html - 2026-01-05T20:28:27.856Z + 2026-01-06T14:23:49.188Z https://docs.axolotl.ai/docs/api/prompt_strategies.stepwise_supervised.html - 2026-01-05T20:28:27.869Z + 2026-01-06T14:23:49.202Z https://docs.axolotl.ai/docs/api/monkeypatch.lora_kernels.html - 2026-01-05T20:28:28.186Z + 2026-01-06T14:23:49.520Z https://docs.axolotl.ai/docs/api/prompt_strategies.messages.chat.html - 2026-01-05T20:28:27.896Z + 2026-01-06T14:23:49.229Z https://docs.axolotl.ai/docs/api/prompt_strategies.user_defined.html - 2026-01-05T20:28:27.832Z + 2026-01-06T14:23:49.165Z https://docs.axolotl.ai/docs/api/core.chat.messages.html - 2026-01-05T20:28:27.284Z + 2026-01-06T14:23:48.598Z https://docs.axolotl.ai/docs/api/core.trainers.mixins.scheduler.html - 2026-01-05T20:28:27.716Z + 2026-01-06T14:23:49.047Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.user_defined.html - 2026-01-05T20:28:27.934Z + 2026-01-06T14:23:49.267Z https://docs.axolotl.ai/docs/api/prompt_strategies.kto.llama3.html - 2026-01-05T20:28:27.946Z + 2026-01-06T14:23:49.279Z https://docs.axolotl.ai/docs/api/utils.schemas.integrations.html - 2026-01-05T20:28:28.537Z + 2026-01-06T14:23:49.875Z https://docs.axolotl.ai/docs/api/convert.html - 2026-01-05T20:28:27.156Z + 2026-01-06T14:23:48.466Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.passthrough.html - 2026-01-05T20:28:27.936Z + 2026-01-06T14:23:49.269Z https://docs.axolotl.ai/docs/api/utils.schemas.config.html - 2026-01-05T20:28:28.456Z + 2026-01-06T14:23:49.793Z https://docs.axolotl.ai/docs/api/utils.schemas.enums.html - 2026-01-05T20:28:28.548Z + 2026-01-06T14:23:49.886Z https://docs.axolotl.ai/docs/api/monkeypatch.btlm_attn_hijack_flash.html - 2026-01-05T20:28:28.198Z + 2026-01-06T14:23:49.532Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.chat_template.html - 2026-01-05T20:28:27.904Z + 2026-01-06T14:23:49.237Z https://docs.axolotl.ai/docs/api/core.trainers.grpo.trainer.html - 2026-01-05T20:28:27.628Z + 2026-01-06T14:23:48.958Z https://docs.axolotl.ai/docs/api/integrations.lm_eval.args.html - 2026-01-05T20:28:28.749Z + 2026-01-06T14:23:50.088Z https://docs.axolotl.ai/docs/api/utils.collators.core.html - 2026-01-05T20:28:28.779Z + 2026-01-06T14:23:50.118Z https://docs.axolotl.ai/docs/api/core.chat.format.shared.html - 2026-01-05T20:28:27.290Z + 2026-01-06T14:23:48.604Z https://docs.axolotl.ai/docs/api/prompt_strategies.orpo.chat_template.html - 2026-01-05T20:28:27.984Z + 2026-01-06T14:23:49.317Z https://docs.axolotl.ai/docs/api/utils.samplers.multipack.html - 2026-01-05T20:28:28.862Z + 2026-01-06T14:23:50.203Z https://docs.axolotl.ai/docs/api/utils.callbacks.qat.html - 2026-01-05T20:28:28.893Z + 2026-01-06T14:23:50.235Z https://docs.axolotl.ai/docs/api/prompt_strategies.chat_template.html - 2026-01-05T20:28:27.789Z + 2026-01-06T14:23:49.121Z https://docs.axolotl.ai/docs/api/utils.schemas.multimodal.html - 2026-01-05T20:28:28.517Z + 2026-01-06T14:23:49.855Z https://docs.axolotl.ai/docs/api/utils.callbacks.comet_.html - 2026-01-05T20:28:28.885Z + 2026-01-06T14:23:50.227Z https://docs.axolotl.ai/docs/api/prompt_strategies.base.html - 2026-01-05T20:28:27.747Z + 2026-01-06T14:23:49.079Z https://docs.axolotl.ai/docs/api/kernels.utils.html - 2026-01-05T20:28:28.132Z + 2026-01-06T14:23:49.465Z https://docs.axolotl.ai/docs/api/cli.merge_lora.html - 2026-01-05T20:28:27.461Z + 2026-01-06T14:23:48.786Z https://docs.axolotl.ai/docs/api/cli.utils.html - 2026-01-05T20:28:27.514Z + 2026-01-06T14:23:48.842Z https://docs.axolotl.ai/docs/api/utils.ctx_managers.sequence_parallel.html - 2026-01-05T20:28:27.745Z + 2026-01-06T14:23:49.077Z https://docs.axolotl.ai/docs/api/index.html - 2026-01-05T20:28:27.039Z + 2026-01-06T14:23:48.345Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.llama3.html - 2026-01-05T20:28:27.917Z + 2026-01-06T14:23:49.250Z https://docs.axolotl.ai/docs/api/monkeypatch.mixtral.html - 2026-01-05T20:28:28.234Z + 2026-01-06T14:23:49.568Z https://docs.axolotl.ai/docs/api/prompt_strategies.orcamini.html - 2026-01-05T20:28:27.882Z + 2026-01-06T14:23:49.215Z https://docs.axolotl.ai/docs/api/core.trainers.grpo.sampler.html - 2026-01-05T20:28:27.643Z + 2026-01-06T14:23:48.973Z https://docs.axolotl.ai/docs/api/utils.lora.html - 2026-01-05T20:28:28.288Z + 2026-01-06T14:23:49.622Z https://docs.axolotl.ai/docs/api/core.trainers.mixins.optimizer.html - 2026-01-05T20:28:27.704Z + 2026-01-06T14:23:49.035Z https://docs.axolotl.ai/docs/api/cli.config.html - 2026-01-05T20:28:27.427Z + 2026-01-06T14:23:48.748Z https://docs.axolotl.ai/docs/api/monkeypatch.multipack.html - 2026-01-05T20:28:28.145Z + 2026-01-06T14:23:49.478Z https://docs.axolotl.ai/docs/api/utils.collators.batching.html - 2026-01-05T20:28:28.802Z + 2026-01-06T14:23:50.142Z https://docs.axolotl.ai/docs/api/utils.quantization.html - 2026-01-05T20:28:28.440Z + 2026-01-06T14:23:49.775Z https://docs.axolotl.ai/docs/api/utils.dict.html - 2026-01-05T20:28:28.396Z + 2026-01-06T14:23:49.731Z https://docs.axolotl.ai/docs/api/kernels.quantize.html - 2026-01-05T20:28:28.131Z + 2026-01-06T14:23:49.464Z https://docs.axolotl.ai/docs/api/utils.schemas.training.html - 2026-01-05T20:28:28.473Z + 2026-01-06T14:23:49.810Z https://docs.axolotl.ai/docs/api/train.html - 2026-01-05T20:28:27.118Z + 2026-01-06T14:23:48.427Z https://docs.axolotl.ai/docs/api/core.datasets.transforms.chat_builder.html - 2026-01-05T20:28:27.306Z + 2026-01-06T14:23:48.620Z https://docs.axolotl.ai/docs/inference.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.701Z https://docs.axolotl.ai/FAQS.html - 2026-01-05T20:25:02.685Z + 2026-01-06T14:20:21.696Z https://docs.axolotl.ai/examples/colab-notebooks/colab-axolotl-example.html - 2026-01-05T20:25:02.696Z + 2026-01-06T14:20:21.707Z https://docs.axolotl.ai/index.html - 2026-01-05T20:25:02.712Z + 2026-01-06T14:20:21.721Z https://docs.axolotl.ai/docs/custom_integrations.html - 2026-01-05T20:25:02.687Z + 2026-01-06T14:20:21.697Z https://docs.axolotl.ai/docs/api/utils.schemas.utils.html - 2026-01-05T20:28:28.554Z + 2026-01-06T14:23:49.893Z https://docs.axolotl.ai/docs/api/kernels.geglu.html - 2026-01-05T20:28:28.109Z + 2026-01-06T14:23:49.442Z https://docs.axolotl.ai/docs/api/core.builders.causal.html - 2026-01-05T20:28:27.233Z + 2026-01-06T14:23:48.547Z https://docs.axolotl.ai/docs/api/core.trainers.mamba.html - 2026-01-05T20:28:27.606Z + 2026-01-06T14:23:48.936Z https://docs.axolotl.ai/docs/api/prompt_strategies.bradley_terry.llama3.html - 2026-01-05T20:28:27.989Z + 2026-01-06T14:23:49.322Z https://docs.axolotl.ai/docs/api/core.datasets.chat.html - 2026-01-05T20:28:27.296Z + 2026-01-06T14:23:48.610Z https://docs.axolotl.ai/docs/api/utils.collators.mm_chat.html - 2026-01-05T20:28:28.812Z + 2026-01-06T14:23:50.153Z https://docs.axolotl.ai/docs/api/prompt_strategies.llama2_chat.html - 2026-01-05T20:28:27.849Z + 2026-01-06T14:23:49.181Z https://docs.axolotl.ai/docs/api/common.const.html - 2026-01-05T20:28:28.757Z + 2026-01-06T14:23:50.096Z https://docs.axolotl.ai/docs/api/cli.quantize.html - 2026-01-05T20:28:27.491Z + 2026-01-06T14:23:48.819Z https://docs.axolotl.ai/docs/api/utils.trainer.html - 2026-01-05T20:28:28.330Z + 2026-01-06T14:23:49.665Z https://docs.axolotl.ai/docs/api/cli.delinearize_llama4.html - 2026-01-05T20:28:27.433Z + 2026-01-06T14:23:48.754Z https://docs.axolotl.ai/docs/api/evaluate.html - 2026-01-05T20:28:27.131Z + 2026-01-06T14:23:48.441Z https://docs.axolotl.ai/docs/api/monkeypatch.mistral_attn_hijack_flash.html - 2026-01-05T20:28:28.143Z + 2026-01-06T14:23:49.476Z https://docs.axolotl.ai/docs/api/loaders.model.html - 2026-01-05T20:28:27.657Z + 2026-01-06T14:23:48.987Z https://docs.axolotl.ai/docs/api/utils.distributed.html - 2026-01-05T20:28:28.389Z + 2026-01-06T14:23:49.725Z https://docs.axolotl.ai/docs/api/utils.model_shard_quant.html - 2026-01-05T20:28:28.295Z + 2026-01-06T14:23:49.629Z https://docs.axolotl.ai/docs/api/kernels.lora.html - 2026-01-05T20:28:28.096Z + 2026-01-06T14:23:49.429Z https://docs.axolotl.ai/docs/api/cli.main.html - 2026-01-05T20:28:27.348Z + 2026-01-06T14:23:48.664Z https://docs.axolotl.ai/docs/api/integrations.spectrum.args.html - 2026-01-05T20:28:28.753Z + 2026-01-06T14:23:50.092Z https://docs.axolotl.ai/docs/api/utils.optimizers.adopt.html - 2026-01-05T20:28:28.406Z + 2026-01-06T14:23:49.741Z https://docs.axolotl.ai/docs/api/cli.cloud.modal_.html - 2026-01-05T20:28:27.512Z + 2026-01-06T14:23:48.840Z https://docs.axolotl.ai/docs/api/monkeypatch.llama_attn_hijack_flash.html - 2026-01-05T20:28:28.139Z + 2026-01-06T14:23:49.472Z https://docs.axolotl.ai/docs/api/core.builders.base.html - 2026-01-05T20:28:27.228Z + 2026-01-06T14:23:48.541Z https://docs.axolotl.ai/docs/api/utils.schemas.trl.html - 2026-01-05T20:28:28.510Z + 2026-01-06T14:23:49.848Z https://docs.axolotl.ai/docs/api/cli.utils.args.html - 2026-01-05T20:28:27.528Z + 2026-01-06T14:23:48.856Z https://docs.axolotl.ai/docs/api/core.trainers.base.html - 2026-01-05T20:28:27.581Z + 2026-01-06T14:23:48.910Z https://docs.axolotl.ai/docs/api/monkeypatch.llama_patch_multipack.html - 2026-01-05T20:28:28.200Z + 2026-01-06T14:23:49.534Z https://docs.axolotl.ai/docs/api/monkeypatch.llama_attn_hijack_xformers.html - 2026-01-05T20:28:28.141Z + 2026-01-06T14:23:49.474Z https://docs.axolotl.ai/docs/api/utils.schemas.model.html - 2026-01-05T20:28:28.464Z + 2026-01-06T14:23:49.802Z https://docs.axolotl.ai/docs/api/prompt_strategies.kto.chatml.html - 2026-01-05T20:28:27.956Z + 2026-01-06T14:23:49.289Z https://docs.axolotl.ai/docs/api/utils.callbacks.mlflow_.html - 2026-01-05T20:28:28.881Z + 2026-01-06T14:23:50.222Z https://docs.axolotl.ai/docs/api/common.datasets.html - 2026-01-05T20:28:28.775Z + 2026-01-06T14:23:50.115Z https://docs.axolotl.ai/docs/api/utils.schemas.datasets.html - 2026-01-05T20:28:28.495Z + 2026-01-06T14:23:49.833Z https://docs.axolotl.ai/docs/api/cli.utils.fetch.html - 2026-01-05T20:28:27.535Z + 2026-01-06T14:23:48.863Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.chatml.html - 2026-01-05T20:28:27.930Z + 2026-01-06T14:23:49.263Z https://docs.axolotl.ai/docs/api/monkeypatch.relora.html - 2026-01-05T20:28:28.149Z + 2026-01-06T14:23:49.482Z https://docs.axolotl.ai/docs/api/cli.evaluate.html - 2026-01-05T20:28:27.368Z + 2026-01-06T14:23:48.685Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.zephyr.html - 2026-01-05T20:28:27.932Z + 2026-01-06T14:23:49.265Z https://docs.axolotl.ai/docs/api/core.trainers.utils.html - 2026-01-05T20:28:27.645Z + 2026-01-06T14:23:48.975Z https://docs.axolotl.ai/docs/api/prompt_strategies.alpaca_w_system.html - 2026-01-05T20:28:27.822Z + 2026-01-06T14:23:49.155Z https://docs.axolotl.ai/docs/api/utils.chat_templates.html - 2026-01-05T20:28:28.281Z + 2026-01-06T14:23:49.616Z https://docs.axolotl.ai/docs/api/utils.data.streaming.html - 2026-01-05T20:28:28.408Z + 2026-01-06T14:23:49.743Z https://docs.axolotl.ai/docs/api/utils.bench.html - 2026-01-05T20:28:28.299Z + 2026-01-06T14:23:49.633Z https://docs.axolotl.ai/docs/api/common.architectures.html - 2026-01-05T20:28:28.755Z + 2026-01-06T14:23:50.094Z https://docs.axolotl.ai/docs/api/cli.checks.html - 2026-01-05T20:28:27.405Z + 2026-01-06T14:23:48.722Z https://docs.axolotl.ai/docs/api/core.trainers.dpo.trainer.html - 2026-01-05T20:28:27.614Z + 2026-01-06T14:23:48.944Z https://docs.axolotl.ai/docs/api/integrations.base.html - 2026-01-05T20:28:28.726Z + 2026-01-06T14:23:50.064Z https://docs.axolotl.ai/docs/api/cli.utils.train.html - 2026-01-05T20:28:27.563Z + 2026-01-06T14:23:48.893Z https://docs.axolotl.ai/docs/api/utils.collators.mamba.html - 2026-01-05T20:28:28.806Z + 2026-01-06T14:23:50.147Z https://docs.axolotl.ai/docs/api/cli.art.html - 2026-01-05T20:28:27.397Z + 2026-01-06T14:23:48.714Z https://docs.axolotl.ai/docs/api/monkeypatch.trainer_fsdp_optim.html - 2026-01-05T20:28:28.211Z + 2026-01-06T14:23:49.545Z https://docs.axolotl.ai/docs/api/logging_config.html - 2026-01-05T20:28:27.220Z + 2026-01-06T14:23:48.533Z https://docs.axolotl.ai/docs/api/utils.freeze.html - 2026-01-05T20:28:28.309Z + 2026-01-06T14:23:49.643Z https://docs.axolotl.ai/docs/api/prompt_strategies.metharme.html - 2026-01-05T20:28:27.877Z + 2026-01-06T14:23:49.210Z https://docs.axolotl.ai/docs/api/prompt_strategies.alpaca_chat.html - 2026-01-05T20:28:27.806Z + 2026-01-06T14:23:49.138Z https://docs.axolotl.ai/docs/api/monkeypatch.stablelm_attn_hijack_flash.html - 2026-01-05T20:28:28.207Z + 2026-01-06T14:23:49.541Z https://docs.axolotl.ai/docs/api/models.mamba.modeling_mamba.html - 2026-01-05T20:28:28.777Z + 2026-01-06T14:23:50.116Z https://docs.axolotl.ai/docs/api/core.trainers.trl.html - 2026-01-05T20:28:27.599Z + 2026-01-06T14:23:48.929Z https://docs.axolotl.ai/docs/api/prompt_strategies.input_output.html - 2026-01-05T20:28:27.863Z + 2026-01-06T14:23:49.196Z https://docs.axolotl.ai/docs/api/loaders.constants.html - 2026-01-05T20:28:27.697Z + 2026-01-06T14:23:49.028Z https://docs.axolotl.ai/docs/api/monkeypatch.data.batch_dataset_fetcher.html - 2026-01-05T20:28:28.233Z + 2026-01-06T14:23:49.567Z https://docs.axolotl.ai/docs/api/cli.vllm_serve.html - 2026-01-05T20:28:27.500Z + 2026-01-06T14:23:48.827Z https://docs.axolotl.ai/docs/api/prompt_tokenizers.html - 2026-01-05T20:28:27.208Z + 2026-01-06T14:23:48.518Z https://docs.axolotl.ai/docs/api/cli.args.html - 2026-01-05T20:28:27.393Z + 2026-01-06T14:23:48.710Z https://docs.axolotl.ai/docs/api/cli.inference.html - 2026-01-05T20:28:27.450Z + 2026-01-06T14:23:48.772Z https://docs.axolotl.ai/docs/api/cli.utils.load.html - 2026-01-05T20:28:27.542Z + 2026-01-06T14:23:48.870Z https://docs.axolotl.ai/docs/api/cli.preprocess.html - 2026-01-05T20:28:27.485Z + 2026-01-06T14:23:48.813Z https://docs.axolotl.ai/docs/api/utils.callbacks.profiler.html - 2026-01-05T20:28:28.874Z + 2026-01-06T14:23:50.216Z https://docs.axolotl.ai/docs/api/utils.callbacks.perplexity.html - 2026-01-05T20:28:28.870Z + 2026-01-06T14:23:50.211Z https://docs.axolotl.ai/docs/api/core.chat.format.chatml.html - 2026-01-05T20:28:27.286Z + 2026-01-06T14:23:48.600Z https://docs.axolotl.ai/docs/api/integrations.grokfast.optimizer.html - 2026-01-05T20:28:28.732Z + 2026-01-06T14:23:50.070Z https://docs.axolotl.ai/docs/api/integrations.kd.trainer.html - 2026-01-05T20:28:28.741Z + 2026-01-06T14:23:50.079Z https://docs.axolotl.ai/docs/api/monkeypatch.unsloth_.html - 2026-01-05T20:28:28.221Z + 2026-01-06T14:23:49.555Z https://docs.axolotl.ai/docs/api/core.chat.format.llama3x.html - 2026-01-05T20:28:27.288Z + 2026-01-06T14:23:48.602Z https://docs.axolotl.ai/docs/models/gemma3n.html - 2026-01-05T20:28:44.241Z + 2026-01-06T14:24:06.291Z https://docs.axolotl.ai/docs/models/qwen3-next.html - 2026-01-05T20:28:44.240Z + 2026-01-06T14:24:06.291Z https://docs.axolotl.ai/docs/models/index.html - 2026-01-05T20:28:44.245Z + 2026-01-06T14:24:06.294Z https://docs.axolotl.ai/docs/models/magistral/think.html - 2026-01-05T20:28:44.237Z + 2026-01-06T14:24:06.287Z https://docs.axolotl.ai/docs/models/kimi-linear.html - 2026-01-05T20:28:44.233Z + 2026-01-06T14:24:06.283Z https://docs.axolotl.ai/docs/models/internvl3_5.html - 2026-01-05T20:28:44.234Z + 2026-01-06T14:24:06.284Z https://docs.axolotl.ai/docs/models/arcee.html - 2026-01-05T20:28:44.235Z + 2026-01-06T14:24:06.285Z https://docs.axolotl.ai/docs/models/LiquidAI.html - 2026-01-05T20:28:44.244Z + 2026-01-06T14:24:06.293Z https://docs.axolotl.ai/docs/models/magistral.html - 2026-01-05T20:28:44.237Z + 2026-01-06T14:24:06.287Z https://docs.axolotl.ai/docs/models/voxtral.html - 2026-01-05T20:28:44.239Z + 2026-01-06T14:24:06.289Z https://docs.axolotl.ai/docs/models/trinity.html - 2026-01-05T20:28:44.234Z + 2026-01-06T14:24:06.285Z https://docs.axolotl.ai/docs/models/ministral.html - 2026-01-05T20:28:44.238Z + 2026-01-06T14:24:06.288Z https://docs.axolotl.ai/docs/models/llama-4.html - 2026-01-05T20:28:44.240Z + 2026-01-06T14:24:06.290Z https://docs.axolotl.ai/docs/models/devstral.html - 2026-01-05T20:28:44.239Z + 2026-01-06T14:24:06.289Z https://docs.axolotl.ai/docs/models/ministral3.html - 2026-01-05T20:28:44.236Z + 2026-01-06T14:24:06.286Z https://docs.axolotl.ai/docs/models/ministral3/vision.html - 2026-01-05T20:28:44.236Z + 2026-01-06T14:24:06.286Z https://docs.axolotl.ai/docs/models/plano.html - 2026-01-05T20:28:44.233Z + 2026-01-06T14:24:06.284Z https://docs.axolotl.ai/docs/reward_modelling.html - 2026-01-05T20:25:02.692Z + 2026-01-06T14:20:21.702Z https://docs.axolotl.ai/docs/quantize.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.702Z https://docs.axolotl.ai/docs/fsdp_qlora.html - 2026-01-05T20:25:02.688Z + 2026-01-06T14:20:21.698Z https://docs.axolotl.ai/docs/nd_parallelism.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.702Z https://docs.axolotl.ai/docs/batch_vs_grad.html - 2026-01-05T20:25:02.687Z + 2026-01-06T14:20:21.697Z https://docs.axolotl.ai/docs/multi-node.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.701Z https://docs.axolotl.ai/docs/rlhf.html - 2026-01-05T20:25:02.692Z + 2026-01-06T14:20:21.702Z https://docs.axolotl.ai/docs/dataset-formats/stepwise_supervised.html - 2026-01-05T20:25:02.687Z + 2026-01-06T14:20:21.698Z https://docs.axolotl.ai/docs/dataset-formats/pretraining.html - 2026-01-05T20:25:02.687Z + 2026-01-06T14:20:21.698Z https://docs.axolotl.ai/docs/dataset-formats/tokenized.html - 2026-01-05T20:25:02.687Z + 2026-01-06T14:20:21.698Z https://docs.axolotl.ai/docs/dataset-formats/template_free.html - 2026-01-05T20:25:02.687Z + 2026-01-06T14:20:21.698Z https://docs.axolotl.ai/docs/multi-gpu.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.701Z https://docs.axolotl.ai/docs/input_output.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.701Z https://docs.axolotl.ai/docs/docker.html - 2026-01-05T20:25:02.688Z + 2026-01-06T14:20:21.698Z https://docs.axolotl.ai/docs/checkpoint_saving.html - 2026-01-05T20:25:02.687Z + 2026-01-06T14:20:21.697Z https://docs.axolotl.ai/docs/multipack.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.702Z https://docs.axolotl.ai/docs/qat.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.702Z https://docs.axolotl.ai/docs/lr_groups.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.701Z https://docs.axolotl.ai/docs/getting-started.html - 2026-01-05T20:25:02.688Z + 2026-01-06T14:20:21.698Z https://docs.axolotl.ai/docs/nccl.html - 2026-01-05T20:25:02.691Z + 2026-01-06T14:20:21.702Z https://docs.axolotl.ai/docs/telemetry.html - 2026-01-05T20:25:02.692Z + 2026-01-06T14:20:21.702Z https://docs.axolotl.ai/docs/unsloth.html - 2026-01-05T20:25:02.692Z + 2026-01-06T14:20:21.702Z https://docs.axolotl.ai/src/axolotl/integrations/LICENSE.html - 2026-01-05T20:25:02.717Z + 2026-01-06T14:20:21.725Z