67 lines
1.9 KiB
YAML
67 lines
1.9 KiB
YAML
# Example full fine-tuning config for a DeepSeek-V3 MoE model using Axolotl's
|
|
# vendored Triton contiguous grouped GEMM kernels.
|
|
# Replace `your-org/deepseek-v3-model` with the name of the model you uploaded to HF.
|
|
|
|
base_model: axolotl-ai-co/deepseek-v3-8b
|
|
model_config_type: deepseek_v3
|
|
trust_remote_code: true
|
|
moe_kernels: true
|
|
|
|
# --- Data ------------------------------------------------------------------
|
|
datasets:
|
|
- path: tatsu-lab/alpaca
|
|
type: alpaca
|
|
|
|
val_set_size: 0.0
|
|
output_dir: ./outputs/deepseek-v3/full-ft
|
|
|
|
sequence_len: 4096
|
|
sample_packing: true
|
|
|
|
# --- Optimisation ----------------------------------------------------------
|
|
num_epochs: 1
|
|
micro_batch_size: 1
|
|
gradient_accumulation_steps: 8
|
|
optimizer: adamw_torch_fused
|
|
learning_rate: 2e-5
|
|
lr_scheduler: cosine
|
|
warmup_ratio: 0.1
|
|
weight_decay: 0.01
|
|
|
|
# --- Precision & Performance -----------------------------------------------
|
|
bf16: auto
|
|
flash_attention: true
|
|
|
|
# enable GC to keep activation memory manageable for the MoE blocks
|
|
gradient_checkpointing: true
|
|
gradient_checkpointing_kwargs:
|
|
use_reentrant: false
|
|
|
|
# Axolotl automatically applies the DeepSeek-V3 MoE monkeypatch when
|
|
# model_config_type is set to `deepseek_v3`, routing matmuls through the
|
|
# vendored Triton kernels.
|
|
|
|
# --- Logging & Saving ------------------------------------------------------
|
|
logging_steps: 1
|
|
evals_per_epoch: 2
|
|
saves_per_epoch: 1
|
|
|
|
# Uncomment the section below for multi-GPU training with FSDP
|
|
# fsdp:
|
|
# - full_shard
|
|
# - auto_wrap
|
|
# fsdp_config:
|
|
# fsdp_limit_all_gathers: true
|
|
# fsdp_sync_module_states: true
|
|
# fsdp_offload_params: true
|
|
# fsdp_use_orig_params: false
|
|
# fsdp_cpu_ram_efficient_loading: true
|
|
# fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
|
|
# fsdp_transformer_layer_cls_to_wrap: DeepseekV3MoE
|
|
# fsdp_state_dict_type: FULL_STATE_DICT
|
|
# fsdp_sharding_strategy: FULL_SHARD
|
|
|
|
# wandb_project:
|
|
# wandb_entity:
|
|
# wandb_name:
|