diff --git a/.nojekyll b/.nojekyll index ce969cc3a..0901dfda8 100644 --- a/.nojekyll +++ b/.nojekyll @@ -1 +1 @@ -9b44d2d6 \ No newline at end of file +24230d64 \ No newline at end of file diff --git a/docs/config-reference.html b/docs/config-reference.html index dbb147e3c..18b714de3 100644 --- a/docs/config-reference.html +++ b/docs/config-reference.html @@ -956,1301 +956,1301 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true}); # Whether to perform weighting in DPO trainer dpo_use_weighting: bool | None -dpo_use_logits_to_keep: bool | None -dpo_label_smoothing: float | None -dpo_norm_loss: bool | None - -# Whether to use Liger kernel for DPO loss. -dpo_use_liger_kernel: bool | None - -dpo_padding_free: bool | None -dpo_generate_during_eval: bool | None - -# A list of one or more datasets to finetune the model with -datasets: Annotated[list[SFTDataset | DPODataset | KTODataset | StepwiseSupervisedDataset], MinLen(1)] | None - # For SFTDataset: - # HuggingFace dataset repo | s3:// | gs:// | path to local file or directory - path: str | None - # name of dataset split to load from - split: str | None - # The type of prompt to use for training. [alpaca, gpteacher, oasst, reflection] - type: str | UserDefinedPrompterType | None - # For UserDefinedPrompterType: - # Custom user instruction prompt - system_prompt: str | None - # Use {system} as key to be replaced - system_format: str | None - field_system: str | None - field_instruction: str | None - field_input: str | None - field_output: str | None - - # Customizable to be single line or multi-line. Use {instruction}/{input} as key to - # be replaced. 'format' can include {input} - format: str | None - # 'no_input_format' cannot include {input} - no_input_format: str | None - input_transform: str | None - # split dataset into N pieces (use with shards_idx) - shards: int | None - # the index of sharded dataset to use - shards_idx: int | None - # process dataset in N sequential chunks for memory efficiency (exclusive with - # `shards`) - preprocess_shards: int | None - conversation: str | None - - # The name of the chat template to use for training, following values are supported: - # tokenizer_default: Uses the chat template that is available in the - # tokenizer_config.json. If the chat template is not available in the tokenizer, it - # will raise an error. This is the default. - # alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates - # are available in the axolotl codebase at src/axolotl/utils/chat_templates.py. - # tokenizer_default_fallback_*: where * is the name of the chat template to fallback - # to if the tokenizer does not have a chat template else default to tokenizer. E.g. - # tokenizer_default_fallback_chatml. jinja: Uses a custom jinja template for the chat - # template. The custom jinja template should be provided in the chat_template_jinja - # field. - chat_template: ChatTemplate | str | None - # Custom jinja chat template or path to jinja file. Used only if `chat_template: - # jinja` or empty. - chat_template_jinja: str | None - # path to source data files - data_files: str | list[str] | None - input_format: str | None - # name of dataset configuration to load - name: str | None - # defines the datatype when path is a file - ds_type: str | None - # For `completion` datasets only, uses the provided field instead of `text` column - field: str | None - field_human: str | None - field_model: str | None - # Key containing the messages (default: "messages") - field_messages: str | None - # Key containing the tools (default: "tools"). Must be a list[dict] and follow [JSON - # schema](https://json-schema.org/learn/getting-started-step-by-step). - field_tools: str | None - # Key containing the reasoning trace (default: "reasoning_content"). - field_thinking: str | None - # The key the chat template expects that indicates the reasoning trace. - template_thinking_key: str | None +dpo_label_smoothing: float | None +dpo_norm_loss: bool | None + +# Whether to use Liger kernel for DPO loss. +dpo_use_liger_kernel: bool | None + +dpo_padding_free: bool | None + +# A list of one or more datasets to finetune the model with +datasets: Annotated[list[SFTDataset | DPODataset | KTODataset | StepwiseSupervisedDataset], MinLen(1)] | None + # For SFTDataset: + # HuggingFace dataset repo | s3:// | gs:// | path to local file or directory + path: str | None + # name of dataset split to load from + split: str | None + # The type of prompt to use for training. [alpaca, gpteacher, oasst, reflection] + type: str | UserDefinedPrompterType | None + # For UserDefinedPrompterType: + # Custom user instruction prompt + system_prompt: str | None + # Use {system} as key to be replaced + system_format: str | None + field_system: str | None + field_instruction: str | None + field_input: str | None + field_output: str | None + + # Customizable to be single line or multi-line. Use {instruction}/{input} as key to + # be replaced. 'format' can include {input} + format: str | None + # 'no_input_format' cannot include {input} + no_input_format: str | None + input_transform: str | None + # split dataset into N pieces (use with shards_idx) + shards: int | None + # the index of sharded dataset to use + shards_idx: int | None + # process dataset in N sequential chunks for memory efficiency (exclusive with + # `shards`) + preprocess_shards: int | None + conversation: str | None + + # The name of the chat template to use for training, following values are supported: + # tokenizer_default: Uses the chat template that is available in the + # tokenizer_config.json. If the chat template is not available in the tokenizer, it + # will raise an error. This is the default. + # alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates + # are available in the axolotl codebase at src/axolotl/utils/chat_templates.py. + # tokenizer_default_fallback_*: where * is the name of the chat template to fallback + # to if the tokenizer does not have a chat template else default to tokenizer. E.g. + # tokenizer_default_fallback_chatml. jinja: Uses a custom jinja template for the chat + # template. The custom jinja template should be provided in the chat_template_jinja + # field. + chat_template: ChatTemplate | str | None + # Custom jinja chat template or path to jinja file. Used only if `chat_template: + # jinja` or empty. + chat_template_jinja: str | None + # path to source data files + data_files: str | list[str] | None + input_format: str | None + # name of dataset configuration to load + name: str | None + # defines the datatype when path is a file + ds_type: str | None + # For `completion` datasets only, uses the provided field instead of `text` column + field: str | None + field_human: str | None + field_model: str | None + # Key containing the messages (default: "messages") + field_messages: str | None + # Key containing the tools (default: "tools"). Must be a list[dict] and follow [JSON + # schema](https://json-schema.org/learn/getting-started-step-by-step). + field_tools: str | None + # Key containing the reasoning trace (default: "reasoning_content"). + field_thinking: str | None + # The key the chat template expects that indicates the reasoning trace. + template_thinking_key: str | None + + message_field_role: str | None - message_field_role: str | None - - message_field_content: str | None - # Mapping of properties from the input dataset to the chat template. (default: - # message_property_mappings={'role':'role', 'content':'content'}) If a property exists - # in the template but not in this mapping, the system will attempt to load it directly - # from the message using the property name as the key. Example: In the mapping below, - # 'from' is loaded from input dataset and used as 'role', while 'value' is loaded and - # used as 'content' in the chat template. - message_property_mappings: dict[str, str] | None - # The key in the message turn that indicates via boolean whether tokens of a turn - # should be considered for training. Useful to selectively train on certain turns - # besides the `roles_to_train`. - message_field_training: str | None - # The key in the message turn that contains the training details. Useful to - # selectively train on certain tokens in a turn. The value of the key is a List[Dict] - # containing `begin_offset` (start character index in content), `end_offset` (end - # character index in content), and `train` (boolean whether to train). - message_field_training_detail: str | None - # (for Qwen3 template only) Whether to split the assistant content based on a - # reasoning trace inside delimited tags - split_thinking: bool | None - logprobs_field: str | None - temperature: float | None - # Roles to train on. The tokens from these roles will be considered for the loss. - roles_to_train: list[str] | None - # Which EOS tokens to train on in the conversation. Possible values are: all: train on - # all EOS tokens, turn (default): train on the EOS token at the end of each trainable - # turn, last: train on the last EOS token in the conversation - train_on_eos: Literal['all', 'turn', 'last'] | None - # Roles mapping in the messages. The format is {target_role: [source_roles]}. All - # source roles will be mapped to the target role. The default is: user: ["human", - # "user"], assistant: ["gpt", "assistant"], system: ["system"], tool: ["tool"] - roles: dict[str, list[str]] | None - # Whether to drop the system turn from the dataset. Only works with chat_template. - # This does not drop the default system message from chat_template if it exists. If - # you wish to, we recommend using a custom jinja template with the default system - # message removed or adding a system turn with empty content. - drop_system_message: bool | None - # Trust remote code for untrusted source - trust_remote_code: bool | None = False - # The specific revision of the dataset to use when loading from the Hugging Face Hub. - # This can be a commit hash, tag, or branch name. If not specified, the latest version - # will be used. This parameter is ignored for local datasets. - revision: str | None - - # For DPODataset: - path: str | None - split: str | None - type: UserDefinedDPOType | str | None - # For UserDefinedDPOType: - field_system: str | None - field_prompt: str | None - field_chosen: str | None - field_rejected: str | None - prompt_format: str | None - chosen_format: str | None - rejected_format: str | None - data_files: list[str] | None - revision: str | None - field_messages: str | None - - # For KTODataset: - path: str | None - split: str | None - type: UserDefinedKTOType | str | None - # For UserDefinedKTOType: - field_system: str | None - field_prompt: str | None - field_completion: str | None - field_label: bool | None - prompt_format: str | None - completion_format: str | None - data_files: list[str] | None - trust_remote_code: bool | None = False - revision: str | None - - # For StepwiseSupervisedDataset: - path: str | None - split: str | None - data_files: list[str] | None - revision: str | None - step_separator: str | None - max_completion_length: int | None - train_on_last_step_only: bool | None - -# A list of one or more datasets to eval the model with. You can use either -# test_datasets, or val_set_size, but not both. -test_datasets: Annotated[list[SFTDataset | DPODataset | KTODataset | StepwiseSupervisedDataset], MinLen(1)] | None - # For SFTDataset: - # HuggingFace dataset repo | s3:// | gs:// | path to local file or directory - path: str | None - # name of dataset split to load from - split: str | None - # The type of prompt to use for training. [alpaca, gpteacher, oasst, reflection] - type: str | UserDefinedPrompterType | None - # For UserDefinedPrompterType: - # Custom user instruction prompt - system_prompt: str | None - # Use {system} as key to be replaced - system_format: str | None - field_system: str | None - field_instruction: str | None - field_input: str | None - field_output: str | None - - # Customizable to be single line or multi-line. Use {instruction}/{input} as key to - # be replaced. 'format' can include {input} - format: str | None - # 'no_input_format' cannot include {input} - no_input_format: str | None - input_transform: str | None - # split dataset into N pieces (use with shards_idx) - shards: int | None - # the index of sharded dataset to use - shards_idx: int | None - # process dataset in N sequential chunks for memory efficiency (exclusive with - # `shards`) - preprocess_shards: int | None - conversation: str | None - - # The name of the chat template to use for training, following values are supported: - # tokenizer_default: Uses the chat template that is available in the - # tokenizer_config.json. If the chat template is not available in the tokenizer, it - # will raise an error. This is the default. - # alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates - # are available in the axolotl codebase at src/axolotl/utils/chat_templates.py. - # tokenizer_default_fallback_*: where * is the name of the chat template to fallback - # to if the tokenizer does not have a chat template else default to tokenizer. E.g. - # tokenizer_default_fallback_chatml. jinja: Uses a custom jinja template for the chat - # template. The custom jinja template should be provided in the chat_template_jinja - # field. - chat_template: ChatTemplate | str | None - # Custom jinja chat template or path to jinja file. Used only if `chat_template: - # jinja` or empty. - chat_template_jinja: str | None - # path to source data files - data_files: str | list[str] | None - input_format: str | None - # name of dataset configuration to load - name: str | None - # defines the datatype when path is a file - ds_type: str | None - # For `completion` datasets only, uses the provided field instead of `text` column - field: str | None - field_human: str | None - field_model: str | None - # Key containing the messages (default: "messages") - field_messages: str | None - # Key containing the tools (default: "tools"). Must be a list[dict] and follow [JSON - # schema](https://json-schema.org/learn/getting-started-step-by-step). - field_tools: str | None - # Key containing the reasoning trace (default: "reasoning_content"). - field_thinking: str | None - # The key the chat template expects that indicates the reasoning trace. - template_thinking_key: str | None + message_field_content: str | None + # Mapping of properties from the input dataset to the chat template. (default: + # message_property_mappings={'role':'role', 'content':'content'}) If a property exists + # in the template but not in this mapping, the system will attempt to load it directly + # from the message using the property name as the key. Example: In the mapping below, + # 'from' is loaded from input dataset and used as 'role', while 'value' is loaded and + # used as 'content' in the chat template. + message_property_mappings: dict[str, str] | None + # The key in the message turn that indicates via boolean whether tokens of a turn + # should be considered for training. Useful to selectively train on certain turns + # besides the `roles_to_train`. + message_field_training: str | None + # The key in the message turn that contains the training details. Useful to + # selectively train on certain tokens in a turn. The value of the key is a List[Dict] + # containing `begin_offset` (start character index in content), `end_offset` (end + # character index in content), and `train` (boolean whether to train). + message_field_training_detail: str | None + # (for Qwen3 template only) Whether to split the assistant content based on a + # reasoning trace inside delimited tags + split_thinking: bool | None + logprobs_field: str | None + temperature: float | None + # Roles to train on. The tokens from these roles will be considered for the loss. + roles_to_train: list[str] | None + # Which EOS tokens to train on in the conversation. Possible values are: all: train on + # all EOS tokens, turn (default): train on the EOS token at the end of each trainable + # turn, last: train on the last EOS token in the conversation + train_on_eos: Literal['all', 'turn', 'last'] | None + # Roles mapping in the messages. The format is {target_role: [source_roles]}. All + # source roles will be mapped to the target role. The default is: user: ["human", + # "user"], assistant: ["gpt", "assistant"], system: ["system"], tool: ["tool"] + roles: dict[str, list[str]] | None + # Whether to drop the system turn from the dataset. Only works with chat_template. + # This does not drop the default system message from chat_template if it exists. If + # you wish to, we recommend using a custom jinja template with the default system + # message removed or adding a system turn with empty content. + drop_system_message: bool | None + # Trust remote code for untrusted source + trust_remote_code: bool | None = False + # The specific revision of the dataset to use when loading from the Hugging Face Hub. + # This can be a commit hash, tag, or branch name. If not specified, the latest version + # will be used. This parameter is ignored for local datasets. + revision: str | None + + # For DPODataset: + path: str | None + split: str | None + type: UserDefinedDPOType | str | None + # For UserDefinedDPOType: + field_system: str | None + field_prompt: str | None + field_chosen: str | None + field_rejected: str | None + prompt_format: str | None + chosen_format: str | None + rejected_format: str | None + data_files: list[str] | None + revision: str | None + field_messages: str | None + + # For KTODataset: + path: str | None + split: str | None + type: UserDefinedKTOType | str | None + # For UserDefinedKTOType: + field_system: str | None + field_prompt: str | None + field_completion: str | None + field_label: bool | None + prompt_format: str | None + completion_format: str | None + data_files: list[str] | None + trust_remote_code: bool | None = False + revision: str | None + + # For StepwiseSupervisedDataset: + path: str | None + split: str | None + data_files: list[str] | None + revision: str | None + step_separator: str | None + max_completion_length: int | None + train_on_last_step_only: bool | None + +# A list of one or more datasets to eval the model with. You can use either +# test_datasets, or val_set_size, but not both. +test_datasets: Annotated[list[SFTDataset | DPODataset | KTODataset | StepwiseSupervisedDataset], MinLen(1)] | None + # For SFTDataset: + # HuggingFace dataset repo | s3:// | gs:// | path to local file or directory + path: str | None + # name of dataset split to load from + split: str | None + # The type of prompt to use for training. [alpaca, gpteacher, oasst, reflection] + type: str | UserDefinedPrompterType | None + # For UserDefinedPrompterType: + # Custom user instruction prompt + system_prompt: str | None + # Use {system} as key to be replaced + system_format: str | None + field_system: str | None + field_instruction: str | None + field_input: str | None + field_output: str | None + + # Customizable to be single line or multi-line. Use {instruction}/{input} as key to + # be replaced. 'format' can include {input} + format: str | None + # 'no_input_format' cannot include {input} + no_input_format: str | None + input_transform: str | None + # split dataset into N pieces (use with shards_idx) + shards: int | None + # the index of sharded dataset to use + shards_idx: int | None + # process dataset in N sequential chunks for memory efficiency (exclusive with + # `shards`) + preprocess_shards: int | None + conversation: str | None + + # The name of the chat template to use for training, following values are supported: + # tokenizer_default: Uses the chat template that is available in the + # tokenizer_config.json. If the chat template is not available in the tokenizer, it + # will raise an error. This is the default. + # alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates + # are available in the axolotl codebase at src/axolotl/utils/chat_templates.py. + # tokenizer_default_fallback_*: where * is the name of the chat template to fallback + # to if the tokenizer does not have a chat template else default to tokenizer. E.g. + # tokenizer_default_fallback_chatml. jinja: Uses a custom jinja template for the chat + # template. The custom jinja template should be provided in the chat_template_jinja + # field. + chat_template: ChatTemplate | str | None + # Custom jinja chat template or path to jinja file. Used only if `chat_template: + # jinja` or empty. + chat_template_jinja: str | None + # path to source data files + data_files: str | list[str] | None + input_format: str | None + # name of dataset configuration to load + name: str | None + # defines the datatype when path is a file + ds_type: str | None + # For `completion` datasets only, uses the provided field instead of `text` column + field: str | None + field_human: str | None + field_model: str | None + # Key containing the messages (default: "messages") + field_messages: str | None + # Key containing the tools (default: "tools"). Must be a list[dict] and follow [JSON + # schema](https://json-schema.org/learn/getting-started-step-by-step). + field_tools: str | None + # Key containing the reasoning trace (default: "reasoning_content"). + field_thinking: str | None + # The key the chat template expects that indicates the reasoning trace. + template_thinking_key: str | None + + message_field_role: str | None - message_field_role: str | None - - message_field_content: str | None - # Mapping of properties from the input dataset to the chat template. (default: - # message_property_mappings={'role':'role', 'content':'content'}) If a property exists - # in the template but not in this mapping, the system will attempt to load it directly - # from the message using the property name as the key. Example: In the mapping below, - # 'from' is loaded from input dataset and used as 'role', while 'value' is loaded and - # used as 'content' in the chat template. - message_property_mappings: dict[str, str] | None - # The key in the message turn that indicates via boolean whether tokens of a turn - # should be considered for training. Useful to selectively train on certain turns - # besides the `roles_to_train`. - message_field_training: str | None - # The key in the message turn that contains the training details. Useful to - # selectively train on certain tokens in a turn. The value of the key is a List[Dict] - # containing `begin_offset` (start character index in content), `end_offset` (end - # character index in content), and `train` (boolean whether to train). - message_field_training_detail: str | None - # (for Qwen3 template only) Whether to split the assistant content based on a - # reasoning trace inside delimited tags - split_thinking: bool | None - logprobs_field: str | None - temperature: float | None - # Roles to train on. The tokens from these roles will be considered for the loss. - roles_to_train: list[str] | None - # Which EOS tokens to train on in the conversation. Possible values are: all: train on - # all EOS tokens, turn (default): train on the EOS token at the end of each trainable - # turn, last: train on the last EOS token in the conversation - train_on_eos: Literal['all', 'turn', 'last'] | None - # Roles mapping in the messages. The format is {target_role: [source_roles]}. All - # source roles will be mapped to the target role. The default is: user: ["human", - # "user"], assistant: ["gpt", "assistant"], system: ["system"], tool: ["tool"] - roles: dict[str, list[str]] | None - # Whether to drop the system turn from the dataset. Only works with chat_template. - # This does not drop the default system message from chat_template if it exists. If - # you wish to, we recommend using a custom jinja template with the default system - # message removed or adding a system turn with empty content. - drop_system_message: bool | None - # Trust remote code for untrusted source - trust_remote_code: bool | None = False - # The specific revision of the dataset to use when loading from the Hugging Face Hub. - # This can be a commit hash, tag, or branch name. If not specified, the latest version - # will be used. This parameter is ignored for local datasets. - revision: str | None - - # For DPODataset: - path: str | None - split: str | None - type: UserDefinedDPOType | str | None - # For UserDefinedDPOType: - field_system: str | None - field_prompt: str | None - field_chosen: str | None - field_rejected: str | None - prompt_format: str | None - chosen_format: str | None - rejected_format: str | None - data_files: list[str] | None - revision: str | None - field_messages: str | None - - # For KTODataset: - path: str | None - split: str | None - type: UserDefinedKTOType | str | None - # For UserDefinedKTOType: - field_system: str | None - field_prompt: str | None - field_completion: str | None - field_label: bool | None - prompt_format: str | None - completion_format: str | None - data_files: list[str] | None - trust_remote_code: bool | None = False - revision: str | None - - # For StepwiseSupervisedDataset: - path: str | None - split: str | None - data_files: list[str] | None - revision: str | None - step_separator: str | None - max_completion_length: int | None - train_on_last_step_only: bool | None - -# If false, the datasets will not be shuffled and will keep their original order in -# `datasets`. The same applies to the `test_datasets` option and the -# `pretraining_dataset` option. Default is true. -shuffle_merged_datasets: bool | None = True -# If true, each dataset in `datasets` will be shuffled before merging. This allows -# curriculum learning strategies to be applied at the dataset level. Default is false. -shuffle_before_merging_datasets: bool | None = False -# Axolotl attempts to save the dataset as an arrow after packing the data together so -# subsequent training attempts load faster, relative path -dataset_prepared_path: str | None -# Num shards for whole dataset -dataset_shard_num: int | None -# Index of shard to use for whole dataset -dataset_shard_idx: int | None -skip_prepare_dataset: bool | None = False -# Number of shards to save the prepared dataset -num_dataset_shards_to_save: int | None - -# Set to HF dataset for type: 'completion' for streaming instead of pre-tokenize -pretraining_dataset: Annotated[list[PretrainingDataset | SFTDataset], MinLen(1)] | None - # For PretrainingDataset: - name: str | None - path: str | None - split: str | None = train - text_column: str | None = text - type: str | None = pretrain - trust_remote_code: bool | None = False - data_files: str | None - skip: int | None - - # For SFTDataset: - # HuggingFace dataset repo | s3:// | gs:// | path to local file or directory - path: str | None - # name of dataset split to load from - split: str | None - # The type of prompt to use for training. [alpaca, gpteacher, oasst, reflection] - type: str | UserDefinedPrompterType | None - # For UserDefinedPrompterType: - # Custom user instruction prompt - system_prompt: str | None - # Use {system} as key to be replaced - system_format: str | None - field_system: str | None - field_instruction: str | None - field_input: str | None - field_output: str | None - - # Customizable to be single line or multi-line. Use {instruction}/{input} as key to - # be replaced. 'format' can include {input} - format: str | None - # 'no_input_format' cannot include {input} - no_input_format: str | None - input_transform: str | None - # split dataset into N pieces (use with shards_idx) - shards: int | None - # the index of sharded dataset to use - shards_idx: int | None - # process dataset in N sequential chunks for memory efficiency (exclusive with - # `shards`) - preprocess_shards: int | None - conversation: str | None - - # The name of the chat template to use for training, following values are supported: - # tokenizer_default: Uses the chat template that is available in the - # tokenizer_config.json. If the chat template is not available in the tokenizer, it - # will raise an error. This is the default. - # alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates - # are available in the axolotl codebase at src/axolotl/utils/chat_templates.py. - # tokenizer_default_fallback_*: where * is the name of the chat template to fallback - # to if the tokenizer does not have a chat template else default to tokenizer. E.g. - # tokenizer_default_fallback_chatml. jinja: Uses a custom jinja template for the chat - # template. The custom jinja template should be provided in the chat_template_jinja - # field. - chat_template: ChatTemplate | str | None - # Custom jinja chat template or path to jinja file. Used only if `chat_template: - # jinja` or empty. - chat_template_jinja: str | None - # path to source data files - data_files: str | list[str] | None - input_format: str | None - # name of dataset configuration to load - name: str | None - # defines the datatype when path is a file - ds_type: str | None - # For `completion` datasets only, uses the provided field instead of `text` column - field: str | None - field_human: str | None - field_model: str | None - # Key containing the messages (default: "messages") - field_messages: str | None - # Key containing the tools (default: "tools"). Must be a list[dict] and follow [JSON - # schema](https://json-schema.org/learn/getting-started-step-by-step). - field_tools: str | None - # Key containing the reasoning trace (default: "reasoning_content"). - field_thinking: str | None - # The key the chat template expects that indicates the reasoning trace. - template_thinking_key: str | None + message_field_content: str | None + # Mapping of properties from the input dataset to the chat template. (default: + # message_property_mappings={'role':'role', 'content':'content'}) If a property exists + # in the template but not in this mapping, the system will attempt to load it directly + # from the message using the property name as the key. Example: In the mapping below, + # 'from' is loaded from input dataset and used as 'role', while 'value' is loaded and + # used as 'content' in the chat template. + message_property_mappings: dict[str, str] | None + # The key in the message turn that indicates via boolean whether tokens of a turn + # should be considered for training. Useful to selectively train on certain turns + # besides the `roles_to_train`. + message_field_training: str | None + # The key in the message turn that contains the training details. Useful to + # selectively train on certain tokens in a turn. The value of the key is a List[Dict] + # containing `begin_offset` (start character index in content), `end_offset` (end + # character index in content), and `train` (boolean whether to train). + message_field_training_detail: str | None + # (for Qwen3 template only) Whether to split the assistant content based on a + # reasoning trace inside delimited tags + split_thinking: bool | None + logprobs_field: str | None + temperature: float | None + # Roles to train on. The tokens from these roles will be considered for the loss. + roles_to_train: list[str] | None + # Which EOS tokens to train on in the conversation. Possible values are: all: train on + # all EOS tokens, turn (default): train on the EOS token at the end of each trainable + # turn, last: train on the last EOS token in the conversation + train_on_eos: Literal['all', 'turn', 'last'] | None + # Roles mapping in the messages. The format is {target_role: [source_roles]}. All + # source roles will be mapped to the target role. The default is: user: ["human", + # "user"], assistant: ["gpt", "assistant"], system: ["system"], tool: ["tool"] + roles: dict[str, list[str]] | None + # Whether to drop the system turn from the dataset. Only works with chat_template. + # This does not drop the default system message from chat_template if it exists. If + # you wish to, we recommend using a custom jinja template with the default system + # message removed or adding a system turn with empty content. + drop_system_message: bool | None + # Trust remote code for untrusted source + trust_remote_code: bool | None = False + # The specific revision of the dataset to use when loading from the Hugging Face Hub. + # This can be a commit hash, tag, or branch name. If not specified, the latest version + # will be used. This parameter is ignored for local datasets. + revision: str | None + + # For DPODataset: + path: str | None + split: str | None + type: UserDefinedDPOType | str | None + # For UserDefinedDPOType: + field_system: str | None + field_prompt: str | None + field_chosen: str | None + field_rejected: str | None + prompt_format: str | None + chosen_format: str | None + rejected_format: str | None + data_files: list[str] | None + revision: str | None + field_messages: str | None + + # For KTODataset: + path: str | None + split: str | None + type: UserDefinedKTOType | str | None + # For UserDefinedKTOType: + field_system: str | None + field_prompt: str | None + field_completion: str | None + field_label: bool | None + prompt_format: str | None + completion_format: str | None + data_files: list[str] | None + trust_remote_code: bool | None = False + revision: str | None + + # For StepwiseSupervisedDataset: + path: str | None + split: str | None + data_files: list[str] | None + revision: str | None + step_separator: str | None + max_completion_length: int | None + train_on_last_step_only: bool | None + +# If false, the datasets will not be shuffled and will keep their original order in +# `datasets`. The same applies to the `test_datasets` option and the +# `pretraining_dataset` option. Default is true. +shuffle_merged_datasets: bool | None = True +# If true, each dataset in `datasets` will be shuffled before merging. This allows +# curriculum learning strategies to be applied at the dataset level. Default is false. +shuffle_before_merging_datasets: bool | None = False +# Axolotl attempts to save the dataset as an arrow after packing the data together so +# subsequent training attempts load faster, relative path +dataset_prepared_path: str | None +# Num shards for whole dataset +dataset_shard_num: int | None +# Index of shard to use for whole dataset +dataset_shard_idx: int | None +skip_prepare_dataset: bool | None = False +# Number of shards to save the prepared dataset +num_dataset_shards_to_save: int | None + +# Set to HF dataset for type: 'completion' for streaming instead of pre-tokenize +pretraining_dataset: Annotated[list[PretrainingDataset | SFTDataset], MinLen(1)] | None + # For PretrainingDataset: + name: str | None + path: str | None + split: str | None = train + text_column: str | None = text + type: str | None = pretrain + trust_remote_code: bool | None = False + data_files: str | None + skip: int | None + + # For SFTDataset: + # HuggingFace dataset repo | s3:// | gs:// | path to local file or directory + path: str | None + # name of dataset split to load from + split: str | None + # The type of prompt to use for training. [alpaca, gpteacher, oasst, reflection] + type: str | UserDefinedPrompterType | None + # For UserDefinedPrompterType: + # Custom user instruction prompt + system_prompt: str | None + # Use {system} as key to be replaced + system_format: str | None + field_system: str | None + field_instruction: str | None + field_input: str | None + field_output: str | None + + # Customizable to be single line or multi-line. Use {instruction}/{input} as key to + # be replaced. 'format' can include {input} + format: str | None + # 'no_input_format' cannot include {input} + no_input_format: str | None + input_transform: str | None + # split dataset into N pieces (use with shards_idx) + shards: int | None + # the index of sharded dataset to use + shards_idx: int | None + # process dataset in N sequential chunks for memory efficiency (exclusive with + # `shards`) + preprocess_shards: int | None + conversation: str | None + + # The name of the chat template to use for training, following values are supported: + # tokenizer_default: Uses the chat template that is available in the + # tokenizer_config.json. If the chat template is not available in the tokenizer, it + # will raise an error. This is the default. + # alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates + # are available in the axolotl codebase at src/axolotl/utils/chat_templates.py. + # tokenizer_default_fallback_*: where * is the name of the chat template to fallback + # to if the tokenizer does not have a chat template else default to tokenizer. E.g. + # tokenizer_default_fallback_chatml. jinja: Uses a custom jinja template for the chat + # template. The custom jinja template should be provided in the chat_template_jinja + # field. + chat_template: ChatTemplate | str | None + # Custom jinja chat template or path to jinja file. Used only if `chat_template: + # jinja` or empty. + chat_template_jinja: str | None + # path to source data files + data_files: str | list[str] | None + input_format: str | None + # name of dataset configuration to load + name: str | None + # defines the datatype when path is a file + ds_type: str | None + # For `completion` datasets only, uses the provided field instead of `text` column + field: str | None + field_human: str | None + field_model: str | None + # Key containing the messages (default: "messages") + field_messages: str | None + # Key containing the tools (default: "tools"). Must be a list[dict] and follow [JSON + # schema](https://json-schema.org/learn/getting-started-step-by-step). + field_tools: str | None + # Key containing the reasoning trace (default: "reasoning_content"). + field_thinking: str | None + # The key the chat template expects that indicates the reasoning trace. + template_thinking_key: str | None + + message_field_role: str | None - message_field_role: str | None - - message_field_content: str | None - # Mapping of properties from the input dataset to the chat template. (default: - # message_property_mappings={'role':'role', 'content':'content'}) If a property exists - # in the template but not in this mapping, the system will attempt to load it directly - # from the message using the property name as the key. Example: In the mapping below, - # 'from' is loaded from input dataset and used as 'role', while 'value' is loaded and - # used as 'content' in the chat template. - message_property_mappings: dict[str, str] | None - # The key in the message turn that indicates via boolean whether tokens of a turn - # should be considered for training. Useful to selectively train on certain turns - # besides the `roles_to_train`. - message_field_training: str | None - # The key in the message turn that contains the training details. Useful to - # selectively train on certain tokens in a turn. The value of the key is a List[Dict] - # containing `begin_offset` (start character index in content), `end_offset` (end - # character index in content), and `train` (boolean whether to train). - message_field_training_detail: str | None - # (for Qwen3 template only) Whether to split the assistant content based on a - # reasoning trace inside delimited tags - split_thinking: bool | None - logprobs_field: str | None - temperature: float | None - # Roles to train on. The tokens from these roles will be considered for the loss. - roles_to_train: list[str] | None - # Which EOS tokens to train on in the conversation. Possible values are: all: train on - # all EOS tokens, turn (default): train on the EOS token at the end of each trainable - # turn, last: train on the last EOS token in the conversation - train_on_eos: Literal['all', 'turn', 'last'] | None - # Roles mapping in the messages. The format is {target_role: [source_roles]}. All - # source roles will be mapped to the target role. The default is: user: ["human", - # "user"], assistant: ["gpt", "assistant"], system: ["system"], tool: ["tool"] - roles: dict[str, list[str]] | None - # Whether to drop the system turn from the dataset. Only works with chat_template. - # This does not drop the default system message from chat_template if it exists. If - # you wish to, we recommend using a custom jinja template with the default system - # message removed or adding a system turn with empty content. - drop_system_message: bool | None - # Trust remote code for untrusted source - trust_remote_code: bool | None = False - # The specific revision of the dataset to use when loading from the Hugging Face Hub. - # This can be a commit hash, tag, or branch name. If not specified, the latest version - # will be used. This parameter is ignored for local datasets. - revision: str | None - -# The maximum number of processes to use while preprocessing your input dataset. This -# defaults to `os.cpu_count()` if not set. For Runpod VMs, it will default to number of -# vCPUs via RUNPOD_CPU_COUNT. -dataset_processes: int | None -# The maximum number of processes to use while preprocessing your input dataset. This -# defaults to `os.cpu_count()` if not set. For Runpod VMs, it will default to number of -# vCPUs via RUNPOD_CPU_COUNT. -dataset_num_proc: int | None - -# Deduplicates datasets and test_datasets with identical entries -dataset_exact_deduplication: bool | None -# Keep dataset in memory while preprocessing. Only needed if cached dataset is taking -# too much storage -dataset_keep_in_memory: bool | None -dataloader_pin_memory: bool | None -dataloader_num_workers: int | None -dataloader_prefetch_factor: int | None -dataloader_drop_last: bool | None + message_field_content: str | None + # Mapping of properties from the input dataset to the chat template. (default: + # message_property_mappings={'role':'role', 'content':'content'}) If a property exists + # in the template but not in this mapping, the system will attempt to load it directly + # from the message using the property name as the key. Example: In the mapping below, + # 'from' is loaded from input dataset and used as 'role', while 'value' is loaded and + # used as 'content' in the chat template. + message_property_mappings: dict[str, str] | None + # The key in the message turn that indicates via boolean whether tokens of a turn + # should be considered for training. Useful to selectively train on certain turns + # besides the `roles_to_train`. + message_field_training: str | None + # The key in the message turn that contains the training details. Useful to + # selectively train on certain tokens in a turn. The value of the key is a List[Dict] + # containing `begin_offset` (start character index in content), `end_offset` (end + # character index in content), and `train` (boolean whether to train). + message_field_training_detail: str | None + # (for Qwen3 template only) Whether to split the assistant content based on a + # reasoning trace inside delimited tags + split_thinking: bool | None + logprobs_field: str | None + temperature: float | None + # Roles to train on. The tokens from these roles will be considered for the loss. + roles_to_train: list[str] | None + # Which EOS tokens to train on in the conversation. Possible values are: all: train on + # all EOS tokens, turn (default): train on the EOS token at the end of each trainable + # turn, last: train on the last EOS token in the conversation + train_on_eos: Literal['all', 'turn', 'last'] | None + # Roles mapping in the messages. The format is {target_role: [source_roles]}. All + # source roles will be mapped to the target role. The default is: user: ["human", + # "user"], assistant: ["gpt", "assistant"], system: ["system"], tool: ["tool"] + roles: dict[str, list[str]] | None + # Whether to drop the system turn from the dataset. Only works with chat_template. + # This does not drop the default system message from chat_template if it exists. If + # you wish to, we recommend using a custom jinja template with the default system + # message removed or adding a system turn with empty content. + drop_system_message: bool | None + # Trust remote code for untrusted source + trust_remote_code: bool | None = False + # The specific revision of the dataset to use when loading from the Hugging Face Hub. + # This can be a commit hash, tag, or branch name. If not specified, the latest version + # will be used. This parameter is ignored for local datasets. + revision: str | None + +# The maximum number of processes to use while preprocessing your input dataset. This +# defaults to `os.cpu_count()` if not set. For Runpod VMs, it will default to number of +# vCPUs via RUNPOD_CPU_COUNT. +dataset_processes: int | None +# The maximum number of processes to use while preprocessing your input dataset. This +# defaults to `os.cpu_count()` if not set. For Runpod VMs, it will default to number of +# vCPUs via RUNPOD_CPU_COUNT. +dataset_num_proc: int | None + +# Deduplicates datasets and test_datasets with identical entries +dataset_exact_deduplication: bool | None +# Keep dataset in memory while preprocessing. Only needed if cached dataset is taking +# too much storage +dataset_keep_in_memory: bool | None +dataloader_pin_memory: bool | None +dataloader_num_workers: int | None +dataloader_prefetch_factor: int | None +dataloader_drop_last: bool | None + +accelerator_config: dict[str, Any] | None -accelerator_config: dict[str, Any] | None +remove_unused_columns: bool | None -remove_unused_columns: bool | None - -# Push prepared dataset to hub - repo_org/repo_name -push_dataset_to_hub: str | None -# Whether to use hf `use_auth_token` for loading datasets. Useful for fetching private -# datasets. Required to be true when used in combination with `push_dataset_to_hub` -hf_use_auth_token: bool | None - -device: Any | None -# Passed through to transformers when loading the model when launched without -# accelerate. Use `sequential` when training w/ model parallelism to limit memory -device_map: Any | None -world_size: int | None -# Don't mess with this, it's here for accelerate and torchrun -local_rank: int | None -ddp: bool | None - -# Seed for reproducibility -seed: int | None -# Advanced DDP Arguments - timeout -ddp_timeout: int | None -# Advanced DDP Arguments - bucket cap in MB -ddp_bucket_cap_mb: int | None -# Advanced DDP Arguments - broadcast buffers -ddp_broadcast_buffers: bool | None -ddp_find_unused_parameters: bool | None - -# Whether to run causal language model evaluation for metrics in -# `eval_causal_lm_metrics` -do_causal_lm_eval: bool | None -# HF evaluate metrics used during evaluation. Default is ['sacrebleu', 'comet', 'ter', -# 'chrf', 'perplexity'] -eval_causal_lm_metrics: list[str] | None -do_bench_eval: bool | None -bench_dataset: str | None -bench_split: str | None -metric_for_best_model: str | None -greater_is_better: bool | None - -# High loss value, indicating the learning has broken down (a good estimate is ~2 times -# the loss at the start of training) -loss_watchdog_threshold: float | None -# Number of high-loss steps in a row before the trainer aborts (default: 3) -loss_watchdog_patience: int | None - -# Run garbage collection every `gc_steps` steps. -1 will run on epoch end and before -# evaluations. Default is 0 (disabled). -gc_steps: int | None - -# Use CUDA bf16. bool or 'full' for `bf16_full_eval`, or 'auto' for automatic detection. -# require >=ampere -bf16: Literal['auto'] | bool | None = auto -# Use CUDA fp16 -fp16: bool | None -# Enable FP8 mixed precision training using TorchAO. Best used in combination with -# torch.compile. -fp8: bool | None -# Enable FSDP float8 all-gather optimization for FP8 training. Can improve training -# speed by 10-15% when FSDP is enabled. -fp8_enable_fsdp_float8_all_gather: bool | None -# No AMP (automatic mixed precision) - require >=ampere -bfloat16: bool | None -# No AMP (automatic mixed precision) -float16: bool | None -# Use CUDA tf32 - require >=ampere -tf32: bool | None -float32: bool | None - -# Whether to use gradient checkpointing. Available options are: true, false, 'offload', -# 'offload_disk'. -# https://huggingface.co/docs/transformers/v4.18.0/en/performance#gradient-checkpointing -gradient_checkpointing: Literal['offload', 'offload_disk'] | bool | None = False -# Additional kwargs to pass to the trainer for gradient checkpointing -gradient_checkpointing_kwargs: dict[str, Any] | None -# Whether to offload activations. Available options are: true, false, 'legacy', 'disk'. -activation_offloading: Literal['legacy', 'disk'] | bool | None = False - -# List of regex patterns for parameter names to keep unfrozen. All other parameters will -# be frozen via requires_grad=False. Note: range-based patterns (e.g. -# embed_tokens.weight$[:32000]) use gradient zeroing rather than a true freeze, so -# weight decay will still apply to the frozen portion and optimizer states are allocated -# for the full parameter. -unfrozen_parameters: list[str] | None - -# The maximum length of an input to train with, this should typically be less than 2048 -# as most models have a token/context limit of 2048 -sequence_len: int = 512 -# What to do when a tokenized row exceeds sequence_len. 'drop' removes the row; -# 'truncate' slices tensors to sequence_len; 'raise' raises a ValueError. Defaults to -# 'drop' for backward compatibility. -excess_length_strategy: Literal['drop', 'truncate', 'raise'] | None -# The maximum length of an input for evaluation. If not specified, defaults to -# sequence_len -eval_sequence_len: int | None -min_sample_len: int | None -# maximum prompt length for RL training -max_prompt_len: int | None -# Use efficient multi-packing with block diagonal attention and per sequence -# position_ids. Recommend set to 'true' -sample_packing: bool | None -# The number of samples packed at a time. Increasing the following values helps with -# packing, but usually only slightly (<%1.) -sample_packing_group_size: int | None = 100000 -# The number of samples which can be packed into one sequence. Increase if using a large -# sequence_len with many short samples. -sample_packing_bin_size: int | None = 200 -# Whether to pack samples sequentially -sample_packing_sequentially: bool | None -# The multiprocessing start method to use for packing. Should be 'fork', 'spawn' or -# 'forkserver' -sample_packing_mp_start_method: str | None -# Set to 'false' if getting errors during eval with sample_packing on -eval_sample_packing: bool | None -# Pad inputs so each step uses constant sized buffers. This will reduce memory -# fragmentation and may prevent OOMs, by re-using memory more efficiently. Defaults to -# True if `sample_packing` enabled -pad_to_sequence_len: bool | None -# Whether to use sequential sampling for curriculum learning -curriculum_sampling: bool | None -multipack_real_batches: bool | None - -# Use batch flattening for speedups when not using sample_packing -batch_flattening: Literal['auto'] | bool | None - -use_pose: bool | None -pose_split_on_token_ids: list[int] | None -pose_max_context_len: int | None -pose_num_chunks: int | None - -pretrain_multipack_buffer_size: int | None -# whether to prevent cross attention for packed sequences during pretraining -pretrain_multipack_attn: bool | None = True -# whether to concatenate samples during pretraining -pretraining_sample_concatenation: bool | None - -# Use streaming mode for loading datasets -streaming: bool | None -# Buffer size for multipack streaming datasets -streaming_multipack_buffer_size: int | None = 10000 - -# Whether to use xformers attention patch https://github.com/facebookresearch/xformers -xformers_attention: bool | None -# Whether to use scaled-dot-product attention https://pytorch.org/docs/stable/generated/ -# torch.nn.functional.scaled_dot_product_attention.html -sdp_attention: bool | None -# Shifted-sparse attention (only llama) - https://arxiv.org/pdf/2309.12307.pdf -s2_attention: bool | None -flex_attention: bool | None -flex_attn_compile_kwargs: dict[str, Any] | None -# Whether to use flash attention patch https://github.com/Dao-AILab/flash-attention -flash_attention: bool | None -# Whether to use flash-attention cross entropy implementation - advanced use only -flash_attn_cross_entropy: bool | None -# Whether to use flash-attention rms norm implementation - advanced use only -flash_attn_rms_norm: bool | None -# Whether to fuse part of the MLP into a single operation -flash_attn_fuse_mlp: bool | None -# Whether to use bettertransformers -flash_optimum: bool | None -# Whether to use SageAttention https://github.com/thu-ml/SageAttention -sage_attention: bool | None +# Push prepared dataset to hub - repo_org/repo_name +push_dataset_to_hub: str | None +# Whether to use hf `use_auth_token` for loading datasets. Useful for fetching private +# datasets. Required to be true when used in combination with `push_dataset_to_hub` +hf_use_auth_token: bool | None + +device: Any | None +# Passed through to transformers when loading the model when launched without +# accelerate. Use `sequential` when training w/ model parallelism to limit memory +device_map: Any | None +world_size: int | None +# Don't mess with this, it's here for accelerate and torchrun +local_rank: int | None +ddp: bool | None + +# Seed for reproducibility +seed: int | None +# Advanced DDP Arguments - timeout +ddp_timeout: int | None +# Advanced DDP Arguments - bucket cap in MB +ddp_bucket_cap_mb: int | None +# Advanced DDP Arguments - broadcast buffers +ddp_broadcast_buffers: bool | None +ddp_find_unused_parameters: bool | None + +# Whether to run causal language model evaluation for metrics in +# `eval_causal_lm_metrics` +do_causal_lm_eval: bool | None +# HF evaluate metrics used during evaluation. Default is ['sacrebleu', 'comet', 'ter', +# 'chrf', 'perplexity'] +eval_causal_lm_metrics: list[str] | None +do_bench_eval: bool | None +bench_dataset: str | None +bench_split: str | None +metric_for_best_model: str | None +greater_is_better: bool | None + +# High loss value, indicating the learning has broken down (a good estimate is ~2 times +# the loss at the start of training) +loss_watchdog_threshold: float | None +# Number of high-loss steps in a row before the trainer aborts (default: 3) +loss_watchdog_patience: int | None + +# Run garbage collection every `gc_steps` steps. -1 will run on epoch end and before +# evaluations. Default is 0 (disabled). +gc_steps: int | None + +# Use CUDA bf16. bool or 'full' for `bf16_full_eval`, or 'auto' for automatic detection. +# require >=ampere +bf16: Literal['auto'] | bool | None = auto +# Use CUDA fp16 +fp16: bool | None +# Enable FP8 mixed precision training using TorchAO. Best used in combination with +# torch.compile. +fp8: bool | None +# Enable FSDP float8 all-gather optimization for FP8 training. Can improve training +# speed by 10-15% when FSDP is enabled. +fp8_enable_fsdp_float8_all_gather: bool | None +# No AMP (automatic mixed precision) - require >=ampere +bfloat16: bool | None +# No AMP (automatic mixed precision) +float16: bool | None +# Use CUDA tf32 - require >=ampere +tf32: bool | None +float32: bool | None + +# Whether to use gradient checkpointing. Available options are: true, false, 'offload', +# 'offload_disk'. +# https://huggingface.co/docs/transformers/v4.18.0/en/performance#gradient-checkpointing +gradient_checkpointing: Literal['offload', 'offload_disk'] | bool | None = False +# Additional kwargs to pass to the trainer for gradient checkpointing +gradient_checkpointing_kwargs: dict[str, Any] | None +# Whether to offload activations. Available options are: true, false, 'legacy', 'disk'. +activation_offloading: Literal['legacy', 'disk'] | bool | None = False + +# List of regex patterns for parameter names to keep unfrozen. All other parameters will +# be frozen via requires_grad=False. Note: range-based patterns (e.g. +# embed_tokens.weight$[:32000]) use gradient zeroing rather than a true freeze, so +# weight decay will still apply to the frozen portion and optimizer states are allocated +# for the full parameter. +unfrozen_parameters: list[str] | None + +# The maximum length of an input to train with, this should typically be less than 2048 +# as most models have a token/context limit of 2048 +sequence_len: int = 512 +# What to do when a tokenized row exceeds sequence_len. 'drop' removes the row; +# 'truncate' slices tensors to sequence_len; 'raise' raises a ValueError. Defaults to +# 'drop' for backward compatibility. +excess_length_strategy: Literal['drop', 'truncate', 'raise'] | None +# The maximum length of an input for evaluation. If not specified, defaults to +# sequence_len +eval_sequence_len: int | None +min_sample_len: int | None +# maximum prompt length for RL training +max_prompt_len: int | None +# Use efficient multi-packing with block diagonal attention and per sequence +# position_ids. Recommend set to 'true' +sample_packing: bool | None +# The number of samples packed at a time. Increasing the following values helps with +# packing, but usually only slightly (<%1.) +sample_packing_group_size: int | None = 100000 +# The number of samples which can be packed into one sequence. Increase if using a large +# sequence_len with many short samples. +sample_packing_bin_size: int | None = 200 +# Whether to pack samples sequentially +sample_packing_sequentially: bool | None +# The multiprocessing start method to use for packing. Should be 'fork', 'spawn' or +# 'forkserver' +sample_packing_mp_start_method: str | None +# Set to 'false' if getting errors during eval with sample_packing on +eval_sample_packing: bool | None +# Pad inputs so each step uses constant sized buffers. This will reduce memory +# fragmentation and may prevent OOMs, by re-using memory more efficiently. Defaults to +# True if `sample_packing` enabled +pad_to_sequence_len: bool | None +# Whether to use sequential sampling for curriculum learning +curriculum_sampling: bool | None +multipack_real_batches: bool | None + +# Use batch flattening for speedups when not using sample_packing +batch_flattening: Literal['auto'] | bool | None + +use_pose: bool | None +pose_split_on_token_ids: list[int] | None +pose_max_context_len: int | None +pose_num_chunks: int | None + +pretrain_multipack_buffer_size: int | None +# whether to prevent cross attention for packed sequences during pretraining +pretrain_multipack_attn: bool | None = True +# whether to concatenate samples during pretraining +pretraining_sample_concatenation: bool | None + +# Use streaming mode for loading datasets +streaming: bool | None +# Buffer size for multipack streaming datasets +streaming_multipack_buffer_size: int | None = 10000 + +# Whether to use xformers attention patch https://github.com/facebookresearch/xformers +xformers_attention: bool | None +# Whether to use scaled-dot-product attention https://pytorch.org/docs/stable/generated/ +# torch.nn.functional.scaled_dot_product_attention.html +sdp_attention: bool | None +# Shifted-sparse attention (only llama) - https://arxiv.org/pdf/2309.12307.pdf +s2_attention: bool | None +flex_attention: bool | None +flex_attn_compile_kwargs: dict[str, Any] | None +# Whether to use flash attention patch https://github.com/Dao-AILab/flash-attention +flash_attention: bool | None +# Whether to use flash-attention cross entropy implementation - advanced use only +flash_attn_cross_entropy: bool | None +# Whether to use flash-attention rms norm implementation - advanced use only +flash_attn_rms_norm: bool | None +# Whether to fuse part of the MLP into a single operation +flash_attn_fuse_mlp: bool | None +# Whether to use bettertransformers +flash_optimum: bool | None +# Whether to use SageAttention https://github.com/thu-ml/SageAttention +sage_attention: bool | None + +eager_attention: bool | None -eager_attention: bool | None - -# Specify a custom attention implementation, used mostly for kernels. -attn_implementation: str | None - -# Which experts implementation to use for MoE models, -experts_implementation: str | None - -# Quantize MoE expert weights on load to reduce VRAM. Requires adapter (lora/qlora) with -# load_in_4bit or load_in_8bit. Requires CUDA (not compatible with ROCm or other -# backends). Note: total parameter count may be reported incorrectly when enabled -# (trainable param count is correct). -quantize_moe_experts: bool = False - -# Whether to use Scaled Softmax (SSMax) attention. Ref: https://arxiv.org/abs/2501.19399 -scaling_softmax: bool | None -# Scaling factor for SSMax attention. Default is 0.43 -scaling_softmax_factor: float | None -# Bias for SSMax attention. Default is 0.0. Note: The paper recommends bias=0 for better -# length generalization. -scaling_softmax_bias: float | None - -unsloth_cross_entropy_loss: bool | None -unsloth_lora_mlp: bool | None -unsloth_lora_qkv: bool | None -unsloth_lora_o: bool | None -unsloth_rms_norm: bool | None -unsloth_rope: bool | None - -# Apply custom LoRA autograd functions and activation function Triton kernels for speed -# and memory savings. See: https://docs.axolotl.ai/docs/lora_optims.html -lora_mlp_kernel: bool | None -# Apply custom LoRA autograd functions and activation function Triton kernels for speed -# and memory savings. See: https://docs.axolotl.ai/docs/lora_optims.html -lora_qkv_kernel: bool | None -# Apply custom LoRA autograd functions and activation function Triton kernels for speed -# and memory savings. See: https://docs.axolotl.ai/docs/lora_optims.html -lora_o_kernel: bool | None - -# Whether to use chunked cross entropy loss for memory efficiency -chunked_cross_entropy: bool | None -# Number of chunks to use for chunked cross entropy loss -chunked_cross_entropy_num_chunks: int | None -# Enable Entropy-Aware Focal Training loss (EAFT) -use_eaft: bool | None -# Exponent for entropy weighting in EAFT (default: 1.0) -eaft_alpha: float | None = 1.0 -# Number of top logits for entropy approximation (default: 20) -eaft_k: int | None = 20 - -# Whether to use ALST tiled mlp for memory efficient long context -tiled_mlp: bool | None - -# Number of shards to use for ALST tiled mlp. If unset, it will be set based on -# seqlen/hidden_size -tiled_mlp_num_shards: int | None - -# Whether to use original mlp for ALST tiled mlp. Otherwise uses a generic MLP based on -# llama. -tiled_mlp_use_original_mlp: bool | None = True +# Specify a custom attention implementation, used mostly for kernels. +attn_implementation: str | None + +# Which experts implementation to use for MoE models, +experts_implementation: str | None + +# Quantize MoE expert weights on load to reduce VRAM. Requires adapter (lora/qlora) with +# load_in_4bit or load_in_8bit. Requires CUDA (not compatible with ROCm or other +# backends). Note: total parameter count may be reported incorrectly when enabled +# (trainable param count is correct). +quantize_moe_experts: bool = False + +# Whether to use Scaled Softmax (SSMax) attention. Ref: https://arxiv.org/abs/2501.19399 +scaling_softmax: bool | None +# Scaling factor for SSMax attention. Default is 0.43 +scaling_softmax_factor: float | None +# Bias for SSMax attention. Default is 0.0. Note: The paper recommends bias=0 for better +# length generalization. +scaling_softmax_bias: float | None + +unsloth_cross_entropy_loss: bool | None +unsloth_lora_mlp: bool | None +unsloth_lora_qkv: bool | None +unsloth_lora_o: bool | None +unsloth_rms_norm: bool | None +unsloth_rope: bool | None + +# Apply custom LoRA autograd functions and activation function Triton kernels for speed +# and memory savings. See: https://docs.axolotl.ai/docs/lora_optims.html +lora_mlp_kernel: bool | None +# Apply custom LoRA autograd functions and activation function Triton kernels for speed +# and memory savings. See: https://docs.axolotl.ai/docs/lora_optims.html +lora_qkv_kernel: bool | None +# Apply custom LoRA autograd functions and activation function Triton kernels for speed +# and memory savings. See: https://docs.axolotl.ai/docs/lora_optims.html +lora_o_kernel: bool | None + +# Whether to use chunked cross entropy loss for memory efficiency +chunked_cross_entropy: bool | None +# Number of chunks to use for chunked cross entropy loss +chunked_cross_entropy_num_chunks: int | None +# Enable Entropy-Aware Focal Training loss (EAFT) +use_eaft: bool | None +# Exponent for entropy weighting in EAFT (default: 1.0) +eaft_alpha: float | None = 1.0 +# Number of top logits for entropy approximation (default: 20) +eaft_k: int | None = 20 + +# Whether to use ALST tiled mlp for memory efficient long context +tiled_mlp: bool | None + +# Number of shards to use for ALST tiled mlp. If unset, it will be set based on +# seqlen/hidden_size +tiled_mlp_num_shards: int | None + +# Whether to use original mlp for ALST tiled mlp. Otherwise uses a generic MLP based on +# llama. +tiled_mlp_use_original_mlp: bool | None = True + +llama4_linearized_experts: bool | None -llama4_linearized_experts: bool | None - -# Deepspeed config path. e.g., deepspeed_configs/zero3.json -deepspeed: str | dict[str, Any] | None -# Whether to use deepcompile for faster training with deepspeed -deepcompile: bool | None -# FSDP configuration -fsdp: list[str] | None - -# FSDP configuration options -fsdp_config: FSDPConfig | None - # For FSDPConfig: - # FSDP version - fsdp_version: int | None - # Enable activation checkpointing to reduce memory usage during forward passes - activation_checkpointing: bool | None - # Offload parameters to CPU to reduce GPU memory usage - offload_params: bool | None - # Synchronize module states across all processes - sync_module_states: bool | None - # Enable CPU RAM efficient loading to reduce memory usage during model loading - cpu_ram_efficient_loading: bool | None - # Disabling this enables swap memory usage for resource-constrained setups when - # offload_params is enabled. - cpu_offload_pin_memory: bool | None - # Use original parameters instead of flattened parameters - use_orig_params: bool | None - - # Type of state dict to use for saving/loading checkpoints - state_dict_type: Literal['FULL_STATE_DICT', 'LOCAL_STATE_DICT', 'SHARDED_STATE_DICT'] | None - # Final state dict type to use after training completion - final_state_dict_type: Literal['FULL_STATE_DICT', 'LOCAL_STATE_DICT', 'SHARDED_STATE_DICT'] | None - - # Policy for automatically wrapping modules with FSDP - auto_wrap_policy: Literal['TRANSFORMER_BASED_WRAP', 'SIZE_BASED_WRAP'] | None - # Class name of transformer layers to wrap (e.g., 'LlamaDecoderLayer') - transformer_layer_cls_to_wrap: str | None - - # Reshard parameters after forward pass to save memory - reshard_after_forward: bool | None - # Mixed precision policy for FSDP (e.g., 'fp16', 'bf16') - mixed_precision_policy: str | None - -# FSDP version -fsdp_version: int | None -fsdp_final_state_dict_type: Literal['FULL_STATE_DICT', 'LOCAL_STATE_DICT', 'SHARDED_STATE_DICT'] | None - -# How much of the dataset to set aside as evaluation. 1 = 100%, 0.50 = 50%, etc. 0 for -# no eval. -val_set_size: float | None = 0.0 - -# Number of devices to shard across. If not set, will use all available devices. -dp_shard_size: int | None -# Number of devices to replicate across. -dp_replicate_size: int | None -# Deprecated: use `context_parallel_size` instead -sequence_parallel_degree: int | None -# Set to a divisor of the number of GPUs available to split sequences into chunks of -# equal size. Use in long context training to prevent OOM when sequences cannot fit into -# a single GPU's VRAM. E.g., if 4 GPUs are available, set this value to 2 to split each -# sequence into two equal-sized subsequences, or set to 4 to split into four equal-sized -# subsequences. See https://docs.axolotl.ai/docs/sequence_parallelism.html for more -# details. -context_parallel_size: int | None -# Optional; strides across the key dimension. Larger values use more memory but should -# make training faster. Must evenly divide the number of KV heads in your model. -heads_k_stride: int | None -# One of 'varlen_llama3', 'batch_ring', 'batch_zigzag', 'batch_stripe'. Defaults to -# 'varlen_llama3' in the sample packing case, and 'batch_ring' in the non-sample packing -# case. -ring_attn_func: RingAttnFunc | None -# Number of tensor parallel processes in TP group. Only supported with DeepSpeed AutoTP. -tensor_parallel_size: int | None - -# Add or change special tokens. If you add tokens here, you don't need to add them to -# the `tokens` list. -special_tokens: SpecialTokensConfig | None - # For SpecialTokensConfig: - bos_token: str | None - eos_token: str | None - pad_token: str | None - unk_token: str | None - additional_special_tokens: list[str] | None - -# Add extra tokens to the tokenizer -tokens: list[str] | None -# Mapping token_id to new_token_string to override reserved added_tokens in the -# tokenizer. Only works for tokens that are not part of the base vocab (aka are -# added_tokens). Can be checked if they exist in tokenizer.json added_tokens. -added_tokens_overrides: dict[int, str] | None - -# Whether to use torch.compile and which backend to use. setting to `auto` will enable -# torch compile when torch>=2.6.0 -torch_compile: Literal['auto'] | bool | None -# Backend to use for torch.compile -torch_compile_backend: str | None -torch_compile_mode: Literal['default', 'reduce-overhead', 'max-autotune'] | None - -# Maximum number of iterations to train for. It precedes num_epochs which means that if -# both are set, num_epochs will not be guaranteed. e.g., when 1 epoch is 1000 steps => -# `num_epochs: 2` and `max_steps: 100` will train for 100 steps -max_steps: int | None -# Number of warmup steps. Cannot use with warmup_ratio -warmup_steps: int | None -# Warmup ratio. Cannot use with warmup_steps -warmup_ratio: float | None -# Leave empty to eval at each epoch, integer for every N steps. float for fraction of -# total steps -eval_steps: int | float | None -# Number of times per epoch to run evals, mutually exclusive with eval_steps -evals_per_epoch: int | None -# Set to `no` to skip evaluation, `epoch` at end of each epoch, leave empty to infer -# from `eval_steps` -eval_strategy: str | None - -# Leave empty to save at each epoch, integer for every N steps. float for fraction of -# total steps -save_steps: int | float | None -# Number of times per epoch to save a checkpoint, mutually exclusive with save_steps -saves_per_epoch: int | None -# Set to `no` to skip checkpoint saves, `epoch` at end of each epoch, `best` when better -# result is achieved, leave empty to infer from `save_steps` -save_strategy: str | None -# Checkpoints saved at a time -save_total_limit: int | None -# Whether to checkpoint a model after the first step of training. Defaults to False. -save_first_step: bool | None - -# Logging frequency -logging_steps: int | None -# Stop training after this many evaluation losses have increased in a row. https://huggi -# ngface.co/transformers/v4.2.2/_modules/transformers/trainer_callback.html#EarlyStoppin -# gCallback -early_stopping_patience: int | None -load_best_model_at_end: bool | None = False -# Save only the model weights, skipping the optimizer. Using this means you can't resume -# from checkpoints. -save_only_model: bool | None = False -# Use tensorboard for logging -use_tensorboard: bool | None -# Enable the pytorch profiler to capture the first N steps of training to the -# output_dir. see https://pytorch.org/blog/understanding-gpu-memory-1/ for more -# information. Snapshots can be visualized @ https://pytorch.org/memory_viz -profiler_steps: int | None -# Which step to start the profiler at. Useful for only capturing a few steps mid-run. -profiler_steps_start: int | None = 0 -# bool of whether to report tokens per second at the end of training. This is not -# supported with pre-training datasets. -include_tokens_per_second: bool | None -# bool of whether to report tokens per second per-gpu during training by measuring -# throughput of non-padding tokens. -include_tkps: bool | None = True -# NEFT https://arxiv.org/abs/2310.05914, set this to a number (paper default is 5) to -# add noise to embeddings. Currently only supported on Llama and Mistral -neftune_noise_alpha: float | None - -# Parameter controlling the relative ratio loss weight in the ORPO loss. Passed to -# `beta` in `ORPOConfig` due to trl mapping. -orpo_alpha: float | None -# Weighting of NLL term in loss from RPO paper -rpo_alpha: float | None -# Target reward margin for the SimPO loss -simpo_gamma: float | None -# Weight of the BC regularizer -cpo_alpha: float | None - -# Factor for desirable loss term in KTO loss -kto_desirable_weight: float | None -# Factor for undesirable loss term in KTO loss -kto_undesirable_weight: float | None -# The beta parameter for the RL training -rl_beta: float | None - -# Defines the max memory usage per gpu on the system. Passed through to transformers -# when loading the model. -max_memory: dict[int | Literal['cpu', 'disk'], int | str] | None -# Limit the memory for all available GPUs to this amount (if an integer, expressed in -# gigabytes); default: unset -gpu_memory_limit: int | str | None -# Whether to use low_cpu_mem_usage -low_cpu_mem_usage: bool | None - -# The name of the chat template to use for training, following values are supported: -# tokenizer_default: Uses the chat template that is available in the -# tokenizer_config.json. If the chat template is not available in the tokenizer, it will -# raise an error. This is the default value. -# alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates -# are available in the axolotl codebase at src/axolotl/utils/chat_templates.py. -# tokenizer_default_fallback_*: where * is the name of the chat template to fallback to. -# E.g. tokenizer_default_fallback_chatml. This is useful when the chat template is not -# available in the tokenizer. jinja: Uses a custom jinja template for the chat template. -# The custom jinja template should be provided in the chat_template_jinja field. The -# selected chat template will be saved to the tokenizer_config.json for easier -# inferencing -chat_template: ChatTemplate | Annotated[str, StringConstraints(pattern='^tokenizer_default_fallback_')] | None -# Custom jinja template or path to jinja file for chat template. This will be only used -# if chat_template is set to `jinja` or `null` (in which case chat_template is -# automatically set to `jinja`). Default is null. -chat_template_jinja: str | None -# Additional kwargs to pass to the chat template. This is useful for customizing the -# chat template. For example, you can pass `thinking=False` to add a generation prompt -# to the chat template. -chat_template_kwargs: dict[str, Any] | None -# Custom EOT (End-of-Turn) tokens to mask/unmask during training. These tokens mark the -# boundaries between conversation turns. For example: ['/INST', '</s>', -# '[/SYSTEM_PROMPT]']. If not specified, defaults to just the model's eos_token. This is -# useful for templates that use multiple delimiter tokens. -eot_tokens: list[str] | None -# Changes the default system message. Currently only supports chatml. -default_system_message: str | None - -# Token index or indices to adjust embedding weights to the mean of the other tokens. -# This is useful when the model has untrained embeddings. -fix_untrained_tokens: int | list[int] | None - -is_preprocess: bool | None -preprocess_iterable: bool | None - -# Total number of tokens - internal use -total_num_tokens: int | None -total_supervised_tokens: int | None -# You can set these packing optimizations AFTER starting a training at least once. The -# trainer will provide recommended values for these values. -sample_packing_eff_est: float | None -axolotl_config_path: str | None - +# Deepspeed config path. e.g., deepspeed_configs/zero3.json +deepspeed: str | dict[str, Any] | None +# Whether to use deepcompile for faster training with deepspeed +deepcompile: bool | None +# FSDP configuration +fsdp: list[str] | None + +# FSDP configuration options +fsdp_config: FSDPConfig | None + # For FSDPConfig: + # FSDP version + fsdp_version: int | None + # Enable activation checkpointing to reduce memory usage during forward passes + activation_checkpointing: bool | None + # Offload parameters to CPU to reduce GPU memory usage + offload_params: bool | None + # Synchronize module states across all processes + sync_module_states: bool | None + # Enable CPU RAM efficient loading to reduce memory usage during model loading + cpu_ram_efficient_loading: bool | None + # Disabling this enables swap memory usage for resource-constrained setups when + # offload_params is enabled. + cpu_offload_pin_memory: bool | None + # Use original parameters instead of flattened parameters + use_orig_params: bool | None + + # Type of state dict to use for saving/loading checkpoints + state_dict_type: Literal['FULL_STATE_DICT', 'LOCAL_STATE_DICT', 'SHARDED_STATE_DICT'] | None + # Final state dict type to use after training completion + final_state_dict_type: Literal['FULL_STATE_DICT', 'LOCAL_STATE_DICT', 'SHARDED_STATE_DICT'] | None + + # Policy for automatically wrapping modules with FSDP + auto_wrap_policy: Literal['TRANSFORMER_BASED_WRAP', 'SIZE_BASED_WRAP'] | None + # Class name of transformer layers to wrap (e.g., 'LlamaDecoderLayer') + transformer_layer_cls_to_wrap: str | None + + # Reshard parameters after forward pass to save memory + reshard_after_forward: bool | None + # Mixed precision policy for FSDP (e.g., 'fp16', 'bf16') + mixed_precision_policy: str | None + +# FSDP version +fsdp_version: int | None +fsdp_final_state_dict_type: Literal['FULL_STATE_DICT', 'LOCAL_STATE_DICT', 'SHARDED_STATE_DICT'] | None + +# How much of the dataset to set aside as evaluation. 1 = 100%, 0.50 = 50%, etc. 0 for +# no eval. +val_set_size: float | None = 0.0 + +# Number of devices to shard across. If not set, will use all available devices. +dp_shard_size: int | None +# Number of devices to replicate across. +dp_replicate_size: int | None +# Deprecated: use `context_parallel_size` instead +sequence_parallel_degree: int | None +# Set to a divisor of the number of GPUs available to split sequences into chunks of +# equal size. Use in long context training to prevent OOM when sequences cannot fit into +# a single GPU's VRAM. E.g., if 4 GPUs are available, set this value to 2 to split each +# sequence into two equal-sized subsequences, or set to 4 to split into four equal-sized +# subsequences. See https://docs.axolotl.ai/docs/sequence_parallelism.html for more +# details. +context_parallel_size: int | None +# Optional; strides across the key dimension. Larger values use more memory but should +# make training faster. Must evenly divide the number of KV heads in your model. +heads_k_stride: int | None +# One of 'varlen_llama3', 'batch_ring', 'batch_zigzag', 'batch_stripe'. Defaults to +# 'varlen_llama3' in the sample packing case, and 'batch_ring' in the non-sample packing +# case. +ring_attn_func: RingAttnFunc | None +# Number of tensor parallel processes in TP group. Only supported with DeepSpeed AutoTP. +tensor_parallel_size: int | None + +# Add or change special tokens. If you add tokens here, you don't need to add them to +# the `tokens` list. +special_tokens: SpecialTokensConfig | None + # For SpecialTokensConfig: + bos_token: str | None + eos_token: str | None + pad_token: str | None + unk_token: str | None + additional_special_tokens: list[str] | None + +# Add extra tokens to the tokenizer +tokens: list[str] | None +# Mapping token_id to new_token_string to override reserved added_tokens in the +# tokenizer. Only works for tokens that are not part of the base vocab (aka are +# added_tokens). Can be checked if they exist in tokenizer.json added_tokens. +added_tokens_overrides: dict[int, str] | None + +# Whether to use torch.compile and which backend to use. setting to `auto` will enable +# torch compile when torch>=2.6.0 +torch_compile: Literal['auto'] | bool | None +# Backend to use for torch.compile +torch_compile_backend: str | None +torch_compile_mode: Literal['default', 'reduce-overhead', 'max-autotune'] | None + +# Maximum number of iterations to train for. It precedes num_epochs which means that if +# both are set, num_epochs will not be guaranteed. e.g., when 1 epoch is 1000 steps => +# `num_epochs: 2` and `max_steps: 100` will train for 100 steps +max_steps: int | None +# Number of warmup steps. Cannot use with warmup_ratio +warmup_steps: int | None +# Warmup ratio. Cannot use with warmup_steps +warmup_ratio: float | None +# Leave empty to eval at each epoch, integer for every N steps. float for fraction of +# total steps +eval_steps: int | float | None +# Number of times per epoch to run evals, mutually exclusive with eval_steps +evals_per_epoch: int | None +# Set to `no` to skip evaluation, `epoch` at end of each epoch, leave empty to infer +# from `eval_steps` +eval_strategy: str | None + +# Leave empty to save at each epoch, integer for every N steps. float for fraction of +# total steps +save_steps: int | float | None +# Number of times per epoch to save a checkpoint, mutually exclusive with save_steps +saves_per_epoch: int | None +# Set to `no` to skip checkpoint saves, `epoch` at end of each epoch, `best` when better +# result is achieved, leave empty to infer from `save_steps` +save_strategy: str | None +# Checkpoints saved at a time +save_total_limit: int | None +# Whether to checkpoint a model after the first step of training. Defaults to False. +save_first_step: bool | None + +# Logging frequency +logging_steps: int | None +# Stop training after this many evaluation losses have increased in a row. https://huggi +# ngface.co/transformers/v4.2.2/_modules/transformers/trainer_callback.html#EarlyStoppin +# gCallback +early_stopping_patience: int | None +load_best_model_at_end: bool | None = False +# Save only the model weights, skipping the optimizer. Using this means you can't resume +# from checkpoints. +save_only_model: bool | None = False +# Use tensorboard for logging +use_tensorboard: bool | None +# Enable the pytorch profiler to capture the first N steps of training to the +# output_dir. see https://pytorch.org/blog/understanding-gpu-memory-1/ for more +# information. Snapshots can be visualized @ https://pytorch.org/memory_viz +profiler_steps: int | None +# Which step to start the profiler at. Useful for only capturing a few steps mid-run. +profiler_steps_start: int | None = 0 +# bool of whether to report tokens per second at the end of training. This is not +# supported with pre-training datasets. +include_tokens_per_second: bool | None +# bool of whether to report tokens per second per-gpu during training by measuring +# throughput of non-padding tokens. +include_tkps: bool | None = True +# NEFT https://arxiv.org/abs/2310.05914, set this to a number (paper default is 5) to +# add noise to embeddings. Currently only supported on Llama and Mistral +neftune_noise_alpha: float | None + +# Parameter controlling the relative ratio loss weight in the ORPO loss. Passed to +# `beta` in `ORPOConfig` due to trl mapping. +orpo_alpha: float | None +# Weighting of NLL term in loss from RPO paper +rpo_alpha: float | None +# Target reward margin for the SimPO loss +simpo_gamma: float | None +# Weight of the BC regularizer +cpo_alpha: float | None + +# Factor for desirable loss term in KTO loss +kto_desirable_weight: float | None +# Factor for undesirable loss term in KTO loss +kto_undesirable_weight: float | None +# The beta parameter for the RL training +rl_beta: float | None + +# Defines the max memory usage per gpu on the system. Passed through to transformers +# when loading the model. +max_memory: dict[int | Literal['cpu', 'disk'], int | str] | None +# Limit the memory for all available GPUs to this amount (if an integer, expressed in +# gigabytes); default: unset +gpu_memory_limit: int | str | None +# Whether to use low_cpu_mem_usage +low_cpu_mem_usage: bool | None + +# The name of the chat template to use for training, following values are supported: +# tokenizer_default: Uses the chat template that is available in the +# tokenizer_config.json. If the chat template is not available in the tokenizer, it will +# raise an error. This is the default value. +# alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates +# are available in the axolotl codebase at src/axolotl/utils/chat_templates.py. +# tokenizer_default_fallback_*: where * is the name of the chat template to fallback to. +# E.g. tokenizer_default_fallback_chatml. This is useful when the chat template is not +# available in the tokenizer. jinja: Uses a custom jinja template for the chat template. +# The custom jinja template should be provided in the chat_template_jinja field. The +# selected chat template will be saved to the tokenizer_config.json for easier +# inferencing +chat_template: ChatTemplate | Annotated[str, StringConstraints(pattern='^tokenizer_default_fallback_')] | None +# Custom jinja template or path to jinja file for chat template. This will be only used +# if chat_template is set to `jinja` or `null` (in which case chat_template is +# automatically set to `jinja`). Default is null. +chat_template_jinja: str | None +# Additional kwargs to pass to the chat template. This is useful for customizing the +# chat template. For example, you can pass `thinking=False` to add a generation prompt +# to the chat template. +chat_template_kwargs: dict[str, Any] | None +# Custom EOT (End-of-Turn) tokens to mask/unmask during training. These tokens mark the +# boundaries between conversation turns. For example: ['/INST', '</s>', +# '[/SYSTEM_PROMPT]']. If not specified, defaults to just the model's eos_token. This is +# useful for templates that use multiple delimiter tokens. +eot_tokens: list[str] | None +# Changes the default system message. Currently only supports chatml. +default_system_message: str | None + +# Token index or indices to adjust embedding weights to the mean of the other tokens. +# This is useful when the model has untrained embeddings. +fix_untrained_tokens: int | list[int] | None + +is_preprocess: bool | None +preprocess_iterable: bool | None + +# Total number of tokens - internal use +total_num_tokens: int | None +total_supervised_tokens: int | None +# You can set these packing optimizations AFTER starting a training at least once. The +# trainer will provide recommended values for these values. +sample_packing_eff_est: float | None +axolotl_config_path: str | None + +# Internal use only - Used to identify which the model is based on +is_falcon_derived_model: bool | None # Internal use only - Used to identify which the model is based on -is_falcon_derived_model: bool | None -# Internal use only - Used to identify which the model is based on -is_llama_derived_model: bool | None -# Internal use only - Used to identify which the model is based on. Please note that if -# you set this to true, `padding_side` will be set to 'left' by default -is_mistral_derived_model: bool | None -# Internal use only - Used to identify which the model is based on -is_qwen_derived_model: bool | None - -# Add plugins to extend the pipeline. See `src/axolotl/integrations` for the available -# plugins or doc below for more details. -# https://docs.axolotl.ai/docs/custom_integrations.html -plugins: list[str] | None -# Enable sample generation during training for monitoring -generate_samples: bool | None = False -# Number of samples to generate at each interval -num_generation_samples: int | None = 3 -# Maximum new tokens to generate per sample -generation_max_new_tokens: int | None = 50 -# Temperature for sample generation (0.0 = greedy) -generation_temperature: float | None = 0.7 -# Nucleus sampling parameter for generation -generation_top_p: float | None -# Top-k sampling parameter for generation -generation_top_k: int | None -# Ratio of input to use as prompt (0.0-1.0) -generation_prompt_ratio: float | None = 0.5 -# Whether to use sampling (vs greedy decoding) -generation_do_sample: bool | None = True - -# This is the huggingface model that contains *.pt, *.safetensors, or *.bin files. This -# can also be a relative path to a model on disk -base_model: str (required) -# If the base_model repo on hf hub doesn't include configuration .json files, You can -# set that here, or leave this empty to default to base_model -base_model_config: str | None -# transformers config class (e.g., 'LlamaConfig', 'MistralConfig'). Defaults to -# AutoConfig. -cls_model_config: str | None -# Optional tokenizer configuration path in case you want to use a different tokenizer -# than the one defined in the base model -tokenizer_config: str | None -# use_fast option for tokenizer loading from_pretrained, default to True -tokenizer_use_fast: bool | None -# Whether to use the legacy tokenizer setting, defaults to True -tokenizer_legacy: bool | None -# Whether to use mistral-common tokenizer. If set to True, it will use the mistral- -# common tokenizer. -tokenizer_use_mistral_common: bool | None -# Corresponding tokenizer for the model AutoTokenizer is a good choice -tokenizer_type: str | None -# transformers processor class -processor_type: str | None -# Whether to save jinja files for tokenizer, transformers default is True -tokenizer_save_jinja_files: bool | None = True -# Trust remote code for untrusted source -trust_remote_code: bool | None - -# Don't move the model to the device before sharding. Set to `false` to revert to legacy -# behavior. -experimental_skip_move_to_device: bool | None = True - -# Use custom kernels, e.g. MegaBlocks. -use_kernels: bool | None - -# Model loading quantization config -model_quantization_config: Literal['Mxfp4Config'] | None -# kwargs for model quantization config -model_quantization_config_kwargs: dict[str, Any] | None - -# Where to save the full-finetuned model to -output_dir: str = ./model-out -# push checkpoints to hub -hub_model_id: str | None -# how to push checkpoints to hub -hub_strategy: str | None -# branch/revision to push to on hub (default: main) -hub_revision: str | None -# Whether to save the model using safetensors format. Defaults to True. -save_safetensors: bool | None = True - -# This will attempt to quantize the model down to 8 bits and use adam 8 bit optimizer -load_in_8bit: bool | None = False -# Use bitsandbytes 4 bit -load_in_4bit: bool | None = False - -# If you want to use 'lora', 'qlora', or 'llama-adapter', or leave blank to train all -# parameters in original model -adapter: Literal['lora', 'qlora', 'llama-adapter'] | None -# If you already have a lora model trained that you want to load, put that here. This -# means after training, if you want to test the model, you should set this to the value -# of `output_dir`. Note that if you merge an adapter to the base model, a new -# subdirectory `merged` will be created under the `output_dir`. -lora_model_dir: str | None -lora_r: int | None -lora_alpha: int | None -lora_fan_in_fan_out: bool | None -lora_target_modules: str | list[str] | None -lora_target_parameters: str | list[str] | None -# If true, will target all linear modules -lora_target_linear: bool | None -# If you added new tokens to the tokenizer, you may need to save some LoRA modules -# because they need to know the new tokens. For LLaMA and Mistral, you need to save -# `embed_tokens` and `lm_head`. It may vary for other models. `embed_tokens` converts -# tokens to embeddings, and `lm_head` converts embeddings to token probabilities. -lora_modules_to_save: list[str] | None -lora_dropout: float | None = 0.0 -# The layer indices to transform, otherwise, apply to all layers -peft_layers_to_transform: list[int] | None -peft_layers_pattern: list[str] | None - -peft: PeftConfig | None - # For PeftConfig: - # Configuration options for loftq initialization for LoRA - loftq_config: LoftQConfig | None - # For LoftQConfig: - # typically 4 bits - loftq_bits: int = 4 - -# Whether to use DoRA. -peft_use_dora: bool | None -# Whether to use RSLoRA. -peft_use_rslora: bool | None -# List of layer indices to replicate. -peft_layer_replication: list[tuple[int, int]] | None -# How to initialize LoRA weights. Default to True which is MS original implementation. -peft_init_lora_weights: bool | str | None -# A list of token indices to fine-tune on the `embed_tokens` layer. Otherwise, a dict -# mapping an embedding layer name to its trainable token indices. See -# https://huggingface.co/docs/peft/v0.17.0/en/developer_guides/lora#efficiently-train- -# tokens-alongside-lora -peft_trainable_token_indices: list[int] | dict[str, list[int]] | None -# Whether to tie adapter weights for tied model weights. See -# https://github.com/huggingface/peft/issues/2864 -peft_ensure_weight_tying: bool | None -# Whether to upcast the LoRA adapter to fp32. This is enabled by default in PEFT. -peft_autocast_adapter_dtype: bool | None - -# load qlora model in sharded format for FSDP using answer.ai technique. -qlora_sharded_model_loading: bool | None = False -# Do the LoRA/PEFT loading on CPU -- this is required if the base model is so large it -# takes up most or all of the available GPU VRAM, e.g. during a model and LoRA merge -lora_on_cpu: bool | None -# Whether you are training a 4-bit GPTQ quantized model -gptq: bool | None -# optional overrides to the bnb 4bit quantization configuration -bnb_config_kwargs: dict[str, Any] | None - -# loraplus learning rate ratio lr_B / lr_A. Recommended value is 2^4. -loraplus_lr_ratio: float | None -# loraplus learning rate for lora embedding layers. Default value is 1e-6. -loraplus_lr_embedding: float | None = 1e-06 +is_llama_derived_model: bool | None +# Internal use only - Used to identify which the model is based on. Please note that if +# you set this to true, `padding_side` will be set to 'left' by default +is_mistral_derived_model: bool | None +# Internal use only - Used to identify which the model is based on +is_qwen_derived_model: bool | None + +# Add plugins to extend the pipeline. See `src/axolotl/integrations` for the available +# plugins or doc below for more details. +# https://docs.axolotl.ai/docs/custom_integrations.html +plugins: list[str] | None +# Enable sample generation during training for monitoring +generate_samples: bool | None = False +# Number of samples to generate at each interval +num_generation_samples: int | None = 3 +# Maximum new tokens to generate per sample +generation_max_new_tokens: int | None = 50 +# Temperature for sample generation (0.0 = greedy) +generation_temperature: float | None = 0.7 +# Nucleus sampling parameter for generation +generation_top_p: float | None +# Top-k sampling parameter for generation +generation_top_k: int | None +# Ratio of input to use as prompt (0.0-1.0) +generation_prompt_ratio: float | None = 0.5 +# Whether to use sampling (vs greedy decoding) +generation_do_sample: bool | None = True + +# This is the huggingface model that contains *.pt, *.safetensors, or *.bin files. This +# can also be a relative path to a model on disk +base_model: str (required) +# If the base_model repo on hf hub doesn't include configuration .json files, You can +# set that here, or leave this empty to default to base_model +base_model_config: str | None +# transformers config class (e.g., 'LlamaConfig', 'MistralConfig'). Defaults to +# AutoConfig. +cls_model_config: str | None +# Optional tokenizer configuration path in case you want to use a different tokenizer +# than the one defined in the base model +tokenizer_config: str | None +# use_fast option for tokenizer loading from_pretrained, default to True +tokenizer_use_fast: bool | None +# Whether to use the legacy tokenizer setting, defaults to True +tokenizer_legacy: bool | None +# Whether to use mistral-common tokenizer. If set to True, it will use the mistral- +# common tokenizer. +tokenizer_use_mistral_common: bool | None +# Corresponding tokenizer for the model AutoTokenizer is a good choice +tokenizer_type: str | None +# transformers processor class +processor_type: str | None +# Whether to save jinja files for tokenizer, transformers default is True +tokenizer_save_jinja_files: bool | None = True +# Trust remote code for untrusted source +trust_remote_code: bool | None + +# Don't move the model to the device before sharding. Set to `false` to revert to legacy +# behavior. +experimental_skip_move_to_device: bool | None = True + +# Use custom kernels, e.g. MegaBlocks. +use_kernels: bool | None + +# Model loading quantization config +model_quantization_config: Literal['Mxfp4Config'] | None +# kwargs for model quantization config +model_quantization_config_kwargs: dict[str, Any] | None + +# Where to save the full-finetuned model to +output_dir: str = ./model-out +# push checkpoints to hub +hub_model_id: str | None +# how to push checkpoints to hub +hub_strategy: str | None +# branch/revision to push to on hub (default: main) +hub_revision: str | None +# Whether to save the model using safetensors format. Defaults to True. +save_safetensors: bool | None = True + +# This will attempt to quantize the model down to 8 bits and use adam 8 bit optimizer +load_in_8bit: bool | None = False +# Use bitsandbytes 4 bit +load_in_4bit: bool | None = False + +# If you want to use 'lora', 'qlora', or 'llama-adapter', or leave blank to train all +# parameters in original model +adapter: Literal['lora', 'qlora', 'llama-adapter'] | None +# If you already have a lora model trained that you want to load, put that here. This +# means after training, if you want to test the model, you should set this to the value +# of `output_dir`. Note that if you merge an adapter to the base model, a new +# subdirectory `merged` will be created under the `output_dir`. +lora_model_dir: str | None +lora_r: int | None +lora_alpha: int | None +lora_fan_in_fan_out: bool | None +lora_target_modules: str | list[str] | None +lora_target_parameters: str | list[str] | None +# If true, will target all linear modules +lora_target_linear: bool | None +# If you added new tokens to the tokenizer, you may need to save some LoRA modules +# because they need to know the new tokens. For LLaMA and Mistral, you need to save +# `embed_tokens` and `lm_head`. It may vary for other models. `embed_tokens` converts +# tokens to embeddings, and `lm_head` converts embeddings to token probabilities. +lora_modules_to_save: list[str] | None +lora_dropout: float | None = 0.0 +# The layer indices to transform, otherwise, apply to all layers +peft_layers_to_transform: list[int] | None +peft_layers_pattern: list[str] | None + +peft: PeftConfig | None + # For PeftConfig: + # Configuration options for loftq initialization for LoRA + loftq_config: LoftQConfig | None + # For LoftQConfig: + # typically 4 bits + loftq_bits: int = 4 + +# Whether to use DoRA. +peft_use_dora: bool | None +# Whether to use RSLoRA. +peft_use_rslora: bool | None +# List of layer indices to replicate. +peft_layer_replication: list[tuple[int, int]] | None +# How to initialize LoRA weights. Default to True which is MS original implementation. +peft_init_lora_weights: bool | str | None +# A list of token indices to fine-tune on the `embed_tokens` layer. Otherwise, a dict +# mapping an embedding layer name to its trainable token indices. See +# https://huggingface.co/docs/peft/v0.17.0/en/developer_guides/lora#efficiently-train- +# tokens-alongside-lora +peft_trainable_token_indices: list[int] | dict[str, list[int]] | None +# Whether to tie adapter weights for tied model weights. See +# https://github.com/huggingface/peft/issues/2864 +peft_ensure_weight_tying: bool | None +# Whether to upcast the LoRA adapter to fp32. This is enabled by default in PEFT. +peft_autocast_adapter_dtype: bool | None + +# load qlora model in sharded format for FSDP using answer.ai technique. +qlora_sharded_model_loading: bool | None = False +# Do the LoRA/PEFT loading on CPU -- this is required if the base model is so large it +# takes up most or all of the available GPU VRAM, e.g. during a model and LoRA merge +lora_on_cpu: bool | None +# Whether you are training a 4-bit GPTQ quantized model +gptq: bool | None +# optional overrides to the bnb 4bit quantization configuration +bnb_config_kwargs: dict[str, Any] | None + +# loraplus learning rate ratio lr_B / lr_A. Recommended value is 2^4. +loraplus_lr_ratio: float | None +# loraplus learning rate for lora embedding layers. Default value is 1e-6. +loraplus_lr_embedding: float | None = 1e-06 + +merge_lora: bool | None -merge_lora: bool | None - -# Whether to use ReLoRA. Use with jagged_restart_*steps options. -relora: bool | None -# threshold for optimizer magnitude when pruning -relora_prune_ratio: float | None -# True to perform lora weight merges on cpu during restarts, for modest gpu memory -# savings -relora_cpu_offload: bool | None - -# how often to reset for jagged restarts -jagged_restart_steps: int | None -# how many warmup steps to take after reset for jagged restarts -jagged_restart_warmup_steps: int | None -# how many anneal steps to take before reset for jagged restarts -jagged_restart_anneal_steps: int | None - -# If greater than 1, backpropagation will be skipped and the gradients will be -# accumulated for the given number of steps. -gradient_accumulation_steps: int | None = 1 -# The number of samples to include in each batch. This is the number of samples sent to -# each GPU. Batch size per gpu = micro_batch_size * gradient_accumulation_steps -micro_batch_size: int | None = 1 -# Total batch size, we do not recommended setting this manually -batch_size: int | None -# per gpu micro batch size for evals, defaults to value of micro_batch_size -eval_batch_size: int | None - -# whether to find batch size that fits in memory. Passed to underlying transformers -# Trainer -auto_find_batch_size: bool | None - -# Whether to mask out or include the human's prompt from the training labels -train_on_inputs: bool | None = False -# Group similarly sized data to minimize padding. May be slower to start, as it must -# download and sort the entire dataset. Note that training loss may have an oscillating -# pattern with this enabled. -group_by_length: bool | None - -learning_rate: str | float (required) -embedding_lr: float | None -embedding_lr_scale: float | None -# Specify weight decay -weight_decay: float | None = 0.0 -# Specify optimizer -optimizer: OptimizerNames | CustomSupportedOptimizers | None = OptimizerNames.ADAMW_TORCH_FUSED -# Dictionary of arguments to pass to the optimizer -optim_args: str | dict[str, Any] | None -# The target modules to optimize, i.e. the module names that you would like to train, -# right now this is used only for GaLore algorithm -optim_target_modules: list[str] | Literal['all_linear'] | None -# Path to torch distx for optim 'adamw_anyprecision' -torchdistx_path: str | None -lr_scheduler: SchedulerType | Literal['one_cycle'] | Literal['rex'] | None = SchedulerType.COSINE -# Specify a scheduler and kwargs to use with the optimizer -lr_scheduler_kwargs: dict[str, Any] | None -lr_quadratic_warmup: bool | None -# decay lr to some percentage of the peak lr, e.g. cosine_min_lr_ratio=0.1 for 10% of -# peak lr -cosine_min_lr_ratio: float | None -# freeze lr at some percentage of the step, e.g. cosine_constant_lr_ratio=0.8 means -# start cosine_min_lr at 80% of training step -cosine_constant_lr_ratio: float | None -# Learning rate div factor -lr_div_factor: float | None - -lr_groups: list[LrGroup] | None - # For LrGroup: - name: str (required) - modules: list[str] (required) - lr: float (required) - -# adamw hyperparams -adam_epsilon: float | None -# only used for CAME Optimizer -adam_epsilon2: float | None +# Whether to use ReLoRA. Use with jagged_restart_*steps options. +relora: bool | None +# threshold for optimizer magnitude when pruning +relora_prune_ratio: float | None +# True to perform lora weight merges on cpu during restarts, for modest gpu memory +# savings +relora_cpu_offload: bool | None + +# how often to reset for jagged restarts +jagged_restart_steps: int | None +# how many warmup steps to take after reset for jagged restarts +jagged_restart_warmup_steps: int | None +# how many anneal steps to take before reset for jagged restarts +jagged_restart_anneal_steps: int | None + +# If greater than 1, backpropagation will be skipped and the gradients will be +# accumulated for the given number of steps. +gradient_accumulation_steps: int | None = 1 +# The number of samples to include in each batch. This is the number of samples sent to +# each GPU. Batch size per gpu = micro_batch_size * gradient_accumulation_steps +micro_batch_size: int | None = 1 +# Total batch size, we do not recommended setting this manually +batch_size: int | None +# per gpu micro batch size for evals, defaults to value of micro_batch_size +eval_batch_size: int | None + +# whether to find batch size that fits in memory. Passed to underlying transformers +# Trainer +auto_find_batch_size: bool | None + +# Whether to mask out or include the human's prompt from the training labels +train_on_inputs: bool | None = False +# Group similarly sized data to minimize padding. May be slower to start, as it must +# download and sort the entire dataset. Note that training loss may have an oscillating +# pattern with this enabled. +group_by_length: bool | None + +learning_rate: str | float (required) +embedding_lr: float | None +embedding_lr_scale: float | None +# Specify weight decay +weight_decay: float | None = 0.0 +# Specify optimizer +optimizer: OptimizerNames | CustomSupportedOptimizers | None = OptimizerNames.ADAMW_TORCH_FUSED +# Dictionary of arguments to pass to the optimizer +optim_args: str | dict[str, Any] | None +# The target modules to optimize, i.e. the module names that you would like to train, +# right now this is used only for GaLore algorithm +optim_target_modules: list[str] | Literal['all_linear'] | None +# Path to torch distx for optim 'adamw_anyprecision' +torchdistx_path: str | None +lr_scheduler: SchedulerType | Literal['one_cycle'] | Literal['rex'] | None = SchedulerType.COSINE +# Specify a scheduler and kwargs to use with the optimizer +lr_scheduler_kwargs: dict[str, Any] | None +lr_quadratic_warmup: bool | None +# decay lr to some percentage of the peak lr, e.g. cosine_min_lr_ratio=0.1 for 10% of +# peak lr +cosine_min_lr_ratio: float | None +# freeze lr at some percentage of the step, e.g. cosine_constant_lr_ratio=0.8 means +# start cosine_min_lr at 80% of training step +cosine_constant_lr_ratio: float | None +# Learning rate div factor +lr_div_factor: float | None + +lr_groups: list[LrGroup] | None + # For LrGroup: + name: str (required) + modules: list[str] (required) + lr: float (required) + +# adamw hyperparams +adam_epsilon: float | None +# only used for CAME Optimizer +adam_epsilon2: float | None +# adamw hyperparams +adam_beta1: float | None # adamw hyperparams -adam_beta1: float | None -# adamw hyperparams -adam_beta2: float | None -# only used for CAME Optimizer -adam_beta3: float | None - -# Dion Optimizer learning rate -dion_lr: float | None -# Dion Optimizer momentum -dion_momentum: float | None -# Dion Optimizer: r/d fraction for low-rank approximation. Used to compute the low-rank -# dimension. -dion_rank_fraction: float | None = 1.0 -# Dion Optimizer: Round up the low-rank dimension to a multiple of this number. This may -# be useful to ensure even sharding. -dion_rank_multiple_of: int | None = 1 - -# Gradient clipping max norm -max_grad_norm: float | None -num_epochs: float = 1.0 - -use_wandb: bool | None -# Set the name of your wandb run -wandb_name: str | None -# Set the ID of your wandb run -wandb_run_id: str | None -# "offline" to save run metadata locally and not sync to the server, "disabled" to turn -# off wandb -wandb_mode: str | None -# Your wandb project name -wandb_project: str | None -# A wandb Team name if using a Team -wandb_entity: str | None -wandb_watch: str | None -# "checkpoint" to log model to wandb Artifacts every `save_steps` or "end" to log only -# at the end of training -wandb_log_model: str | None - -use_mlflow: bool | None -# URI to mlflow -mlflow_tracking_uri: str | None -# Your experiment name -mlflow_experiment_name: str | None -# Your run name -mlflow_run_name: str | None -# set to true to copy each saved checkpoint on each save to mlflow artifact registry -hf_mlflow_log_artifacts: bool | None - -# Enable or disable Comet integration. -use_comet: bool | None -# API key for Comet. Recommended to set via `comet login`. -comet_api_key: str | None -# Workspace name in Comet. Defaults to the user's default workspace. -comet_workspace: str | None -# Project name in Comet. Defaults to Uncategorized. -comet_project_name: str | None -# Identifier for the experiment. Used to append data to an existing experiment or -# control the key of new experiments. Default to a random key. -comet_experiment_key: str | None -# Create a new experiment ("create") or log to an existing one ("get"). Default -# ("get_or_create") auto-selects based on configuration. -comet_mode: str | None -# Set to True to log data to Comet server, or False for offline storage. Default is -# True. -comet_online: bool | None -# Dictionary for additional configuration settings, see the doc for more details. -comet_experiment_config: dict[str, Any] | None - -use_trackio: bool | None -# Your trackio project name -trackio_project_name: str | None -# Set the name of your trackio run -trackio_run_name: str | None -# Hugging Face Space ID to sync dashboard to (optional, runs locally if not provided) -trackio_space_id: str | None - -# Enable OpenTelemetry metrics collection and Prometheus export -use_otel_metrics: bool | None = False -# Host to bind the OpenTelemetry metrics server to -otel_metrics_host: str | None = localhost -# Port for the Prometheus metrics HTTP server -otel_metrics_port: int | None = 8000 - -# the number of activate layers in LISA -lisa_n_layers: int | None -# how often to switch layers in LISA -lisa_step_interval: int | None -# path under the model to access the layers -lisa_layers_attribute: str | None = model.layers - -gradio_title: str | None -gradio_share: bool | None -gradio_server_name: str | None -gradio_server_port: int | None -gradio_max_new_tokens: int | None -gradio_temperature: float | None - -use_ray: bool = False -ray_run_name: str | None -ray_num_workers: int = 1 -resources_per_worker: dict - -# The size of the image to resize to. It can be an integer (resized into padded-square -# image) or a tuple (width, height).If not provided, we will attempt to load from -# preprocessor.size, otherwise, images won't be resized. -image_size: int | tuple[int, int] | None -# The resampling algorithm to use for image resizing. Default is bilinear. Please refer -# to PIL.Image.Resampling for more details. -image_resize_algorithm: Literal['bilinear', 'bicubic', 'lanczos'] | Resampling | None - -# optional overrides to the base model configuration -overrides_of_model_config: dict[str, Any] | None -# optional overrides the base model loading from_pretrained -overrides_of_model_kwargs: dict[str, Any] | None -# If you want to specify the type of model to load, AutoModelForCausalLM is a good -# choice too -type_of_model: str | None -# You can specify to choose a specific model revision from huggingface hub -revision_of_model: str | None - -max_packed_sequence_len: int | None -rope_scaling: Any | None -noisy_embedding_alpha: float | None -dpo_beta: float | None -evaluation_strategy: str | None -eval_table_size: int | None -eval_max_new_tokens: int | None +adam_beta2: float | None +# only used for CAME Optimizer +adam_beta3: float | None + +# Dion Optimizer learning rate +dion_lr: float | None +# Dion Optimizer momentum +dion_momentum: float | None +# Dion Optimizer: r/d fraction for low-rank approximation. Used to compute the low-rank +# dimension. +dion_rank_fraction: float | None = 1.0 +# Dion Optimizer: Round up the low-rank dimension to a multiple of this number. This may +# be useful to ensure even sharding. +dion_rank_multiple_of: int | None = 1 + +# Gradient clipping max norm +max_grad_norm: float | None +num_epochs: float = 1.0 + +use_wandb: bool | None +# Set the name of your wandb run +wandb_name: str | None +# Set the ID of your wandb run +wandb_run_id: str | None +# "offline" to save run metadata locally and not sync to the server, "disabled" to turn +# off wandb +wandb_mode: str | None +# Your wandb project name +wandb_project: str | None +# A wandb Team name if using a Team +wandb_entity: str | None +wandb_watch: str | None +# "checkpoint" to log model to wandb Artifacts every `save_steps` or "end" to log only +# at the end of training +wandb_log_model: str | None + +use_mlflow: bool | None +# URI to mlflow +mlflow_tracking_uri: str | None +# Your experiment name +mlflow_experiment_name: str | None +# Your run name +mlflow_run_name: str | None +# set to true to copy each saved checkpoint on each save to mlflow artifact registry +hf_mlflow_log_artifacts: bool | None + +# Enable or disable Comet integration. +use_comet: bool | None +# API key for Comet. Recommended to set via `comet login`. +comet_api_key: str | None +# Workspace name in Comet. Defaults to the user's default workspace. +comet_workspace: str | None +# Project name in Comet. Defaults to Uncategorized. +comet_project_name: str | None +# Identifier for the experiment. Used to append data to an existing experiment or +# control the key of new experiments. Default to a random key. +comet_experiment_key: str | None +# Create a new experiment ("create") or log to an existing one ("get"). Default +# ("get_or_create") auto-selects based on configuration. +comet_mode: str | None +# Set to True to log data to Comet server, or False for offline storage. Default is +# True. +comet_online: bool | None +# Dictionary for additional configuration settings, see the doc for more details. +comet_experiment_config: dict[str, Any] | None + +use_trackio: bool | None +# Your trackio project name +trackio_project_name: str | None +# Set the name of your trackio run +trackio_run_name: str | None +# Hugging Face Space ID to sync dashboard to (optional, runs locally if not provided) +trackio_space_id: str | None + +# Enable OpenTelemetry metrics collection and Prometheus export +use_otel_metrics: bool | None = False +# Host to bind the OpenTelemetry metrics server to +otel_metrics_host: str | None = localhost +# Port for the Prometheus metrics HTTP server +otel_metrics_port: int | None = 8000 + +# the number of activate layers in LISA +lisa_n_layers: int | None +# how often to switch layers in LISA +lisa_step_interval: int | None +# path under the model to access the layers +lisa_layers_attribute: str | None = model.layers + +gradio_title: str | None +gradio_share: bool | None +gradio_server_name: str | None +gradio_server_port: int | None +gradio_max_new_tokens: int | None +gradio_temperature: float | None + +use_ray: bool = False +ray_run_name: str | None +ray_num_workers: int = 1 +resources_per_worker: dict + +# The size of the image to resize to. It can be an integer (resized into padded-square +# image) or a tuple (width, height).If not provided, we will attempt to load from +# preprocessor.size, otherwise, images won't be resized. +image_size: int | tuple[int, int] | None +# The resampling algorithm to use for image resizing. Default is bilinear. Please refer +# to PIL.Image.Resampling for more details. +image_resize_algorithm: Literal['bilinear', 'bicubic', 'lanczos'] | Resampling | None + +# optional overrides to the base model configuration +overrides_of_model_config: dict[str, Any] | None +# optional overrides the base model loading from_pretrained +overrides_of_model_kwargs: dict[str, Any] | None +# If you want to specify the type of model to load, AutoModelForCausalLM is a good +# choice too +type_of_model: str | None +# You can specify to choose a specific model revision from huggingface hub +revision_of_model: str | None + +max_packed_sequence_len: int | None +rope_scaling: Any | None +noisy_embedding_alpha: float | None +dpo_beta: float | None +evaluation_strategy: str | None +eval_table_size: int | None +eval_max_new_tokens: int | None +dpo_use_logits_to_keep: bool | None +dpo_generate_during_eval: bool | None diff --git a/docs/custom_integrations.html b/docs/custom_integrations.html index 20c1e847b..5f97886c8 100644 --- a/docs/custom_integrations.html +++ b/docs/custom_integrations.html @@ -787,7 +787,10 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});
  • Kernels Integration
    • @@ -815,7 +818,7 @@ gtag('config', 'G-9KYCVJBNMQ', { 'anonymize_ip': true});
  • Liger Kernels
  • Spectrum @@ -1231,7 +1234,7 @@ The quick brown fox jumps over the loud dog "batched_mm": batched_mm_experts_forward, "grouped_mm": grouped_mm_experts_forward, } -

    In our custom integration, we add support for ScatterMoE, which is even more efficient and faster than grouped_mm.

    +

    In our custom integration, we add support for ScatterMoE and SonicMoE, which are more efficient and faster than grouped_mm.

    Usage

    Add the following to your axolotl YAML config:

    @@ -1239,21 +1242,50 @@ The quick brown fox jumps over the loud dog - axolotl.integrations.kernels.KernelsPlugin use_kernels: true -use_scattermoe: true -

    Important: Setting experts_implementation is incompatible with use_scattermoe.

    + +use_scattermoe: true +use_sonicmoe: true +

    Important: Setting experts_implementation is incompatible with custom kernel options.

    +
    +
    +

    SonicMoE installation

    +

    Prerequisites: +- NVIDIA Hopper (H100, H200) or Blackwell (B200, GB200) GPU +- CUDA 12.9+ (13.0+ for B300) +- PyTorch 2.7+ (2.9.1 recommended) +- For B300: Triton 3.6.0

    +
    pip install --ignore-requires-python --no-deps "sonic-moe @ git+https://github.com/Dao-AILab/sonic-moe.git@116e2df0a41874f77fa0ad269ce7df3f0cfcb956" && pip install nvidia-cutlass-dsl==4.4.0 quack-kernels==0.2.5
    +

    See the SonicMoE installation guide for the latest prerequisite details.

    +

    Note: Blackwell support is in upstream beta. On Blackwell GPUs, Axolotl automatically sets USE_QUACK_GEMM=1 to enable the Blackwell kernels.

    How It Works

    The KernelsPlugin runs before model loading and:

    +
    +
    +

    ScatterMoE

      -
    1. Registers the ScatterMoE kernel from the axolotl-ai-co/scattermoe Hub repo.
      2. +
    2. Registers the ScatterMoE kernel from the local libs/scattermoe_lora package (includes fused LoRA support via Triton kernels).
    3. Patches the model’s SparseMoeBlock forward method with the optimized ScatterMoE implementation.
    -

    This works for any MoE model in transformers that uses a SparseMoeBlock class (Mixtral, Qwen2-MoE, OLMoE, etc.).

    +
    +
    +

    SonicMoE

    +
      +
    1. Resolves the model’s MoE block class(es) from constants.py.
      2. +
    2. Patches the forward method with SonicMoE’s optimized kernels and registers a weight converter for the interleaved gate/up projection format.
      3. +
    3. Supports both softmax->topk and sigmoid->topk routing strategies.
      4. +
    +

    Both paths use the shared resolve_moe_block_classes utility in constants.py for model-type-to-class resolution.

    +
    +

    Supported Models

    +

    See constants.py for the full list of supported model types (Qwen2-MoE, Qwen3-MoE, OLMoE, Mixtral, DeepSeek-V3, GLM-MoE, MiniMax, etc.).

    +

    Limitations

    -

    ScatterMoE uses a softmax -> topk routing, so results may be different for some model arch as baseline (GPT-OSS, GLM_MOE_DSA).

    +

    ScatterMoE uses a softmax -> topk routing, so results may be different for some model architectures as baseline (GPT-OSS, etc). Incompatible with GLM_MOE_DSA (GLM 5) and GLM4_MOE_LITE (GLM 4.7 Flash) at the moment.

    +

    SonicMoE supports both softmax->topk and sigmoid->topk routing, covering a wider range of architectures.

    ScatterMoE does not work for GLM4.7 Flash (glm4_moe_lite) atm.

    @@ -1266,21 +1298,21 @@ The quick brown fox jumps over the loud dog

    Knowledge Distillation (KD)

    Usage

    -
    plugins:
-  - "axolotl.integrations.kd.KDPlugin"
-
-kd_trainer: True
-kd_ce_alpha: 0.1
-kd_alpha: 0.9
-kd_temperature: 1.0
-
-torch_compile: True  # torch>=2.6.0, recommended to reduce vram
-
-datasets:
-  - path: ...
-    type: "axolotl.integrations.kd.chat_template"
-    field_messages: "messages_combined"
-    logprobs_field: "llm_text_generation_vllm_logprobs"  # for kd only, field of logprobs
    +
    plugins:
+  - "axolotl.integrations.kd.KDPlugin"
+
+kd_trainer: True
+kd_ce_alpha: 0.1
+kd_alpha: 0.9
+kd_temperature: 1.0
+
+torch_compile: True  # torch>=2.6.0, recommended to reduce vram
+
+datasets:
+  - path: ...
+    type: "axolotl.integrations.kd.chat_template"
+    field_messages: "messages_combined"
+    logprobs_field: "llm_text_generation_vllm_logprobs"  # for kd only, field of logprobs

    An example dataset can be found at axolotl-ai-co/evolkit-logprobs-pipeline-75k-v2-sample

    Please see reference here

    @@ -1295,7 +1327,7 @@ The quick brown fox jumps over the loud dog

    Requirements

    This will install all necessary dependencies to fine-tune sparsified models using the integration.

    @@ -1304,25 +1336,25 @@ The quick brown fox jumps over the loud dog

    Usage

    To enable sparse fine-tuning with this integration, include the plugin in your Axolotl config:

    -
    plugins:
-  - axolotl.integrations.llm_compressor.LLMCompressorPlugin
-
-llmcompressor:
-  recipe:
-    finetuning_stage:
-      finetuning_modifiers:
-        ConstantPruningModifier:
-          targets: [
-            're:.*q_proj.weight',
-            're:.*k_proj.weight',
-            're:.*v_proj.weight',
-            're:.*o_proj.weight',
-            're:.*gate_proj.weight',
-            're:.*up_proj.weight',
-            're:.*down_proj.weight',
-          ]
-          start: 0
-  save_compressed: true
    +
    plugins:
+  - axolotl.integrations.llm_compressor.LLMCompressorPlugin
+
+llmcompressor:
+  recipe:
+    finetuning_stage:
+      finetuning_modifiers:
+        ConstantPruningModifier:
+          targets: [
+            're:.*q_proj.weight',
+            're:.*k_proj.weight',
+            're:.*v_proj.weight',
+            're:.*o_proj.weight',
+            're:.*gate_proj.weight',
+            're:.*up_proj.weight',
+            're:.*down_proj.weight',
+          ]
+          start: 0
+  save_compressed: true

    This plugin does not apply pruning or sparsification itself — it is intended for fine-tuning models that have already been sparsified.

    Pre-sparsified checkpoints can be: - Generated using LLMCompressor @@ -1349,22 +1381,22 @@ The quick brown fox jumps over the loud dog

    After fine-tuning your sparse model, you can leverage vLLM for efficient inference. You can also use LLMCompressor to apply additional quantization to your fine-tuned sparse model before inference for even greater performance benefits.:

    -
    from vllm import LLM, SamplingParams
-
-prompts = [
-    "Hello, my name is",
-    "The president of the United States is",
-    "The capital of France is",
-    "The future of AI is",
-]
-sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
-llm = LLM("path/to/your/sparse/model")
-outputs = llm.generate(prompts, sampling_params)
-
-for output in outputs:
-    prompt = output.prompt
-    generated_text = output.outputs[0].text
-    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
    +
    from vllm import LLM, SamplingParams
+
+prompts = [
+    "Hello, my name is",
+    "The president of the United States is",
+    "The capital of France is",
+    "The future of AI is",
+]
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
+llm = LLM("path/to/your/sparse/model")
+outputs = llm.generate(prompts, sampling_params)
+
+for output in outputs:
+    prompt = output.prompt
+    generated_text = output.outputs[0].text
+    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")

    For more details on vLLM’s capabilities and advanced configuration options, see the official vLLM documentation.

    @@ -1385,37 +1417,37 @@ sparse model before inference for even greater performance benefits.:

    1. Post-Training Evaluation

    When training with the plugin enabled, evaluation runs automatically after training completes:

    -
    plugins:
-  - axolotl.integrations.lm_eval.LMEvalPlugin
-
-lm_eval_tasks:
-  - gsm8k
-  - hellaswag
-  - arc_easy
-
-lm_eval_batch_size: # Batch size for evaluation
-
-output_dir:
    +
    plugins:
+  - axolotl.integrations.lm_eval.LMEvalPlugin
+
+lm_eval_tasks:
+  - gsm8k
+  - hellaswag
+  - arc_easy
+
+lm_eval_batch_size: # Batch size for evaluation
+
+output_dir:

    Run training as usual:

    -
    axolotl train config.yml
    +
    axolotl train config.yml

    2. Standalone CLI Evaluation

    Evaluate any model directly without training:

    -
    lm_eval_model: meta-llama/Llama-2-7b-hf
-
-plugins:
-  - axolotl.integrations.lm_eval.LMEvalPlugin
-
-lm_eval_tasks:
-  - gsm8k
-  - hellaswag
-  - arc_easy
-
-lm_eval_batch_size: 8
-output_dir: ./outputs
    +
    lm_eval_model: meta-llama/Llama-2-7b-hf
+
+plugins:
+  - axolotl.integrations.lm_eval.LMEvalPlugin
+
+lm_eval_tasks:
+  - gsm8k
+  - hellaswag
+  - arc_easy
+
+lm_eval_batch_size: 8
+output_dir: ./outputs

    Run evaluation:

    -
    axolotl lm-eval config.yml
    +
    axolotl lm-eval config.yml

    Model Selection Priority

    @@ -1428,16 +1460,16 @@ sparse model before inference for even greater performance benefits.:

    Citation

    -
    @misc{eval-harness,
-  author       = {Gao, Leo and Tow, Jonathan and Abbasi, Baber and Biderman, Stella and Black, Sid and DiPofi, Anthony and Foster, Charles and Golding, Laurence and Hsu, Jeffrey and Le Noac'h, Alain and Li, Haonan and McDonell, Kyle and Muennighoff, Niklas and Ociepa, Chris and Phang, Jason and Reynolds, Laria and Schoelkopf, Hailey and Skowron, Aviya and Sutawika, Lintang and Tang, Eric and Thite, Anish and Wang, Ben and Wang, Kevin and Zou, Andy},
-  title        = {A framework for few-shot language model evaluation},
-  month        = 07,
-  year         = 2024,
-  publisher    = {Zenodo},
-  version      = {v0.4.3},
-  doi          = {10.5281/zenodo.12608602},
-  url          = {https://zenodo.org/records/12608602}
-}
    +
    @misc{eval-harness,
+  author       = {Gao, Leo and Tow, Jonathan and Abbasi, Baber and Biderman, Stella and Black, Sid and DiPofi, Anthony and Foster, Charles and Golding, Laurence and Hsu, Jeffrey and Le Noac'h, Alain and Li, Haonan and McDonell, Kyle and Muennighoff, Niklas and Ociepa, Chris and Phang, Jason and Reynolds, Laria and Schoelkopf, Hailey and Skowron, Aviya and Sutawika, Lintang and Tang, Eric and Thite, Anish and Wang, Ben and Wang, Kevin and Zou, Andy},
+  title        = {A framework for few-shot language model evaluation},
+  month        = 07,
+  year         = 2024,
+  publisher    = {Zenodo},
+  version      = {v0.4.3},
+  doi          = {10.5281/zenodo.12608602},
+  url          = {https://zenodo.org/records/12608602}
+}

    Please see reference here

    @@ -1452,18 +1484,18 @@ sparse model before inference for even greater performance benefits.:

    See https://github.com/linkedin/Liger-Kernel

    Usage

    -
    plugins:
-  - axolotl.integrations.liger.LigerPlugin
-liger_rope: true
-liger_rms_norm: true
-liger_glu_activation: true
-liger_layer_norm: true
-liger_fused_linear_cross_entropy: true
-
-liger_use_token_scaling: true
    +
    plugins:
+  - axolotl.integrations.liger.LigerPlugin
+liger_rope: true
+liger_rms_norm: true
+liger_glu_activation: true
+liger_layer_norm: true
+liger_fused_linear_cross_entropy: true
+
+liger_use_token_scaling: true
    -
    -

    Supported Models

    +
    +

    Supported Models

    • deepseek_v2
    • gemma
      • @@ -1486,16 +1518,16 @@ sparse model before inference for even greater performance benefits.:

    Citation

    -
    @article{hsu2024ligerkernelefficienttriton,
-      title={Liger Kernel: Efficient Triton Kernels for LLM Training},
-      author={Pin-Lun Hsu and Yun Dai and Vignesh Kothapalli and Qingquan Song and Shao Tang and Siyu Zhu and Steven Shimizu and Shivam Sahni and Haowen Ning and Yanning Chen},
-      year={2024},
-      eprint={2410.10989},
-      archivePrefix={arXiv},
-      primaryClass={cs.LG},
-      url={https://arxiv.org/abs/2410.10989},
-      journal={arXiv preprint arXiv:2410.10989},
-}
    +
    @article{hsu2024ligerkernelefficienttriton,
+      title={Liger Kernel: Efficient Triton Kernels for LLM Training},
+      author={Pin-Lun Hsu and Yun Dai and Vignesh Kothapalli and Qingquan Song and Shao Tang and Siyu Zhu and Steven Shimizu and Shivam Sahni and Haowen Ning and Yanning Chen},
+      year={2024},
+      eprint={2410.10989},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG},
+      url={https://arxiv.org/abs/2410.10989},
+      journal={arXiv preprint arXiv:2410.10989},
+}

    Please see reference here

    @@ -1511,23 +1543,23 @@ By identifying the top n% of layers with the highest SNR, you can optimize train

    Usage

    -
    plugins:
-  - axolotl.integrations.spectrum.SpectrumPlugin
-
-spectrum_top_fraction: 0.5
-spectrum_model_name: meta-llama/Meta-Llama-3.1-8B
    +
    plugins:
+  - axolotl.integrations.spectrum.SpectrumPlugin
+
+spectrum_top_fraction: 0.5
+spectrum_model_name: meta-llama/Meta-Llama-3.1-8B

    Citation

    -
    @misc{hartford2024spectrumtargetedtrainingsignal,
-      title={Spectrum: Targeted Training on Signal to Noise Ratio},
-      author={Eric Hartford and Lucas Atkins and Fernando Fernandes Neto and David Golchinfar},
-      year={2024},
-      eprint={2406.06623},
-      archivePrefix={arXiv},
-      primaryClass={cs.LG},
-      url={https://arxiv.org/abs/2406.06623},
-}
    +
    @misc{hartford2024spectrumtargetedtrainingsignal,
+      title={Spectrum: Targeted Training on Signal to Noise Ratio},
+      author={Eric Hartford and Lucas Atkins and Fernando Fernandes Neto and David Golchinfar},
+      year={2024},
+      eprint={2406.06623},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG},
+      url={https://arxiv.org/abs/2406.06623},
+}

    Please see reference here

    @@ -1551,7 +1583,7 @@ By identifying the top n% of layers with the highest SNR, you can optimize train

    Installation

    -
    pip install swanlab
    +
    pip install swanlab

    Quick Start

    @@ -1563,23 +1595,23 @@ By identifying the top n% of layers with the highest SNR, you can optimize train

    2. Configure Axolotl Config File

    Add SwanLab configuration to your Axolotl YAML config:

    -
    plugins:
-  - axolotl.integrations.swanlab.SwanLabPlugin
-
-use_swanlab: true
-swanlab_project: my-llm-project
-swanlab_experiment_name: qwen-finetune-v1
-swanlab_mode: cloud  # Options: cloud, local, offline, disabled
-swanlab_workspace: my-team  # Optional: organization name
-swanlab_api_key: YOUR_API_KEY  # Optional: can also use env var SWANLAB_API_KEY
    +
    plugins:
+  - axolotl.integrations.swanlab.SwanLabPlugin
+
+use_swanlab: true
+swanlab_project: my-llm-project
+swanlab_experiment_name: qwen-finetune-v1
+swanlab_mode: cloud  # Options: cloud, local, offline, disabled
+swanlab_workspace: my-team  # Optional: organization name
+swanlab_api_key: YOUR_API_KEY  # Optional: can also use env var SWANLAB_API_KEY

    3. Run Training

    -
    export SWANLAB_API_KEY=your-api-key-here
-
-swanlab login
-
-accelerate launch -m axolotl.cli.train your-config.yaml
    +
    export SWANLAB_API_KEY=your-api-key-here
+
+swanlab login
+
+accelerate launch -m axolotl.cli.train your-config.yaml

    Configuration Options

    @@ -1721,46 +1753,46 @@ By identifying the top n% of layers with the highest SNR, you can optimize train

    Example 1: Basic Cloud Sync

    -
    plugins:
-  - axolotl.integrations.swanlab.SwanLabPlugin
-
-use_swanlab: true
-swanlab_project: llama-finetune
-swanlab_experiment_name: llama-3-8b-instruct-v1
-swanlab_mode: cloud
    -
    -
    -

    Example 2: Offline/Local Mode

    plugins:
   - axolotl.integrations.swanlab.SwanLabPlugin
 
 use_swanlab: true
-swanlab_project: local-experiments
-swanlab_experiment_name: test-run-1
-swanlab_mode: local  # or 'offline'
    +swanlab_project: llama-finetune +swanlab_experiment_name: llama-3-8b-instruct-v1 +swanlab_mode: cloud
    -
    -

    Example 3: Team Workspace

    +
    +

    Example 2: Offline/Local Mode

    plugins:
   - axolotl.integrations.swanlab.SwanLabPlugin
 
 use_swanlab: true
-swanlab_project: research-project
-swanlab_experiment_name: experiment-42
-swanlab_workspace: my-research-team
-swanlab_mode: cloud
    +swanlab_project: local-experiments +swanlab_experiment_name: test-run-1 +swanlab_mode: local # or 'offline'
    -
    -

    Example 4: Private Deployment

    +
    +

    Example 3: Team Workspace

    plugins:
   - axolotl.integrations.swanlab.SwanLabPlugin
 
 use_swanlab: true
-swanlab_project: internal-project
-swanlab_experiment_name: secure-training
-swanlab_mode: cloud
-swanlab_web_host: https://swanlab.yourcompany.com
-swanlab_api_host: https://api.swanlab.yourcompany.com
    +swanlab_project: research-project +swanlab_experiment_name: experiment-42 +swanlab_workspace: my-research-team +swanlab_mode: cloud +
    +
    +

    Example 4: Private Deployment

    +
    plugins:
+  - axolotl.integrations.swanlab.SwanLabPlugin
+
+use_swanlab: true
+swanlab_project: internal-project
+swanlab_experiment_name: secure-training
+swanlab_mode: cloud
+swanlab_web_host: https://swanlab.yourcompany.com
+swanlab_api_host: https://api.swanlab.yourcompany.com

    Team Notifications with Lark (Feishu)

    @@ -1781,20 +1813,6 @@ By identifying the top n% of layers with the highest SNR, you can optimize train

    Example 5: Basic Lark Notifications

    Send training notifications to a Lark group chat:

    -
    plugins:
-  - axolotl.integrations.swanlab.SwanLabPlugin
-
-use_swanlab: true
-swanlab_project: production-training
-swanlab_experiment_name: llama-3-finetune-v2
-swanlab_mode: cloud
-
-swanlab_lark_webhook_url: https://open.feishu.cn/open-apis/bot/v2/hook/xxxxxxxxxx
    -

    Note: This configuration will work, but you’ll see a security warning recommending HMAC secret configuration.

    -
    - +

  • Verify webhook in Lark: Test webhook manually (see above)

  • Check distributed training: Only rank 0 sends notifications

    -
    # If running multi-GPU, check rank 0 logs specifically
-grep "Registered Lark" logs/rank_0.log
    • +
    # If running multi-GPU, check rank 0 logs specifically
+grep "Registered Lark" logs/rank_0.log

  • Verify SwanLab is initialized: Lark callback needs SwanLab to be running

    -
    use_swanlab: true  # Must be enabled
-swanlab_project: my-project  # Must be set
    • +
    use_swanlab: true  # Must be enabled
+swanlab_project: my-project  # Must be set

  • Check Lark bot permissions: Ensure bot is added to the target group chat

    • @@ -2726,16 +2758,16 @@ INFO: Registered Lark notification callback with HMAC authentication

    Custom Logging

    You can add custom metrics in your callbacks:

    -
    import swanlab
-
-swanlab.log({
-    "custom_metric": value,
-    "epoch": epoch_num
-})
    +
    import swanlab
+
+swanlab.log({
+    "custom_metric": value,
+    "epoch": epoch_num
+})

    Experiment Comparison

    -
    swanlab compare run1 run2 run3
    +
    swanlab compare run1 run2 run3

    Support

    @@ -2798,10 +2830,10 @@ Warning

    If you could not load your integration, please ensure you are pip installing in editable mode.

    -
    pip install -e .
    +
    pip install -e .

    and correctly spelled the integration name in the config file.

    -
    plugins:
-  - axolotl.integrations.your_integration_name.YourIntegrationPlugin
    +
    plugins:
+  - axolotl.integrations.your_integration_name.YourIntegrationPlugin
    diff --git a/search.json b/search.json index c7308b5cd..7f88f3e6f 100644 --- a/search.json +++ b/search.json @@ -2660,7 +2660,7 @@ "href": "docs/config-reference.html", "title": "Config Reference", "section": "", - "text": "# Allow overwrite yml config using from cli\nstrict: bool | None = False\n# Resume from a specific checkpoint dir\nresume_from_checkpoint: str | None\n# If resume_from_checkpoint isn't set and you simply want it to start where it left off.\n# Be careful with this being turned on between different models.\nauto_resume_from_checkpoints: bool | None\n# Resize the model embeddings when new tokens are added to multiples of 32. This is\n# reported to improve training speed on some models\nresize_token_embeddings_to_32x: bool | None\nmean_resizing_embeddings: bool | None = False\n\n# Whether to shrink the embeddings to len(tokenizer). By default, we won't shrink.\nshrink_embeddings: bool | None\n# Don't upcast the embeddings to float32 when using PEFT. Useful for low-VRAM GPUs\nembeddings_skip_upcast: bool | None\n# Reinitialize model weights randomly instead of loading pretrained weights\nreinit_weights: bool | None\n\n# module to custom trainer class to use for training\ntrainer_cls: str | None\n\n# Use RL training: 'dpo', 'ipo', 'kto', 'simpo', 'orpo', 'grpo'\nrl: RLType | None\n\ntrl: TRLConfig | None\n # For TRLConfig:\n # Beta parameter for the RL training. Same as `rl_beta`. Use\n beta: float | None\n # Maximum length of the completion for RL training.\n max_completion_length: int | None\n\n # Whether to use VLLM for RL training.\n use_vllm: bool = False\n # VLLM mode to use, one of 'server' or 'colocate'\n vllm_mode: Literal['server', 'colocate'] | None\n # Host of the vLLM server to connect to.\n vllm_server_host: str | None = 0.0.0.0\n # Port of the vLLM server to connect to.\n vllm_server_port: int | None = 8000\n # Total timeout (in seconds) to wait for the vLLM server to respond.\n vllm_server_timeout: int | None\n # Regex for vLLM guided decoding.\n vllm_guided_decoding_regex: str | None\n\n # List of reward functions to load. Paths must be importable from current dir.\n reward_funcs: list[str] | None\n # List of reward weights for the reward functions.\n reward_weights: list[float] | None\n # Number of generations to sample.\n num_generations: int | None\n # Whether to log completions.\n log_completions: bool | None = False\n # Number of completions to print when log_completions is True.\n num_completions_to_print: int | None\n # Controls whether importance sampling ratios are computed at the `'token'` or\n # `'sequence'` level. For GSPO, use `sequence`, default is None which corresponds to\n # the original GRPO paper.\n importance_sampling_level: Literal['sequence', 'token'] | None\n\n # Whether to sync the reference model.\n sync_ref_model: bool | None = False\n # Mixup alpha for the reference model.\n ref_model_mixup_alpha: float | None = 0.9\n # Sync steps for the reference model.\n ref_model_sync_steps: int | None = 64\n # Whether to scale rewards by their standard deviation.\n scale_rewards: bool = True\n\n # Sampling temperature for the GRPO policy.\n temperature: float | None\n # Top-p sampling probability for the generation policy.\n top_p: float | None\n # Top-k sampling for the generation policy.\n top_k: int | None\n # Minimum probability for the generation policy.\n min_p: float | None\n # Penalty for tokens that appear in prompt and generated text.\n repetition_penalty: float | None\n # Number of iterations per batch (μ) for GRPO.\n num_iterations: int | None\n # Epsilon value for clipping in the GRPO algorithm.\n epsilon: float | None\n # Upper-bound epsilon value for clipping in the GRPO algorithm.\n epsilon_high: float | None\n # Whether to use Liger loss for GRPO.\n use_liger_loss: bool | None\n # Loss formulation to use. Supported values: grpo, bnpo, dr_grpo.\n loss_type: str | None\n # Whether to exclude truncated completions from loss calculation.\n mask_truncated_completions: bool = False\n # Enable sleep mode for vLLM to offload VRAM when idle\n vllm_enable_sleep_mode: bool | None\n # Path to custom rollout function. Must be importable from current dir.\n rollout_func: str | None\n # Multi-objective reward aggregation strategy. 'sum_then_normalize' (GRPO default):\n # weights and sums rewards first, then normalizes. 'normalize_then_sum' (GDPO):\n # normalizes each reward independently, then sums.\n multi_objective_aggregation: Literal['sum_then_normalize', 'normalize_then_sum'] | None\n\nvllm: VllmConfig | None\n # For VllmConfig:\n # Device to use for VLLM\n device: str | None = auto\n # Tensor parallel size for VLLM\n tensor_parallel_size: int | None\n # Data parallel size for VLLM\n data_parallel_size: int | None\n # GPU memory utilization for VLLM\n gpu_memory_utilization: float | None = 0.9\n # Data type for VLLM\n dtype: str | None = auto\n # Maximum length of the model context for VLLM\n max_model_len: int | None\n # Enable prefix caching for VLLM\n enable_prefix_caching: bool | None\n # Host for the vLLM server to start on\n host: str | None = 0.0.0.0\n # Port of the vLLM server to start on\n port: int | None = 8000\n\n # Enable reasoning for VLLM\n enable_reasoning: bool | None\n # Reasoning parser for VLLM\n reasoning_parser: str | None\n\nqat: QATConfig | None\n # For QATConfig:\n # Fake quantization layout to use for activation quantization.\n activation_dtype: TorchAOQuantDType | None\n # Fake quantization layout to use for weight quantization.\n weight_dtype: TorchAOQuantDType = TorchAOQuantDType.int8\n # Quantize embedding\n quantize_embedding: bool | None = False\n # The number of elements in each group for per-group fake quantization\n group_size: int | None = 32\n # The number of steps to apply fake quantization after\n fake_quant_after_n_steps: int | None\n\nquantization: PTQConfig | None\n # For PTQConfig:\n # Fake quantization layout to use for weight quantization.\n weight_dtype: TorchAOQuantDType = TorchAOQuantDType.int8\n # Fake quantization layout to use for activation quantization.\n activation_dtype: TorchAOQuantDType | None\n # Whether to quantize the embedding layer.\n quantize_embedding: bool | None\n # The number of elements in each group for per-group fake quantization\n group_size: int | None = 32\n\n# Reward modelling: `True` or `False`\nreward_model: bool | None\n\n# Configuration for dynamic checkpointing (trigger by file or signal). Set 'enabled:\n# true' to activate this feature.\ndynamic_checkpoint: DynamicCheckpointConfig | None\n # For DynamicCheckpointConfig:\n # Enable dynamic checkpoint triggering during training. Create a file\n # 'axolotl_checkpoint.save' in the configured `output_dir` to trigger.\n enabled: bool = False\n # Check for trigger file every N steps (reduces I/O overhead). Default: 100\n check_interval: int = 10\n # Custom trigger filename (optional). If not specified, defaults to\n # 'axolotl_checkpoint.save'. Specify a filename (not a full path) to override the\n # default.\n trigger_file_path: str = \n\n# Process reward modelling: `True` or `False`\nprocess_reward_model: bool | None\n# Coefficient to incentivize the reward model to output mean-zero rewards (proposed by\n# https://huggingface.co/papers/2312.09244, Eq. 2). Recommended value: `0.01`.\ncenter_rewards_coefficient: float | None\nnum_labels: int | None\n\n# Whether to perform weighting in DPO trainer\ndpo_use_weighting: bool | None\ndpo_use_logits_to_keep: bool | None\ndpo_label_smoothing: float | None\ndpo_norm_loss: bool | None\n\n# Whether to use Liger kernel for DPO loss.\ndpo_use_liger_kernel: bool | None\n\ndpo_padding_free: bool | None\ndpo_generate_during_eval: bool | None\n\n# A list of one or more datasets to finetune the model with\ndatasets: Annotated[list[SFTDataset | DPODataset | KTODataset | StepwiseSupervisedDataset], MinLen(1)] | None\n # For SFTDataset:\n # HuggingFace dataset repo | s3:// | gs:// | path to local file or directory\n path: str | None\n # name of dataset split to load from\n split: str | None\n # The type of prompt to use for training. [alpaca, gpteacher, oasst, reflection]\n type: str | UserDefinedPrompterType | None\n # For UserDefinedPrompterType:\n # Custom user instruction prompt\n system_prompt: str | None\n # Use {system} as key to be replaced\n system_format: str | None\n field_system: str | None\n field_instruction: str | None\n field_input: str | None\n field_output: str | None\n\n # Customizable to be single line or multi-line. Use {instruction}/{input} as key to\n # be replaced. 'format' can include {input}\n format: str | None\n # 'no_input_format' cannot include {input}\n no_input_format: str | None\n input_transform: str | None\n # split dataset into N pieces (use with shards_idx)\n shards: int | None\n # the index of sharded dataset to use\n shards_idx: int | None\n # process dataset in N sequential chunks for memory efficiency (exclusive with\n # `shards`)\n preprocess_shards: int | None\n conversation: str | None\n\n # The name of the chat template to use for training, following values are supported:\n # tokenizer_default: Uses the chat template that is available in the\n # tokenizer_config.json. If the chat template is not available in the tokenizer, it\n # will raise an error. This is the default.\n # alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates\n # are available in the axolotl codebase at src/axolotl/utils/chat_templates.py.\n # tokenizer_default_fallback_*: where * is the name of the chat template to fallback\n # to if the tokenizer does not have a chat template else default to tokenizer. E.g.\n # tokenizer_default_fallback_chatml. jinja: Uses a custom jinja template for the chat\n # template. The custom jinja template should be provided in the chat_template_jinja\n # field.\n chat_template: ChatTemplate | str | None\n # Custom jinja chat template or path to jinja file. Used only if `chat_template:\n # jinja` or empty.\n chat_template_jinja: str | None\n # path to source data files\n data_files: str | list[str] | None\n input_format: str | None\n # name of dataset configuration to load\n name: str | None\n # defines the datatype when path is a file\n ds_type: str | None\n # For `completion` datasets only, uses the provided field instead of `text` column\n field: str | None\n field_human: str | None\n field_model: str | None\n # Key containing the messages (default: \"messages\")\n field_messages: str | None\n # Key containing the tools (default: \"tools\"). Must be a list[dict] and follow [JSON\n # schema](https://json-schema.org/learn/getting-started-step-by-step).\n field_tools: str | None\n # Key containing the reasoning trace (default: \"reasoning_content\").\n field_thinking: str | None\n # The key the chat template expects that indicates the reasoning trace.\n template_thinking_key: str | None\n\n message_field_role: str | None\n\n message_field_content: str | None\n # Mapping of properties from the input dataset to the chat template. (default:\n # message_property_mappings={'role':'role', 'content':'content'}) If a property exists\n # in the template but not in this mapping, the system will attempt to load it directly\n # from the message using the property name as the key. Example: In the mapping below,\n # 'from' is loaded from input dataset and used as 'role', while 'value' is loaded and\n # used as 'content' in the chat template.\n message_property_mappings: dict[str, str] | None\n # The key in the message turn that indicates via boolean whether tokens of a turn\n # should be considered for training. Useful to selectively train on certain turns\n # besides the `roles_to_train`.\n message_field_training: str | None\n # The key in the message turn that contains the training details. Useful to\n # selectively train on certain tokens in a turn. The value of the key is a List[Dict]\n # containing `begin_offset` (start character index in content), `end_offset` (end\n # character index in content), and `train` (boolean whether to train).\n message_field_training_detail: str | None\n # (for Qwen3 template only) Whether to split the assistant content based on a\n # reasoning trace inside delimited tags\n split_thinking: bool | None\n logprobs_field: str | None\n temperature: float | None\n # Roles to train on. The tokens from these roles will be considered for the loss.\n roles_to_train: list[str] | None\n # Which EOS tokens to train on in the conversation. Possible values are: all: train on\n # all EOS tokens, turn (default): train on the EOS token at the end of each trainable\n # turn, last: train on the last EOS token in the conversation\n train_on_eos: Literal['all', 'turn', 'last'] | None\n # Roles mapping in the messages. The format is {target_role: [source_roles]}. All\n # source roles will be mapped to the target role. The default is: user: [\"human\",\n # \"user\"], assistant: [\"gpt\", \"assistant\"], system: [\"system\"], tool: [\"tool\"]\n roles: dict[str, list[str]] | None\n # Whether to drop the system turn from the dataset. Only works with chat_template.\n # This does not drop the default system message from chat_template if it exists. If\n # you wish to, we recommend using a custom jinja template with the default system\n # message removed or adding a system turn with empty content.\n drop_system_message: bool | None\n # Trust remote code for untrusted source\n trust_remote_code: bool | None = False\n # The specific revision of the dataset to use when loading from the Hugging Face Hub.\n # This can be a commit hash, tag, or branch name. If not specified, the latest version\n # will be used. This parameter is ignored for local datasets.\n revision: str | None\n\n # For DPODataset:\n path: str | None\n split: str | None\n type: UserDefinedDPOType | str | None\n # For UserDefinedDPOType:\n field_system: str | None\n field_prompt: str | None\n field_chosen: str | None\n field_rejected: str | None\n prompt_format: str | None\n chosen_format: str | None\n rejected_format: str | None\n data_files: list[str] | None\n revision: str | None\n field_messages: str | None\n\n # For KTODataset:\n path: str | None\n split: str | None\n type: UserDefinedKTOType | str | None\n # For UserDefinedKTOType:\n field_system: str | None\n field_prompt: str | None\n field_completion: str | None\n field_label: bool | None\n prompt_format: str | None\n completion_format: str | None\n data_files: list[str] | None\n trust_remote_code: bool | None = False\n revision: str | None\n\n # For StepwiseSupervisedDataset:\n path: str | None\n split: str | None\n data_files: list[str] | None\n revision: str | None\n step_separator: str | None\n max_completion_length: int | None\n train_on_last_step_only: bool | None\n\n# A list of one or more datasets to eval the model with. You can use either\n# test_datasets, or val_set_size, but not both.\ntest_datasets: Annotated[list[SFTDataset | DPODataset | KTODataset | StepwiseSupervisedDataset], MinLen(1)] | None\n # For SFTDataset:\n # HuggingFace dataset repo | s3:// | gs:// | path to local file or directory\n path: str | None\n # name of dataset split to load from\n split: str | None\n # The type of prompt to use for training. [alpaca, gpteacher, oasst, reflection]\n type: str | UserDefinedPrompterType | None\n # For UserDefinedPrompterType:\n # Custom user instruction prompt\n system_prompt: str | None\n # Use {system} as key to be replaced\n system_format: str | None\n field_system: str | None\n field_instruction: str | None\n field_input: str | None\n field_output: str | None\n\n # Customizable to be single line or multi-line. Use {instruction}/{input} as key to\n # be replaced. 'format' can include {input}\n format: str | None\n # 'no_input_format' cannot include {input}\n no_input_format: str | None\n input_transform: str | None\n # split dataset into N pieces (use with shards_idx)\n shards: int | None\n # the index of sharded dataset to use\n shards_idx: int | None\n # process dataset in N sequential chunks for memory efficiency (exclusive with\n # `shards`)\n preprocess_shards: int | None\n conversation: str | None\n\n # The name of the chat template to use for training, following values are supported:\n # tokenizer_default: Uses the chat template that is available in the\n # tokenizer_config.json. If the chat template is not available in the tokenizer, it\n # will raise an error. This is the default.\n # alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates\n # are available in the axolotl codebase at src/axolotl/utils/chat_templates.py.\n # tokenizer_default_fallback_*: where * is the name of the chat template to fallback\n # to if the tokenizer does not have a chat template else default to tokenizer. E.g.\n # tokenizer_default_fallback_chatml. jinja: Uses a custom jinja template for the chat\n # template. The custom jinja template should be provided in the chat_template_jinja\n # field.\n chat_template: ChatTemplate | str | None\n # Custom jinja chat template or path to jinja file. Used only if `chat_template:\n # jinja` or empty.\n chat_template_jinja: str | None\n # path to source data files\n data_files: str | list[str] | None\n input_format: str | None\n # name of dataset configuration to load\n name: str | None\n # defines the datatype when path is a file\n ds_type: str | None\n # For `completion` datasets only, uses the provided field instead of `text` column\n field: str | None\n field_human: str | None\n field_model: str | None\n # Key containing the messages (default: \"messages\")\n field_messages: str | None\n # Key containing the tools (default: \"tools\"). Must be a list[dict] and follow [JSON\n # schema](https://json-schema.org/learn/getting-started-step-by-step).\n field_tools: str | None\n # Key containing the reasoning trace (default: \"reasoning_content\").\n field_thinking: str | None\n # The key the chat template expects that indicates the reasoning trace.\n template_thinking_key: str | None\n\n message_field_role: str | None\n\n message_field_content: str | None\n # Mapping of properties from the input dataset to the chat template. (default:\n # message_property_mappings={'role':'role', 'content':'content'}) If a property exists\n # in the template but not in this mapping, the system will attempt to load it directly\n # from the message using the property name as the key. Example: In the mapping below,\n # 'from' is loaded from input dataset and used as 'role', while 'value' is loaded and\n # used as 'content' in the chat template.\n message_property_mappings: dict[str, str] | None\n # The key in the message turn that indicates via boolean whether tokens of a turn\n # should be considered for training. Useful to selectively train on certain turns\n # besides the `roles_to_train`.\n message_field_training: str | None\n # The key in the message turn that contains the training details. Useful to\n # selectively train on certain tokens in a turn. The value of the key is a List[Dict]\n # containing `begin_offset` (start character index in content), `end_offset` (end\n # character index in content), and `train` (boolean whether to train).\n message_field_training_detail: str | None\n # (for Qwen3 template only) Whether to split the assistant content based on a\n # reasoning trace inside delimited tags\n split_thinking: bool | None\n logprobs_field: str | None\n temperature: float | None\n # Roles to train on. The tokens from these roles will be considered for the loss.\n roles_to_train: list[str] | None\n # Which EOS tokens to train on in the conversation. Possible values are: all: train on\n # all EOS tokens, turn (default): train on the EOS token at the end of each trainable\n # turn, last: train on the last EOS token in the conversation\n train_on_eos: Literal['all', 'turn', 'last'] | None\n # Roles mapping in the messages. The format is {target_role: [source_roles]}. All\n # source roles will be mapped to the target role. The default is: user: [\"human\",\n # \"user\"], assistant: [\"gpt\", \"assistant\"], system: [\"system\"], tool: [\"tool\"]\n roles: dict[str, list[str]] | None\n # Whether to drop the system turn from the dataset. Only works with chat_template.\n # This does not drop the default system message from chat_template if it exists. If\n # you wish to, we recommend using a custom jinja template with the default system\n # message removed or adding a system turn with empty content.\n drop_system_message: bool | None\n # Trust remote code for untrusted source\n trust_remote_code: bool | None = False\n # The specific revision of the dataset to use when loading from the Hugging Face Hub.\n # This can be a commit hash, tag, or branch name. If not specified, the latest version\n # will be used. This parameter is ignored for local datasets.\n revision: str | None\n\n # For DPODataset:\n path: str | None\n split: str | None\n type: UserDefinedDPOType | str | None\n # For UserDefinedDPOType:\n field_system: str | None\n field_prompt: str | None\n field_chosen: str | None\n field_rejected: str | None\n prompt_format: str | None\n chosen_format: str | None\n rejected_format: str | None\n data_files: list[str] | None\n revision: str | None\n field_messages: str | None\n\n # For KTODataset:\n path: str | None\n split: str | None\n type: UserDefinedKTOType | str | None\n # For UserDefinedKTOType:\n field_system: str | None\n field_prompt: str | None\n field_completion: str | None\n field_label: bool | None\n prompt_format: str | None\n completion_format: str | None\n data_files: list[str] | None\n trust_remote_code: bool | None = False\n revision: str | None\n\n # For StepwiseSupervisedDataset:\n path: str | None\n split: str | None\n data_files: list[str] | None\n revision: str | None\n step_separator: str | None\n max_completion_length: int | None\n train_on_last_step_only: bool | None\n\n# If false, the datasets will not be shuffled and will keep their original order in\n# `datasets`. The same applies to the `test_datasets` option and the\n# `pretraining_dataset` option. Default is true.\nshuffle_merged_datasets: bool | None = True\n# If true, each dataset in `datasets` will be shuffled before merging. This allows\n# curriculum learning strategies to be applied at the dataset level. Default is false.\nshuffle_before_merging_datasets: bool | None = False\n# Axolotl attempts to save the dataset as an arrow after packing the data together so\n# subsequent training attempts load faster, relative path\ndataset_prepared_path: str | None\n# Num shards for whole dataset\ndataset_shard_num: int | None\n# Index of shard to use for whole dataset\ndataset_shard_idx: int | None\nskip_prepare_dataset: bool | None = False\n# Number of shards to save the prepared dataset\nnum_dataset_shards_to_save: int | None\n\n# Set to HF dataset for type: 'completion' for streaming instead of pre-tokenize\npretraining_dataset: Annotated[list[PretrainingDataset | SFTDataset], MinLen(1)] | None\n # For PretrainingDataset:\n name: str | None\n path: str | None\n split: str | None = train\n text_column: str | None = text\n type: str | None = pretrain\n trust_remote_code: bool | None = False\n data_files: str | None\n skip: int | None\n\n # For SFTDataset:\n # HuggingFace dataset repo | s3:// | gs:// | path to local file or directory\n path: str | None\n # name of dataset split to load from\n split: str | None\n # The type of prompt to use for training. [alpaca, gpteacher, oasst, reflection]\n type: str | UserDefinedPrompterType | None\n # For UserDefinedPrompterType:\n # Custom user instruction prompt\n system_prompt: str | None\n # Use {system} as key to be replaced\n system_format: str | None\n field_system: str | None\n field_instruction: str | None\n field_input: str | None\n field_output: str | None\n\n # Customizable to be single line or multi-line. Use {instruction}/{input} as key to\n # be replaced. 'format' can include {input}\n format: str | None\n # 'no_input_format' cannot include {input}\n no_input_format: str | None\n input_transform: str | None\n # split dataset into N pieces (use with shards_idx)\n shards: int | None\n # the index of sharded dataset to use\n shards_idx: int | None\n # process dataset in N sequential chunks for memory efficiency (exclusive with\n # `shards`)\n preprocess_shards: int | None\n conversation: str | None\n\n # The name of the chat template to use for training, following values are supported:\n # tokenizer_default: Uses the chat template that is available in the\n # tokenizer_config.json. If the chat template is not available in the tokenizer, it\n # will raise an error. This is the default.\n # alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates\n # are available in the axolotl codebase at src/axolotl/utils/chat_templates.py.\n # tokenizer_default_fallback_*: where * is the name of the chat template to fallback\n # to if the tokenizer does not have a chat template else default to tokenizer. E.g.\n # tokenizer_default_fallback_chatml. jinja: Uses a custom jinja template for the chat\n # template. The custom jinja template should be provided in the chat_template_jinja\n # field.\n chat_template: ChatTemplate | str | None\n # Custom jinja chat template or path to jinja file. Used only if `chat_template:\n # jinja` or empty.\n chat_template_jinja: str | None\n # path to source data files\n data_files: str | list[str] | None\n input_format: str | None\n # name of dataset configuration to load\n name: str | None\n # defines the datatype when path is a file\n ds_type: str | None\n # For `completion` datasets only, uses the provided field instead of `text` column\n field: str | None\n field_human: str | None\n field_model: str | None\n # Key containing the messages (default: \"messages\")\n field_messages: str | None\n # Key containing the tools (default: \"tools\"). Must be a list[dict] and follow [JSON\n # schema](https://json-schema.org/learn/getting-started-step-by-step).\n field_tools: str | None\n # Key containing the reasoning trace (default: \"reasoning_content\").\n field_thinking: str | None\n # The key the chat template expects that indicates the reasoning trace.\n template_thinking_key: str | None\n\n message_field_role: str | None\n\n message_field_content: str | None\n # Mapping of properties from the input dataset to the chat template. (default:\n # message_property_mappings={'role':'role', 'content':'content'}) If a property exists\n # in the template but not in this mapping, the system will attempt to load it directly\n # from the message using the property name as the key. Example: In the mapping below,\n # 'from' is loaded from input dataset and used as 'role', while 'value' is loaded and\n # used as 'content' in the chat template.\n message_property_mappings: dict[str, str] | None\n # The key in the message turn that indicates via boolean whether tokens of a turn\n # should be considered for training. Useful to selectively train on certain turns\n # besides the `roles_to_train`.\n message_field_training: str | None\n # The key in the message turn that contains the training details. Useful to\n # selectively train on certain tokens in a turn. The value of the key is a List[Dict]\n # containing `begin_offset` (start character index in content), `end_offset` (end\n # character index in content), and `train` (boolean whether to train).\n message_field_training_detail: str | None\n # (for Qwen3 template only) Whether to split the assistant content based on a\n # reasoning trace inside delimited tags\n split_thinking: bool | None\n logprobs_field: str | None\n temperature: float | None\n # Roles to train on. The tokens from these roles will be considered for the loss.\n roles_to_train: list[str] | None\n # Which EOS tokens to train on in the conversation. Possible values are: all: train on\n # all EOS tokens, turn (default): train on the EOS token at the end of each trainable\n # turn, last: train on the last EOS token in the conversation\n train_on_eos: Literal['all', 'turn', 'last'] | None\n # Roles mapping in the messages. The format is {target_role: [source_roles]}. All\n # source roles will be mapped to the target role. The default is: user: [\"human\",\n # \"user\"], assistant: [\"gpt\", \"assistant\"], system: [\"system\"], tool: [\"tool\"]\n roles: dict[str, list[str]] | None\n # Whether to drop the system turn from the dataset. Only works with chat_template.\n # This does not drop the default system message from chat_template if it exists. If\n # you wish to, we recommend using a custom jinja template with the default system\n # message removed or adding a system turn with empty content.\n drop_system_message: bool | None\n # Trust remote code for untrusted source\n trust_remote_code: bool | None = False\n # The specific revision of the dataset to use when loading from the Hugging Face Hub.\n # This can be a commit hash, tag, or branch name. If not specified, the latest version\n # will be used. This parameter is ignored for local datasets.\n revision: str | None\n\n# The maximum number of processes to use while preprocessing your input dataset. This\n# defaults to `os.cpu_count()` if not set. For Runpod VMs, it will default to number of\n# vCPUs via RUNPOD_CPU_COUNT.\ndataset_processes: int | None\n# The maximum number of processes to use while preprocessing your input dataset. This\n# defaults to `os.cpu_count()` if not set. For Runpod VMs, it will default to number of\n# vCPUs via RUNPOD_CPU_COUNT.\ndataset_num_proc: int | None\n\n# Deduplicates datasets and test_datasets with identical entries\ndataset_exact_deduplication: bool | None\n# Keep dataset in memory while preprocessing. Only needed if cached dataset is taking\n# too much storage\ndataset_keep_in_memory: bool | None\ndataloader_pin_memory: bool | None\ndataloader_num_workers: int | None\ndataloader_prefetch_factor: int | None\ndataloader_drop_last: bool | None\n\naccelerator_config: dict[str, Any] | None\n\nremove_unused_columns: bool | None\n\n# Push prepared dataset to hub - repo_org/repo_name\npush_dataset_to_hub: str | None\n# Whether to use hf `use_auth_token` for loading datasets. Useful for fetching private\n# datasets. Required to be true when used in combination with `push_dataset_to_hub`\nhf_use_auth_token: bool | None\n\ndevice: Any | None\n# Passed through to transformers when loading the model when launched without\n# accelerate. Use `sequential` when training w/ model parallelism to limit memory\ndevice_map: Any | None\nworld_size: int | None\n# Don't mess with this, it's here for accelerate and torchrun\nlocal_rank: int | None\nddp: bool | None\n\n# Seed for reproducibility\nseed: int | None\n# Advanced DDP Arguments - timeout\nddp_timeout: int | None\n# Advanced DDP Arguments - bucket cap in MB\nddp_bucket_cap_mb: int | None\n# Advanced DDP Arguments - broadcast buffers\nddp_broadcast_buffers: bool | None\nddp_find_unused_parameters: bool | None\n\n# Whether to run causal language model evaluation for metrics in\n# `eval_causal_lm_metrics`\ndo_causal_lm_eval: bool | None\n# HF evaluate metrics used during evaluation. Default is ['sacrebleu', 'comet', 'ter',\n# 'chrf', 'perplexity']\neval_causal_lm_metrics: list[str] | None\ndo_bench_eval: bool | None\nbench_dataset: str | None\nbench_split: str | None\nmetric_for_best_model: str | None\ngreater_is_better: bool | None\n\n# High loss value, indicating the learning has broken down (a good estimate is ~2 times\n# the loss at the start of training)\nloss_watchdog_threshold: float | None\n# Number of high-loss steps in a row before the trainer aborts (default: 3)\nloss_watchdog_patience: int | None\n\n# Run garbage collection every `gc_steps` steps. -1 will run on epoch end and before\n# evaluations. Default is 0 (disabled).\ngc_steps: int | None\n\n# Use CUDA bf16. bool or 'full' for `bf16_full_eval`, or 'auto' for automatic detection.\n# require >=ampere\nbf16: Literal['auto'] | bool | None = auto\n# Use CUDA fp16\nfp16: bool | None\n# Enable FP8 mixed precision training using TorchAO. Best used in combination with\n# torch.compile.\nfp8: bool | None\n# Enable FSDP float8 all-gather optimization for FP8 training. Can improve training\n# speed by 10-15% when FSDP is enabled.\nfp8_enable_fsdp_float8_all_gather: bool | None\n# No AMP (automatic mixed precision) - require >=ampere\nbfloat16: bool | None\n# No AMP (automatic mixed precision)\nfloat16: bool | None\n# Use CUDA tf32 - require >=ampere\ntf32: bool | None\nfloat32: bool | None\n\n# Whether to use gradient checkpointing. Available options are: true, false, 'offload',\n# 'offload_disk'.\n# https://huggingface.co/docs/transformers/v4.18.0/en/performance#gradient-checkpointing\ngradient_checkpointing: Literal['offload', 'offload_disk'] | bool | None = False\n# Additional kwargs to pass to the trainer for gradient checkpointing\ngradient_checkpointing_kwargs: dict[str, Any] | None\n# Whether to offload activations. Available options are: true, false, 'legacy', 'disk'.\nactivation_offloading: Literal['legacy', 'disk'] | bool | None = False\n\n# List of regex patterns for parameter names to keep unfrozen. All other parameters will\n# be frozen via requires_grad=False. Note: range-based patterns (e.g.\n# embed_tokens.weight$[:32000]) use gradient zeroing rather than a true freeze, so\n# weight decay will still apply to the frozen portion and optimizer states are allocated\n# for the full parameter.\nunfrozen_parameters: list[str] | None\n\n# The maximum length of an input to train with, this should typically be less than 2048\n# as most models have a token/context limit of 2048\nsequence_len: int = 512\n# What to do when a tokenized row exceeds sequence_len. 'drop' removes the row;\n# 'truncate' slices tensors to sequence_len; 'raise' raises a ValueError. Defaults to\n# 'drop' for backward compatibility.\nexcess_length_strategy: Literal['drop', 'truncate', 'raise'] | None\n# The maximum length of an input for evaluation. If not specified, defaults to\n# sequence_len\neval_sequence_len: int | None\nmin_sample_len: int | None\n# maximum prompt length for RL training\nmax_prompt_len: int | None\n# Use efficient multi-packing with block diagonal attention and per sequence\n# position_ids. Recommend set to 'true'\nsample_packing: bool | None\n# The number of samples packed at a time. Increasing the following values helps with\n# packing, but usually only slightly (<%1.)\nsample_packing_group_size: int | None = 100000\n# The number of samples which can be packed into one sequence. Increase if using a large\n# sequence_len with many short samples.\nsample_packing_bin_size: int | None = 200\n# Whether to pack samples sequentially\nsample_packing_sequentially: bool | None\n# The multiprocessing start method to use for packing. Should be 'fork', 'spawn' or\n# 'forkserver'\nsample_packing_mp_start_method: str | None\n# Set to 'false' if getting errors during eval with sample_packing on\neval_sample_packing: bool | None\n# Pad inputs so each step uses constant sized buffers. This will reduce memory\n# fragmentation and may prevent OOMs, by re-using memory more efficiently. Defaults to\n# True if `sample_packing` enabled\npad_to_sequence_len: bool | None\n# Whether to use sequential sampling for curriculum learning\ncurriculum_sampling: bool | None\nmultipack_real_batches: bool | None\n\n# Use batch flattening for speedups when not using sample_packing\nbatch_flattening: Literal['auto'] | bool | None\n\nuse_pose: bool | None\npose_split_on_token_ids: list[int] | None\npose_max_context_len: int | None\npose_num_chunks: int | None\n\npretrain_multipack_buffer_size: int | None\n# whether to prevent cross attention for packed sequences during pretraining\npretrain_multipack_attn: bool | None = True\n# whether to concatenate samples during pretraining\npretraining_sample_concatenation: bool | None\n\n# Use streaming mode for loading datasets\nstreaming: bool | None\n# Buffer size for multipack streaming datasets\nstreaming_multipack_buffer_size: int | None = 10000\n\n# Whether to use xformers attention patch https://github.com/facebookresearch/xformers\nxformers_attention: bool | None\n# Whether to use scaled-dot-product attention https://pytorch.org/docs/stable/generated/\n# torch.nn.functional.scaled_dot_product_attention.html\nsdp_attention: bool | None\n# Shifted-sparse attention (only llama) - https://arxiv.org/pdf/2309.12307.pdf\ns2_attention: bool | None\nflex_attention: bool | None\nflex_attn_compile_kwargs: dict[str, Any] | None\n# Whether to use flash attention patch https://github.com/Dao-AILab/flash-attention\nflash_attention: bool | None\n# Whether to use flash-attention cross entropy implementation - advanced use only\nflash_attn_cross_entropy: bool | None\n# Whether to use flash-attention rms norm implementation - advanced use only\nflash_attn_rms_norm: bool | None\n# Whether to fuse part of the MLP into a single operation\nflash_attn_fuse_mlp: bool | None\n# Whether to use bettertransformers\nflash_optimum: bool | None\n# Whether to use SageAttention https://github.com/thu-ml/SageAttention\nsage_attention: bool | None\n\neager_attention: bool | None\n\n# Specify a custom attention implementation, used mostly for kernels.\nattn_implementation: str | None\n\n# Which experts implementation to use for MoE models,\nexperts_implementation: str | None\n\n# Quantize MoE expert weights on load to reduce VRAM. Requires adapter (lora/qlora) with\n# load_in_4bit or load_in_8bit. Requires CUDA (not compatible with ROCm or other\n# backends). Note: total parameter count may be reported incorrectly when enabled\n# (trainable param count is correct).\nquantize_moe_experts: bool = False\n\n# Whether to use Scaled Softmax (SSMax) attention. Ref: https://arxiv.org/abs/2501.19399\nscaling_softmax: bool | None\n# Scaling factor for SSMax attention. Default is 0.43\nscaling_softmax_factor: float | None\n# Bias for SSMax attention. Default is 0.0. Note: The paper recommends bias=0 for better\n# length generalization.\nscaling_softmax_bias: float | None\n\nunsloth_cross_entropy_loss: bool | None\nunsloth_lora_mlp: bool | None\nunsloth_lora_qkv: bool | None\nunsloth_lora_o: bool | None\nunsloth_rms_norm: bool | None\nunsloth_rope: bool | None\n\n# Apply custom LoRA autograd functions and activation function Triton kernels for speed\n# and memory savings. See: https://docs.axolotl.ai/docs/lora_optims.html\nlora_mlp_kernel: bool | None\n# Apply custom LoRA autograd functions and activation function Triton kernels for speed\n# and memory savings. See: https://docs.axolotl.ai/docs/lora_optims.html\nlora_qkv_kernel: bool | None\n# Apply custom LoRA autograd functions and activation function Triton kernels for speed\n# and memory savings. See: https://docs.axolotl.ai/docs/lora_optims.html\nlora_o_kernel: bool | None\n\n# Whether to use chunked cross entropy loss for memory efficiency\nchunked_cross_entropy: bool | None\n# Number of chunks to use for chunked cross entropy loss\nchunked_cross_entropy_num_chunks: int | None\n# Enable Entropy-Aware Focal Training loss (EAFT)\nuse_eaft: bool | None\n# Exponent for entropy weighting in EAFT (default: 1.0)\neaft_alpha: float | None = 1.0\n# Number of top logits for entropy approximation (default: 20)\neaft_k: int | None = 20\n\n# Whether to use ALST tiled mlp for memory efficient long context\ntiled_mlp: bool | None\n\n# Number of shards to use for ALST tiled mlp. If unset, it will be set based on\n# seqlen/hidden_size\ntiled_mlp_num_shards: int | None\n\n# Whether to use original mlp for ALST tiled mlp. Otherwise uses a generic MLP based on\n# llama.\ntiled_mlp_use_original_mlp: bool | None = True\n\nllama4_linearized_experts: bool | None\n\n# Deepspeed config path. e.g., deepspeed_configs/zero3.json\ndeepspeed: str | dict[str, Any] | None\n# Whether to use deepcompile for faster training with deepspeed\ndeepcompile: bool | None\n# FSDP configuration\nfsdp: list[str] | None\n\n# FSDP configuration options\nfsdp_config: FSDPConfig | None\n # For FSDPConfig:\n # FSDP version\n fsdp_version: int | None\n # Enable activation checkpointing to reduce memory usage during forward passes\n activation_checkpointing: bool | None\n # Offload parameters to CPU to reduce GPU memory usage\n offload_params: bool | None\n # Synchronize module states across all processes\n sync_module_states: bool | None\n # Enable CPU RAM efficient loading to reduce memory usage during model loading\n cpu_ram_efficient_loading: bool | None\n # Disabling this enables swap memory usage for resource-constrained setups when\n # offload_params is enabled.\n cpu_offload_pin_memory: bool | None\n # Use original parameters instead of flattened parameters\n use_orig_params: bool | None\n\n # Type of state dict to use for saving/loading checkpoints\n state_dict_type: Literal['FULL_STATE_DICT', 'LOCAL_STATE_DICT', 'SHARDED_STATE_DICT'] | None\n # Final state dict type to use after training completion\n final_state_dict_type: Literal['FULL_STATE_DICT', 'LOCAL_STATE_DICT', 'SHARDED_STATE_DICT'] | None\n\n # Policy for automatically wrapping modules with FSDP\n auto_wrap_policy: Literal['TRANSFORMER_BASED_WRAP', 'SIZE_BASED_WRAP'] | None\n # Class name of transformer layers to wrap (e.g., 'LlamaDecoderLayer')\n transformer_layer_cls_to_wrap: str | None\n\n # Reshard parameters after forward pass to save memory\n reshard_after_forward: bool | None\n # Mixed precision policy for FSDP (e.g., 'fp16', 'bf16')\n mixed_precision_policy: str | None\n\n# FSDP version\nfsdp_version: int | None\nfsdp_final_state_dict_type: Literal['FULL_STATE_DICT', 'LOCAL_STATE_DICT', 'SHARDED_STATE_DICT'] | None\n\n# How much of the dataset to set aside as evaluation. 1 = 100%, 0.50 = 50%, etc. 0 for\n# no eval.\nval_set_size: float | None = 0.0\n\n# Number of devices to shard across. If not set, will use all available devices.\ndp_shard_size: int | None\n# Number of devices to replicate across.\ndp_replicate_size: int | None\n# Deprecated: use `context_parallel_size` instead\nsequence_parallel_degree: int | None\n# Set to a divisor of the number of GPUs available to split sequences into chunks of\n# equal size. Use in long context training to prevent OOM when sequences cannot fit into\n# a single GPU's VRAM. E.g., if 4 GPUs are available, set this value to 2 to split each\n# sequence into two equal-sized subsequences, or set to 4 to split into four equal-sized\n# subsequences. See https://docs.axolotl.ai/docs/sequence_parallelism.html for more\n# details.\ncontext_parallel_size: int | None\n# Optional; strides across the key dimension. Larger values use more memory but should\n# make training faster. Must evenly divide the number of KV heads in your model.\nheads_k_stride: int | None\n# One of 'varlen_llama3', 'batch_ring', 'batch_zigzag', 'batch_stripe'. Defaults to\n# 'varlen_llama3' in the sample packing case, and 'batch_ring' in the non-sample packing\n# case.\nring_attn_func: RingAttnFunc | None\n# Number of tensor parallel processes in TP group. Only supported with DeepSpeed AutoTP.\ntensor_parallel_size: int | None\n\n# Add or change special tokens. If you add tokens here, you don't need to add them to\n# the `tokens` list.\nspecial_tokens: SpecialTokensConfig | None\n # For SpecialTokensConfig:\n bos_token: str | None\n eos_token: str | None\n pad_token: str | None\n unk_token: str | None\n additional_special_tokens: list[str] | None\n\n# Add extra tokens to the tokenizer\ntokens: list[str] | None\n# Mapping token_id to new_token_string to override reserved added_tokens in the\n# tokenizer. Only works for tokens that are not part of the base vocab (aka are\n# added_tokens). Can be checked if they exist in tokenizer.json added_tokens.\nadded_tokens_overrides: dict[int, str] | None\n\n# Whether to use torch.compile and which backend to use. setting to `auto` will enable\n# torch compile when torch>=2.6.0\ntorch_compile: Literal['auto'] | bool | None\n# Backend to use for torch.compile\ntorch_compile_backend: str | None\ntorch_compile_mode: Literal['default', 'reduce-overhead', 'max-autotune'] | None\n\n# Maximum number of iterations to train for. It precedes num_epochs which means that if\n# both are set, num_epochs will not be guaranteed. e.g., when 1 epoch is 1000 steps =>\n# `num_epochs: 2` and `max_steps: 100` will train for 100 steps\nmax_steps: int | None\n# Number of warmup steps. Cannot use with warmup_ratio\nwarmup_steps: int | None\n# Warmup ratio. Cannot use with warmup_steps\nwarmup_ratio: float | None\n# Leave empty to eval at each epoch, integer for every N steps. float for fraction of\n# total steps\neval_steps: int | float | None\n# Number of times per epoch to run evals, mutually exclusive with eval_steps\nevals_per_epoch: int | None\n# Set to `no` to skip evaluation, `epoch` at end of each epoch, leave empty to infer\n# from `eval_steps`\neval_strategy: str | None\n\n# Leave empty to save at each epoch, integer for every N steps. float for fraction of\n# total steps\nsave_steps: int | float | None\n# Number of times per epoch to save a checkpoint, mutually exclusive with save_steps\nsaves_per_epoch: int | None\n# Set to `no` to skip checkpoint saves, `epoch` at end of each epoch, `best` when better\n# result is achieved, leave empty to infer from `save_steps`\nsave_strategy: str | None\n# Checkpoints saved at a time\nsave_total_limit: int | None\n# Whether to checkpoint a model after the first step of training. Defaults to False.\nsave_first_step: bool | None\n\n# Logging frequency\nlogging_steps: int | None\n# Stop training after this many evaluation losses have increased in a row. https://huggi\n# ngface.co/transformers/v4.2.2/_modules/transformers/trainer_callback.html#EarlyStoppin\n# gCallback\nearly_stopping_patience: int | None\nload_best_model_at_end: bool | None = False\n# Save only the model weights, skipping the optimizer. Using this means you can't resume\n# from checkpoints.\nsave_only_model: bool | None = False\n# Use tensorboard for logging\nuse_tensorboard: bool | None\n# Enable the pytorch profiler to capture the first N steps of training to the\n# output_dir. see https://pytorch.org/blog/understanding-gpu-memory-1/ for more\n# information. Snapshots can be visualized @ https://pytorch.org/memory_viz\nprofiler_steps: int | None\n# Which step to start the profiler at. Useful for only capturing a few steps mid-run.\nprofiler_steps_start: int | None = 0\n# bool of whether to report tokens per second at the end of training. This is not\n# supported with pre-training datasets.\ninclude_tokens_per_second: bool | None\n# bool of whether to report tokens per second per-gpu during training by measuring\n# throughput of non-padding tokens.\ninclude_tkps: bool | None = True\n# NEFT https://arxiv.org/abs/2310.05914, set this to a number (paper default is 5) to\n# add noise to embeddings. Currently only supported on Llama and Mistral\nneftune_noise_alpha: float | None\n\n# Parameter controlling the relative ratio loss weight in the ORPO loss. Passed to\n# `beta` in `ORPOConfig` due to trl mapping.\norpo_alpha: float | None\n# Weighting of NLL term in loss from RPO paper\nrpo_alpha: float | None\n# Target reward margin for the SimPO loss\nsimpo_gamma: float | None\n# Weight of the BC regularizer\ncpo_alpha: float | None\n\n# Factor for desirable loss term in KTO loss\nkto_desirable_weight: float | None\n# Factor for undesirable loss term in KTO loss\nkto_undesirable_weight: float | None\n# The beta parameter for the RL training\nrl_beta: float | None\n\n# Defines the max memory usage per gpu on the system. Passed through to transformers\n# when loading the model.\nmax_memory: dict[int | Literal['cpu', 'disk'], int | str] | None\n# Limit the memory for all available GPUs to this amount (if an integer, expressed in\n# gigabytes); default: unset\ngpu_memory_limit: int | str | None\n# Whether to use low_cpu_mem_usage\nlow_cpu_mem_usage: bool | None\n\n# The name of the chat template to use for training, following values are supported:\n# tokenizer_default: Uses the chat template that is available in the\n# tokenizer_config.json. If the chat template is not available in the tokenizer, it will\n# raise an error. This is the default value.\n# alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates\n# are available in the axolotl codebase at src/axolotl/utils/chat_templates.py.\n# tokenizer_default_fallback_*: where * is the name of the chat template to fallback to.\n# E.g. tokenizer_default_fallback_chatml. This is useful when the chat template is not\n# available in the tokenizer. jinja: Uses a custom jinja template for the chat template.\n# The custom jinja template should be provided in the chat_template_jinja field. The\n# selected chat template will be saved to the tokenizer_config.json for easier\n# inferencing\nchat_template: ChatTemplate | Annotated[str, StringConstraints(pattern='^tokenizer_default_fallback_')] | None\n# Custom jinja template or path to jinja file for chat template. This will be only used\n# if chat_template is set to `jinja` or `null` (in which case chat_template is\n# automatically set to `jinja`). Default is null.\nchat_template_jinja: str | None\n# Additional kwargs to pass to the chat template. This is useful for customizing the\n# chat template. For example, you can pass `thinking=False` to add a generation prompt\n# to the chat template.\nchat_template_kwargs: dict[str, Any] | None\n# Custom EOT (End-of-Turn) tokens to mask/unmask during training. These tokens mark the\n# boundaries between conversation turns. For example: ['/INST', '</s>',\n# '[/SYSTEM_PROMPT]']. If not specified, defaults to just the model's eos_token. This is\n# useful for templates that use multiple delimiter tokens.\neot_tokens: list[str] | None\n# Changes the default system message. Currently only supports chatml.\ndefault_system_message: str | None\n\n# Token index or indices to adjust embedding weights to the mean of the other tokens.\n# This is useful when the model has untrained embeddings.\nfix_untrained_tokens: int | list[int] | None\n\nis_preprocess: bool | None\npreprocess_iterable: bool | None\n\n# Total number of tokens - internal use\ntotal_num_tokens: int | None\ntotal_supervised_tokens: int | None\n# You can set these packing optimizations AFTER starting a training at least once. The\n# trainer will provide recommended values for these values.\nsample_packing_eff_est: float | None\naxolotl_config_path: str | None\n\n# Internal use only - Used to identify which the model is based on\nis_falcon_derived_model: bool | None\n# Internal use only - Used to identify which the model is based on\nis_llama_derived_model: bool | None\n# Internal use only - Used to identify which the model is based on. Please note that if\n# you set this to true, `padding_side` will be set to 'left' by default\nis_mistral_derived_model: bool | None\n# Internal use only - Used to identify which the model is based on\nis_qwen_derived_model: bool | None\n\n# Add plugins to extend the pipeline. See `src/axolotl/integrations` for the available\n# plugins or doc below for more details.\n# https://docs.axolotl.ai/docs/custom_integrations.html\nplugins: list[str] | None\n# Enable sample generation during training for monitoring\ngenerate_samples: bool | None = False\n# Number of samples to generate at each interval\nnum_generation_samples: int | None = 3\n# Maximum new tokens to generate per sample\ngeneration_max_new_tokens: int | None = 50\n# Temperature for sample generation (0.0 = greedy)\ngeneration_temperature: float | None = 0.7\n# Nucleus sampling parameter for generation\ngeneration_top_p: float | None\n# Top-k sampling parameter for generation\ngeneration_top_k: int | None\n# Ratio of input to use as prompt (0.0-1.0)\ngeneration_prompt_ratio: float | None = 0.5\n# Whether to use sampling (vs greedy decoding)\ngeneration_do_sample: bool | None = True\n\n# This is the huggingface model that contains *.pt, *.safetensors, or *.bin files. This\n# can also be a relative path to a model on disk\nbase_model: str (required)\n# If the base_model repo on hf hub doesn't include configuration .json files, You can\n# set that here, or leave this empty to default to base_model\nbase_model_config: str | None\n# transformers config class (e.g., 'LlamaConfig', 'MistralConfig'). Defaults to\n# AutoConfig.\ncls_model_config: str | None\n# Optional tokenizer configuration path in case you want to use a different tokenizer\n# than the one defined in the base model\ntokenizer_config: str | None\n# use_fast option for tokenizer loading from_pretrained, default to True\ntokenizer_use_fast: bool | None\n# Whether to use the legacy tokenizer setting, defaults to True\ntokenizer_legacy: bool | None\n# Whether to use mistral-common tokenizer. If set to True, it will use the mistral-\n# common tokenizer.\ntokenizer_use_mistral_common: bool | None\n# Corresponding tokenizer for the model AutoTokenizer is a good choice\ntokenizer_type: str | None\n# transformers processor class\nprocessor_type: str | None\n# Whether to save jinja files for tokenizer, transformers default is True\ntokenizer_save_jinja_files: bool | None = True\n# Trust remote code for untrusted source\ntrust_remote_code: bool | None\n\n# Don't move the model to the device before sharding. Set to `false` to revert to legacy\n# behavior.\nexperimental_skip_move_to_device: bool | None = True\n\n# Use custom kernels, e.g. MegaBlocks.\nuse_kernels: bool | None\n\n# Model loading quantization config\nmodel_quantization_config: Literal['Mxfp4Config'] | None\n# kwargs for model quantization config\nmodel_quantization_config_kwargs: dict[str, Any] | None\n\n# Where to save the full-finetuned model to\noutput_dir: str = ./model-out\n# push checkpoints to hub\nhub_model_id: str | None\n# how to push checkpoints to hub\nhub_strategy: str | None\n# branch/revision to push to on hub (default: main)\nhub_revision: str | None\n# Whether to save the model using safetensors format. Defaults to True.\nsave_safetensors: bool | None = True\n\n# This will attempt to quantize the model down to 8 bits and use adam 8 bit optimizer\nload_in_8bit: bool | None = False\n# Use bitsandbytes 4 bit\nload_in_4bit: bool | None = False\n\n# If you want to use 'lora', 'qlora', or 'llama-adapter', or leave blank to train all\n# parameters in original model\nadapter: Literal['lora', 'qlora', 'llama-adapter'] | None\n# If you already have a lora model trained that you want to load, put that here. This\n# means after training, if you want to test the model, you should set this to the value\n# of `output_dir`. Note that if you merge an adapter to the base model, a new\n# subdirectory `merged` will be created under the `output_dir`.\nlora_model_dir: str | None\nlora_r: int | None\nlora_alpha: int | None\nlora_fan_in_fan_out: bool | None\nlora_target_modules: str | list[str] | None\nlora_target_parameters: str | list[str] | None\n# If true, will target all linear modules\nlora_target_linear: bool | None\n# If you added new tokens to the tokenizer, you may need to save some LoRA modules\n# because they need to know the new tokens. For LLaMA and Mistral, you need to save\n# `embed_tokens` and `lm_head`. It may vary for other models. `embed_tokens` converts\n# tokens to embeddings, and `lm_head` converts embeddings to token probabilities.\nlora_modules_to_save: list[str] | None\nlora_dropout: float | None = 0.0\n# The layer indices to transform, otherwise, apply to all layers\npeft_layers_to_transform: list[int] | None\npeft_layers_pattern: list[str] | None\n\npeft: PeftConfig | None\n # For PeftConfig:\n # Configuration options for loftq initialization for LoRA\n loftq_config: LoftQConfig | None\n # For LoftQConfig:\n # typically 4 bits\n loftq_bits: int = 4\n\n# Whether to use DoRA.\npeft_use_dora: bool | None\n# Whether to use RSLoRA.\npeft_use_rslora: bool | None\n# List of layer indices to replicate.\npeft_layer_replication: list[tuple[int, int]] | None\n# How to initialize LoRA weights. Default to True which is MS original implementation.\npeft_init_lora_weights: bool | str | None\n# A list of token indices to fine-tune on the `embed_tokens` layer. Otherwise, a dict\n# mapping an embedding layer name to its trainable token indices. See\n# https://huggingface.co/docs/peft/v0.17.0/en/developer_guides/lora#efficiently-train-\n# tokens-alongside-lora\npeft_trainable_token_indices: list[int] | dict[str, list[int]] | None\n# Whether to tie adapter weights for tied model weights. See\n# https://github.com/huggingface/peft/issues/2864\npeft_ensure_weight_tying: bool | None\n# Whether to upcast the LoRA adapter to fp32. This is enabled by default in PEFT.\npeft_autocast_adapter_dtype: bool | None\n\n# load qlora model in sharded format for FSDP using answer.ai technique.\nqlora_sharded_model_loading: bool | None = False\n# Do the LoRA/PEFT loading on CPU -- this is required if the base model is so large it\n# takes up most or all of the available GPU VRAM, e.g. during a model and LoRA merge\nlora_on_cpu: bool | None\n# Whether you are training a 4-bit GPTQ quantized model\ngptq: bool | None\n# optional overrides to the bnb 4bit quantization configuration\nbnb_config_kwargs: dict[str, Any] | None\n\n# loraplus learning rate ratio lr_B / lr_A. Recommended value is 2^4.\nloraplus_lr_ratio: float | None\n# loraplus learning rate for lora embedding layers. Default value is 1e-6.\nloraplus_lr_embedding: float | None = 1e-06\n\nmerge_lora: bool | None\n\n# Whether to use ReLoRA. Use with jagged_restart_*steps options.\nrelora: bool | None\n# threshold for optimizer magnitude when pruning\nrelora_prune_ratio: float | None\n# True to perform lora weight merges on cpu during restarts, for modest gpu memory\n# savings\nrelora_cpu_offload: bool | None\n\n# how often to reset for jagged restarts\njagged_restart_steps: int | None\n# how many warmup steps to take after reset for jagged restarts\njagged_restart_warmup_steps: int | None\n# how many anneal steps to take before reset for jagged restarts\njagged_restart_anneal_steps: int | None\n\n# If greater than 1, backpropagation will be skipped and the gradients will be\n# accumulated for the given number of steps.\ngradient_accumulation_steps: int | None = 1\n# The number of samples to include in each batch. This is the number of samples sent to\n# each GPU. Batch size per gpu = micro_batch_size * gradient_accumulation_steps\nmicro_batch_size: int | None = 1\n# Total batch size, we do not recommended setting this manually\nbatch_size: int | None\n# per gpu micro batch size for evals, defaults to value of micro_batch_size\neval_batch_size: int | None\n\n# whether to find batch size that fits in memory. Passed to underlying transformers\n# Trainer\nauto_find_batch_size: bool | None\n\n# Whether to mask out or include the human's prompt from the training labels\ntrain_on_inputs: bool | None = False\n# Group similarly sized data to minimize padding. May be slower to start, as it must\n# download and sort the entire dataset. Note that training loss may have an oscillating\n# pattern with this enabled.\ngroup_by_length: bool | None\n\nlearning_rate: str | float (required)\nembedding_lr: float | None\nembedding_lr_scale: float | None\n# Specify weight decay\nweight_decay: float | None = 0.0\n# Specify optimizer\noptimizer: OptimizerNames | CustomSupportedOptimizers | None = OptimizerNames.ADAMW_TORCH_FUSED\n# Dictionary of arguments to pass to the optimizer\noptim_args: str | dict[str, Any] | None\n# The target modules to optimize, i.e. the module names that you would like to train,\n# right now this is used only for GaLore algorithm\noptim_target_modules: list[str] | Literal['all_linear'] | None\n# Path to torch distx for optim 'adamw_anyprecision'\ntorchdistx_path: str | None\nlr_scheduler: SchedulerType | Literal['one_cycle'] | Literal['rex'] | None = SchedulerType.COSINE\n# Specify a scheduler and kwargs to use with the optimizer\nlr_scheduler_kwargs: dict[str, Any] | None\nlr_quadratic_warmup: bool | None\n# decay lr to some percentage of the peak lr, e.g. cosine_min_lr_ratio=0.1 for 10% of\n# peak lr\ncosine_min_lr_ratio: float | None\n# freeze lr at some percentage of the step, e.g. cosine_constant_lr_ratio=0.8 means\n# start cosine_min_lr at 80% of training step\ncosine_constant_lr_ratio: float | None\n# Learning rate div factor\nlr_div_factor: float | None\n\nlr_groups: list[LrGroup] | None\n # For LrGroup:\n name: str (required)\n modules: list[str] (required)\n lr: float (required)\n\n# adamw hyperparams\nadam_epsilon: float | None\n# only used for CAME Optimizer\nadam_epsilon2: float | None\n# adamw hyperparams\nadam_beta1: float | None\n# adamw hyperparams\nadam_beta2: float | None\n# only used for CAME Optimizer\nadam_beta3: float | None\n\n# Dion Optimizer learning rate\ndion_lr: float | None\n# Dion Optimizer momentum\ndion_momentum: float | None\n# Dion Optimizer: r/d fraction for low-rank approximation. Used to compute the low-rank\n# dimension.\ndion_rank_fraction: float | None = 1.0\n# Dion Optimizer: Round up the low-rank dimension to a multiple of this number. This may\n# be useful to ensure even sharding.\ndion_rank_multiple_of: int | None = 1\n\n# Gradient clipping max norm\nmax_grad_norm: float | None\nnum_epochs: float = 1.0\n\nuse_wandb: bool | None\n# Set the name of your wandb run\nwandb_name: str | None\n# Set the ID of your wandb run\nwandb_run_id: str | None\n# \"offline\" to save run metadata locally and not sync to the server, \"disabled\" to turn\n# off wandb\nwandb_mode: str | None\n# Your wandb project name\nwandb_project: str | None\n# A wandb Team name if using a Team\nwandb_entity: str | None\nwandb_watch: str | None\n# \"checkpoint\" to log model to wandb Artifacts every `save_steps` or \"end\" to log only\n# at the end of training\nwandb_log_model: str | None\n\nuse_mlflow: bool | None\n# URI to mlflow\nmlflow_tracking_uri: str | None\n# Your experiment name\nmlflow_experiment_name: str | None\n# Your run name\nmlflow_run_name: str | None\n# set to true to copy each saved checkpoint on each save to mlflow artifact registry\nhf_mlflow_log_artifacts: bool | None\n\n# Enable or disable Comet integration.\nuse_comet: bool | None\n# API key for Comet. Recommended to set via `comet login`.\ncomet_api_key: str | None\n# Workspace name in Comet. Defaults to the user's default workspace.\ncomet_workspace: str | None\n# Project name in Comet. Defaults to Uncategorized.\ncomet_project_name: str | None\n# Identifier for the experiment. Used to append data to an existing experiment or\n# control the key of new experiments. Default to a random key.\ncomet_experiment_key: str | None\n# Create a new experiment (\"create\") or log to an existing one (\"get\"). Default\n# (\"get_or_create\") auto-selects based on configuration.\ncomet_mode: str | None\n# Set to True to log data to Comet server, or False for offline storage. Default is\n# True.\ncomet_online: bool | None\n# Dictionary for additional configuration settings, see the doc for more details.\ncomet_experiment_config: dict[str, Any] | None\n\nuse_trackio: bool | None\n# Your trackio project name\ntrackio_project_name: str | None\n# Set the name of your trackio run\ntrackio_run_name: str | None\n# Hugging Face Space ID to sync dashboard to (optional, runs locally if not provided)\ntrackio_space_id: str | None\n\n# Enable OpenTelemetry metrics collection and Prometheus export\nuse_otel_metrics: bool | None = False\n# Host to bind the OpenTelemetry metrics server to\notel_metrics_host: str | None = localhost\n# Port for the Prometheus metrics HTTP server\notel_metrics_port: int | None = 8000\n\n# the number of activate layers in LISA\nlisa_n_layers: int | None\n# how often to switch layers in LISA\nlisa_step_interval: int | None\n# path under the model to access the layers\nlisa_layers_attribute: str | None = model.layers\n\ngradio_title: str | None\ngradio_share: bool | None\ngradio_server_name: str | None\ngradio_server_port: int | None\ngradio_max_new_tokens: int | None\ngradio_temperature: float | None\n\nuse_ray: bool = False\nray_run_name: str | None\nray_num_workers: int = 1\nresources_per_worker: dict\n\n# The size of the image to resize to. It can be an integer (resized into padded-square\n# image) or a tuple (width, height).If not provided, we will attempt to load from\n# preprocessor.size, otherwise, images won't be resized.\nimage_size: int | tuple[int, int] | None\n# The resampling algorithm to use for image resizing. Default is bilinear. Please refer\n# to PIL.Image.Resampling for more details.\nimage_resize_algorithm: Literal['bilinear', 'bicubic', 'lanczos'] | Resampling | None\n\n# optional overrides to the base model configuration\noverrides_of_model_config: dict[str, Any] | None\n# optional overrides the base model loading from_pretrained\noverrides_of_model_kwargs: dict[str, Any] | None\n# If you want to specify the type of model to load, AutoModelForCausalLM is a good\n# choice too\ntype_of_model: str | None\n# You can specify to choose a specific model revision from huggingface hub\nrevision_of_model: str | None\n\nmax_packed_sequence_len: int | None\nrope_scaling: Any | None\nnoisy_embedding_alpha: float | None\ndpo_beta: float | None\nevaluation_strategy: str | None\neval_table_size: int | None\neval_max_new_tokens: int | None", + "text": "# Allow overwrite yml config using from cli\nstrict: bool | None = False\n# Resume from a specific checkpoint dir\nresume_from_checkpoint: str | None\n# If resume_from_checkpoint isn't set and you simply want it to start where it left off.\n# Be careful with this being turned on between different models.\nauto_resume_from_checkpoints: bool | None\n# Resize the model embeddings when new tokens are added to multiples of 32. This is\n# reported to improve training speed on some models\nresize_token_embeddings_to_32x: bool | None\nmean_resizing_embeddings: bool | None = False\n\n# Whether to shrink the embeddings to len(tokenizer). By default, we won't shrink.\nshrink_embeddings: bool | None\n# Don't upcast the embeddings to float32 when using PEFT. Useful for low-VRAM GPUs\nembeddings_skip_upcast: bool | None\n# Reinitialize model weights randomly instead of loading pretrained weights\nreinit_weights: bool | None\n\n# module to custom trainer class to use for training\ntrainer_cls: str | None\n\n# Use RL training: 'dpo', 'ipo', 'kto', 'simpo', 'orpo', 'grpo'\nrl: RLType | None\n\ntrl: TRLConfig | None\n # For TRLConfig:\n # Beta parameter for the RL training. Same as `rl_beta`. Use\n beta: float | None\n # Maximum length of the completion for RL training.\n max_completion_length: int | None\n\n # Whether to use VLLM for RL training.\n use_vllm: bool = False\n # VLLM mode to use, one of 'server' or 'colocate'\n vllm_mode: Literal['server', 'colocate'] | None\n # Host of the vLLM server to connect to.\n vllm_server_host: str | None = 0.0.0.0\n # Port of the vLLM server to connect to.\n vllm_server_port: int | None = 8000\n # Total timeout (in seconds) to wait for the vLLM server to respond.\n vllm_server_timeout: int | None\n # Regex for vLLM guided decoding.\n vllm_guided_decoding_regex: str | None\n\n # List of reward functions to load. Paths must be importable from current dir.\n reward_funcs: list[str] | None\n # List of reward weights for the reward functions.\n reward_weights: list[float] | None\n # Number of generations to sample.\n num_generations: int | None\n # Whether to log completions.\n log_completions: bool | None = False\n # Number of completions to print when log_completions is True.\n num_completions_to_print: int | None\n # Controls whether importance sampling ratios are computed at the `'token'` or\n # `'sequence'` level. For GSPO, use `sequence`, default is None which corresponds to\n # the original GRPO paper.\n importance_sampling_level: Literal['sequence', 'token'] | None\n\n # Whether to sync the reference model.\n sync_ref_model: bool | None = False\n # Mixup alpha for the reference model.\n ref_model_mixup_alpha: float | None = 0.9\n # Sync steps for the reference model.\n ref_model_sync_steps: int | None = 64\n # Whether to scale rewards by their standard deviation.\n scale_rewards: bool = True\n\n # Sampling temperature for the GRPO policy.\n temperature: float | None\n # Top-p sampling probability for the generation policy.\n top_p: float | None\n # Top-k sampling for the generation policy.\n top_k: int | None\n # Minimum probability for the generation policy.\n min_p: float | None\n # Penalty for tokens that appear in prompt and generated text.\n repetition_penalty: float | None\n # Number of iterations per batch (μ) for GRPO.\n num_iterations: int | None\n # Epsilon value for clipping in the GRPO algorithm.\n epsilon: float | None\n # Upper-bound epsilon value for clipping in the GRPO algorithm.\n epsilon_high: float | None\n # Whether to use Liger loss for GRPO.\n use_liger_loss: bool | None\n # Loss formulation to use. Supported values: grpo, bnpo, dr_grpo.\n loss_type: str | None\n # Whether to exclude truncated completions from loss calculation.\n mask_truncated_completions: bool = False\n # Enable sleep mode for vLLM to offload VRAM when idle\n vllm_enable_sleep_mode: bool | None\n # Path to custom rollout function. Must be importable from current dir.\n rollout_func: str | None\n # Multi-objective reward aggregation strategy. 'sum_then_normalize' (GRPO default):\n # weights and sums rewards first, then normalizes. 'normalize_then_sum' (GDPO):\n # normalizes each reward independently, then sums.\n multi_objective_aggregation: Literal['sum_then_normalize', 'normalize_then_sum'] | None\n\nvllm: VllmConfig | None\n # For VllmConfig:\n # Device to use for VLLM\n device: str | None = auto\n # Tensor parallel size for VLLM\n tensor_parallel_size: int | None\n # Data parallel size for VLLM\n data_parallel_size: int | None\n # GPU memory utilization for VLLM\n gpu_memory_utilization: float | None = 0.9\n # Data type for VLLM\n dtype: str | None = auto\n # Maximum length of the model context for VLLM\n max_model_len: int | None\n # Enable prefix caching for VLLM\n enable_prefix_caching: bool | None\n # Host for the vLLM server to start on\n host: str | None = 0.0.0.0\n # Port of the vLLM server to start on\n port: int | None = 8000\n\n # Enable reasoning for VLLM\n enable_reasoning: bool | None\n # Reasoning parser for VLLM\n reasoning_parser: str | None\n\nqat: QATConfig | None\n # For QATConfig:\n # Fake quantization layout to use for activation quantization.\n activation_dtype: TorchAOQuantDType | None\n # Fake quantization layout to use for weight quantization.\n weight_dtype: TorchAOQuantDType = TorchAOQuantDType.int8\n # Quantize embedding\n quantize_embedding: bool | None = False\n # The number of elements in each group for per-group fake quantization\n group_size: int | None = 32\n # The number of steps to apply fake quantization after\n fake_quant_after_n_steps: int | None\n\nquantization: PTQConfig | None\n # For PTQConfig:\n # Fake quantization layout to use for weight quantization.\n weight_dtype: TorchAOQuantDType = TorchAOQuantDType.int8\n # Fake quantization layout to use for activation quantization.\n activation_dtype: TorchAOQuantDType | None\n # Whether to quantize the embedding layer.\n quantize_embedding: bool | None\n # The number of elements in each group for per-group fake quantization\n group_size: int | None = 32\n\n# Reward modelling: `True` or `False`\nreward_model: bool | None\n\n# Configuration for dynamic checkpointing (trigger by file or signal). Set 'enabled:\n# true' to activate this feature.\ndynamic_checkpoint: DynamicCheckpointConfig | None\n # For DynamicCheckpointConfig:\n # Enable dynamic checkpoint triggering during training. Create a file\n # 'axolotl_checkpoint.save' in the configured `output_dir` to trigger.\n enabled: bool = False\n # Check for trigger file every N steps (reduces I/O overhead). Default: 100\n check_interval: int = 10\n # Custom trigger filename (optional). If not specified, defaults to\n # 'axolotl_checkpoint.save'. Specify a filename (not a full path) to override the\n # default.\n trigger_file_path: str = \n\n# Process reward modelling: `True` or `False`\nprocess_reward_model: bool | None\n# Coefficient to incentivize the reward model to output mean-zero rewards (proposed by\n# https://huggingface.co/papers/2312.09244, Eq. 2). Recommended value: `0.01`.\ncenter_rewards_coefficient: float | None\nnum_labels: int | None\n\n# Whether to perform weighting in DPO trainer\ndpo_use_weighting: bool | None\ndpo_label_smoothing: float | None\ndpo_norm_loss: bool | None\n\n# Whether to use Liger kernel for DPO loss.\ndpo_use_liger_kernel: bool | None\n\ndpo_padding_free: bool | None\n\n# A list of one or more datasets to finetune the model with\ndatasets: Annotated[list[SFTDataset | DPODataset | KTODataset | StepwiseSupervisedDataset], MinLen(1)] | None\n # For SFTDataset:\n # HuggingFace dataset repo | s3:// | gs:// | path to local file or directory\n path: str | None\n # name of dataset split to load from\n split: str | None\n # The type of prompt to use for training. [alpaca, gpteacher, oasst, reflection]\n type: str | UserDefinedPrompterType | None\n # For UserDefinedPrompterType:\n # Custom user instruction prompt\n system_prompt: str | None\n # Use {system} as key to be replaced\n system_format: str | None\n field_system: str | None\n field_instruction: str | None\n field_input: str | None\n field_output: str | None\n\n # Customizable to be single line or multi-line. Use {instruction}/{input} as key to\n # be replaced. 'format' can include {input}\n format: str | None\n # 'no_input_format' cannot include {input}\n no_input_format: str | None\n input_transform: str | None\n # split dataset into N pieces (use with shards_idx)\n shards: int | None\n # the index of sharded dataset to use\n shards_idx: int | None\n # process dataset in N sequential chunks for memory efficiency (exclusive with\n # `shards`)\n preprocess_shards: int | None\n conversation: str | None\n\n # The name of the chat template to use for training, following values are supported:\n # tokenizer_default: Uses the chat template that is available in the\n # tokenizer_config.json. If the chat template is not available in the tokenizer, it\n # will raise an error. This is the default.\n # alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates\n # are available in the axolotl codebase at src/axolotl/utils/chat_templates.py.\n # tokenizer_default_fallback_*: where * is the name of the chat template to fallback\n # to if the tokenizer does not have a chat template else default to tokenizer. E.g.\n # tokenizer_default_fallback_chatml. jinja: Uses a custom jinja template for the chat\n # template. The custom jinja template should be provided in the chat_template_jinja\n # field.\n chat_template: ChatTemplate | str | None\n # Custom jinja chat template or path to jinja file. Used only if `chat_template:\n # jinja` or empty.\n chat_template_jinja: str | None\n # path to source data files\n data_files: str | list[str] | None\n input_format: str | None\n # name of dataset configuration to load\n name: str | None\n # defines the datatype when path is a file\n ds_type: str | None\n # For `completion` datasets only, uses the provided field instead of `text` column\n field: str | None\n field_human: str | None\n field_model: str | None\n # Key containing the messages (default: \"messages\")\n field_messages: str | None\n # Key containing the tools (default: \"tools\"). Must be a list[dict] and follow [JSON\n # schema](https://json-schema.org/learn/getting-started-step-by-step).\n field_tools: str | None\n # Key containing the reasoning trace (default: \"reasoning_content\").\n field_thinking: str | None\n # The key the chat template expects that indicates the reasoning trace.\n template_thinking_key: str | None\n\n message_field_role: str | None\n\n message_field_content: str | None\n # Mapping of properties from the input dataset to the chat template. (default:\n # message_property_mappings={'role':'role', 'content':'content'}) If a property exists\n # in the template but not in this mapping, the system will attempt to load it directly\n # from the message using the property name as the key. Example: In the mapping below,\n # 'from' is loaded from input dataset and used as 'role', while 'value' is loaded and\n # used as 'content' in the chat template.\n message_property_mappings: dict[str, str] | None\n # The key in the message turn that indicates via boolean whether tokens of a turn\n # should be considered for training. Useful to selectively train on certain turns\n # besides the `roles_to_train`.\n message_field_training: str | None\n # The key in the message turn that contains the training details. Useful to\n # selectively train on certain tokens in a turn. The value of the key is a List[Dict]\n # containing `begin_offset` (start character index in content), `end_offset` (end\n # character index in content), and `train` (boolean whether to train).\n message_field_training_detail: str | None\n # (for Qwen3 template only) Whether to split the assistant content based on a\n # reasoning trace inside delimited tags\n split_thinking: bool | None\n logprobs_field: str | None\n temperature: float | None\n # Roles to train on. The tokens from these roles will be considered for the loss.\n roles_to_train: list[str] | None\n # Which EOS tokens to train on in the conversation. Possible values are: all: train on\n # all EOS tokens, turn (default): train on the EOS token at the end of each trainable\n # turn, last: train on the last EOS token in the conversation\n train_on_eos: Literal['all', 'turn', 'last'] | None\n # Roles mapping in the messages. The format is {target_role: [source_roles]}. All\n # source roles will be mapped to the target role. The default is: user: [\"human\",\n # \"user\"], assistant: [\"gpt\", \"assistant\"], system: [\"system\"], tool: [\"tool\"]\n roles: dict[str, list[str]] | None\n # Whether to drop the system turn from the dataset. Only works with chat_template.\n # This does not drop the default system message from chat_template if it exists. If\n # you wish to, we recommend using a custom jinja template with the default system\n # message removed or adding a system turn with empty content.\n drop_system_message: bool | None\n # Trust remote code for untrusted source\n trust_remote_code: bool | None = False\n # The specific revision of the dataset to use when loading from the Hugging Face Hub.\n # This can be a commit hash, tag, or branch name. If not specified, the latest version\n # will be used. This parameter is ignored for local datasets.\n revision: str | None\n\n # For DPODataset:\n path: str | None\n split: str | None\n type: UserDefinedDPOType | str | None\n # For UserDefinedDPOType:\n field_system: str | None\n field_prompt: str | None\n field_chosen: str | None\n field_rejected: str | None\n prompt_format: str | None\n chosen_format: str | None\n rejected_format: str | None\n data_files: list[str] | None\n revision: str | None\n field_messages: str | None\n\n # For KTODataset:\n path: str | None\n split: str | None\n type: UserDefinedKTOType | str | None\n # For UserDefinedKTOType:\n field_system: str | None\n field_prompt: str | None\n field_completion: str | None\n field_label: bool | None\n prompt_format: str | None\n completion_format: str | None\n data_files: list[str] | None\n trust_remote_code: bool | None = False\n revision: str | None\n\n # For StepwiseSupervisedDataset:\n path: str | None\n split: str | None\n data_files: list[str] | None\n revision: str | None\n step_separator: str | None\n max_completion_length: int | None\n train_on_last_step_only: bool | None\n\n# A list of one or more datasets to eval the model with. You can use either\n# test_datasets, or val_set_size, but not both.\ntest_datasets: Annotated[list[SFTDataset | DPODataset | KTODataset | StepwiseSupervisedDataset], MinLen(1)] | None\n # For SFTDataset:\n # HuggingFace dataset repo | s3:// | gs:// | path to local file or directory\n path: str | None\n # name of dataset split to load from\n split: str | None\n # The type of prompt to use for training. [alpaca, gpteacher, oasst, reflection]\n type: str | UserDefinedPrompterType | None\n # For UserDefinedPrompterType:\n # Custom user instruction prompt\n system_prompt: str | None\n # Use {system} as key to be replaced\n system_format: str | None\n field_system: str | None\n field_instruction: str | None\n field_input: str | None\n field_output: str | None\n\n # Customizable to be single line or multi-line. Use {instruction}/{input} as key to\n # be replaced. 'format' can include {input}\n format: str | None\n # 'no_input_format' cannot include {input}\n no_input_format: str | None\n input_transform: str | None\n # split dataset into N pieces (use with shards_idx)\n shards: int | None\n # the index of sharded dataset to use\n shards_idx: int | None\n # process dataset in N sequential chunks for memory efficiency (exclusive with\n # `shards`)\n preprocess_shards: int | None\n conversation: str | None\n\n # The name of the chat template to use for training, following values are supported:\n # tokenizer_default: Uses the chat template that is available in the\n # tokenizer_config.json. If the chat template is not available in the tokenizer, it\n # will raise an error. This is the default.\n # alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates\n # are available in the axolotl codebase at src/axolotl/utils/chat_templates.py.\n # tokenizer_default_fallback_*: where * is the name of the chat template to fallback\n # to if the tokenizer does not have a chat template else default to tokenizer. E.g.\n # tokenizer_default_fallback_chatml. jinja: Uses a custom jinja template for the chat\n # template. The custom jinja template should be provided in the chat_template_jinja\n # field.\n chat_template: ChatTemplate | str | None\n # Custom jinja chat template or path to jinja file. Used only if `chat_template:\n # jinja` or empty.\n chat_template_jinja: str | None\n # path to source data files\n data_files: str | list[str] | None\n input_format: str | None\n # name of dataset configuration to load\n name: str | None\n # defines the datatype when path is a file\n ds_type: str | None\n # For `completion` datasets only, uses the provided field instead of `text` column\n field: str | None\n field_human: str | None\n field_model: str | None\n # Key containing the messages (default: \"messages\")\n field_messages: str | None\n # Key containing the tools (default: \"tools\"). Must be a list[dict] and follow [JSON\n # schema](https://json-schema.org/learn/getting-started-step-by-step).\n field_tools: str | None\n # Key containing the reasoning trace (default: \"reasoning_content\").\n field_thinking: str | None\n # The key the chat template expects that indicates the reasoning trace.\n template_thinking_key: str | None\n\n message_field_role: str | None\n\n message_field_content: str | None\n # Mapping of properties from the input dataset to the chat template. (default:\n # message_property_mappings={'role':'role', 'content':'content'}) If a property exists\n # in the template but not in this mapping, the system will attempt to load it directly\n # from the message using the property name as the key. Example: In the mapping below,\n # 'from' is loaded from input dataset and used as 'role', while 'value' is loaded and\n # used as 'content' in the chat template.\n message_property_mappings: dict[str, str] | None\n # The key in the message turn that indicates via boolean whether tokens of a turn\n # should be considered for training. Useful to selectively train on certain turns\n # besides the `roles_to_train`.\n message_field_training: str | None\n # The key in the message turn that contains the training details. Useful to\n # selectively train on certain tokens in a turn. The value of the key is a List[Dict]\n # containing `begin_offset` (start character index in content), `end_offset` (end\n # character index in content), and `train` (boolean whether to train).\n message_field_training_detail: str | None\n # (for Qwen3 template only) Whether to split the assistant content based on a\n # reasoning trace inside delimited tags\n split_thinking: bool | None\n logprobs_field: str | None\n temperature: float | None\n # Roles to train on. The tokens from these roles will be considered for the loss.\n roles_to_train: list[str] | None\n # Which EOS tokens to train on in the conversation. Possible values are: all: train on\n # all EOS tokens, turn (default): train on the EOS token at the end of each trainable\n # turn, last: train on the last EOS token in the conversation\n train_on_eos: Literal['all', 'turn', 'last'] | None\n # Roles mapping in the messages. The format is {target_role: [source_roles]}. All\n # source roles will be mapped to the target role. The default is: user: [\"human\",\n # \"user\"], assistant: [\"gpt\", \"assistant\"], system: [\"system\"], tool: [\"tool\"]\n roles: dict[str, list[str]] | None\n # Whether to drop the system turn from the dataset. Only works with chat_template.\n # This does not drop the default system message from chat_template if it exists. If\n # you wish to, we recommend using a custom jinja template with the default system\n # message removed or adding a system turn with empty content.\n drop_system_message: bool | None\n # Trust remote code for untrusted source\n trust_remote_code: bool | None = False\n # The specific revision of the dataset to use when loading from the Hugging Face Hub.\n # This can be a commit hash, tag, or branch name. If not specified, the latest version\n # will be used. This parameter is ignored for local datasets.\n revision: str | None\n\n # For DPODataset:\n path: str | None\n split: str | None\n type: UserDefinedDPOType | str | None\n # For UserDefinedDPOType:\n field_system: str | None\n field_prompt: str | None\n field_chosen: str | None\n field_rejected: str | None\n prompt_format: str | None\n chosen_format: str | None\n rejected_format: str | None\n data_files: list[str] | None\n revision: str | None\n field_messages: str | None\n\n # For KTODataset:\n path: str | None\n split: str | None\n type: UserDefinedKTOType | str | None\n # For UserDefinedKTOType:\n field_system: str | None\n field_prompt: str | None\n field_completion: str | None\n field_label: bool | None\n prompt_format: str | None\n completion_format: str | None\n data_files: list[str] | None\n trust_remote_code: bool | None = False\n revision: str | None\n\n # For StepwiseSupervisedDataset:\n path: str | None\n split: str | None\n data_files: list[str] | None\n revision: str | None\n step_separator: str | None\n max_completion_length: int | None\n train_on_last_step_only: bool | None\n\n# If false, the datasets will not be shuffled and will keep their original order in\n# `datasets`. The same applies to the `test_datasets` option and the\n# `pretraining_dataset` option. Default is true.\nshuffle_merged_datasets: bool | None = True\n# If true, each dataset in `datasets` will be shuffled before merging. This allows\n# curriculum learning strategies to be applied at the dataset level. Default is false.\nshuffle_before_merging_datasets: bool | None = False\n# Axolotl attempts to save the dataset as an arrow after packing the data together so\n# subsequent training attempts load faster, relative path\ndataset_prepared_path: str | None\n# Num shards for whole dataset\ndataset_shard_num: int | None\n# Index of shard to use for whole dataset\ndataset_shard_idx: int | None\nskip_prepare_dataset: bool | None = False\n# Number of shards to save the prepared dataset\nnum_dataset_shards_to_save: int | None\n\n# Set to HF dataset for type: 'completion' for streaming instead of pre-tokenize\npretraining_dataset: Annotated[list[PretrainingDataset | SFTDataset], MinLen(1)] | None\n # For PretrainingDataset:\n name: str | None\n path: str | None\n split: str | None = train\n text_column: str | None = text\n type: str | None = pretrain\n trust_remote_code: bool | None = False\n data_files: str | None\n skip: int | None\n\n # For SFTDataset:\n # HuggingFace dataset repo | s3:// | gs:// | path to local file or directory\n path: str | None\n # name of dataset split to load from\n split: str | None\n # The type of prompt to use for training. [alpaca, gpteacher, oasst, reflection]\n type: str | UserDefinedPrompterType | None\n # For UserDefinedPrompterType:\n # Custom user instruction prompt\n system_prompt: str | None\n # Use {system} as key to be replaced\n system_format: str | None\n field_system: str | None\n field_instruction: str | None\n field_input: str | None\n field_output: str | None\n\n # Customizable to be single line or multi-line. Use {instruction}/{input} as key to\n # be replaced. 'format' can include {input}\n format: str | None\n # 'no_input_format' cannot include {input}\n no_input_format: str | None\n input_transform: str | None\n # split dataset into N pieces (use with shards_idx)\n shards: int | None\n # the index of sharded dataset to use\n shards_idx: int | None\n # process dataset in N sequential chunks for memory efficiency (exclusive with\n # `shards`)\n preprocess_shards: int | None\n conversation: str | None\n\n # The name of the chat template to use for training, following values are supported:\n # tokenizer_default: Uses the chat template that is available in the\n # tokenizer_config.json. If the chat template is not available in the tokenizer, it\n # will raise an error. This is the default.\n # alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates\n # are available in the axolotl codebase at src/axolotl/utils/chat_templates.py.\n # tokenizer_default_fallback_*: where * is the name of the chat template to fallback\n # to if the tokenizer does not have a chat template else default to tokenizer. E.g.\n # tokenizer_default_fallback_chatml. jinja: Uses a custom jinja template for the chat\n # template. The custom jinja template should be provided in the chat_template_jinja\n # field.\n chat_template: ChatTemplate | str | None\n # Custom jinja chat template or path to jinja file. Used only if `chat_template:\n # jinja` or empty.\n chat_template_jinja: str | None\n # path to source data files\n data_files: str | list[str] | None\n input_format: str | None\n # name of dataset configuration to load\n name: str | None\n # defines the datatype when path is a file\n ds_type: str | None\n # For `completion` datasets only, uses the provided field instead of `text` column\n field: str | None\n field_human: str | None\n field_model: str | None\n # Key containing the messages (default: \"messages\")\n field_messages: str | None\n # Key containing the tools (default: \"tools\"). Must be a list[dict] and follow [JSON\n # schema](https://json-schema.org/learn/getting-started-step-by-step).\n field_tools: str | None\n # Key containing the reasoning trace (default: \"reasoning_content\").\n field_thinking: str | None\n # The key the chat template expects that indicates the reasoning trace.\n template_thinking_key: str | None\n\n message_field_role: str | None\n\n message_field_content: str | None\n # Mapping of properties from the input dataset to the chat template. (default:\n # message_property_mappings={'role':'role', 'content':'content'}) If a property exists\n # in the template but not in this mapping, the system will attempt to load it directly\n # from the message using the property name as the key. Example: In the mapping below,\n # 'from' is loaded from input dataset and used as 'role', while 'value' is loaded and\n # used as 'content' in the chat template.\n message_property_mappings: dict[str, str] | None\n # The key in the message turn that indicates via boolean whether tokens of a turn\n # should be considered for training. Useful to selectively train on certain turns\n # besides the `roles_to_train`.\n message_field_training: str | None\n # The key in the message turn that contains the training details. Useful to\n # selectively train on certain tokens in a turn. The value of the key is a List[Dict]\n # containing `begin_offset` (start character index in content), `end_offset` (end\n # character index in content), and `train` (boolean whether to train).\n message_field_training_detail: str | None\n # (for Qwen3 template only) Whether to split the assistant content based on a\n # reasoning trace inside delimited tags\n split_thinking: bool | None\n logprobs_field: str | None\n temperature: float | None\n # Roles to train on. The tokens from these roles will be considered for the loss.\n roles_to_train: list[str] | None\n # Which EOS tokens to train on in the conversation. Possible values are: all: train on\n # all EOS tokens, turn (default): train on the EOS token at the end of each trainable\n # turn, last: train on the last EOS token in the conversation\n train_on_eos: Literal['all', 'turn', 'last'] | None\n # Roles mapping in the messages. The format is {target_role: [source_roles]}. All\n # source roles will be mapped to the target role. The default is: user: [\"human\",\n # \"user\"], assistant: [\"gpt\", \"assistant\"], system: [\"system\"], tool: [\"tool\"]\n roles: dict[str, list[str]] | None\n # Whether to drop the system turn from the dataset. Only works with chat_template.\n # This does not drop the default system message from chat_template if it exists. If\n # you wish to, we recommend using a custom jinja template with the default system\n # message removed or adding a system turn with empty content.\n drop_system_message: bool | None\n # Trust remote code for untrusted source\n trust_remote_code: bool | None = False\n # The specific revision of the dataset to use when loading from the Hugging Face Hub.\n # This can be a commit hash, tag, or branch name. If not specified, the latest version\n # will be used. This parameter is ignored for local datasets.\n revision: str | None\n\n# The maximum number of processes to use while preprocessing your input dataset. This\n# defaults to `os.cpu_count()` if not set. For Runpod VMs, it will default to number of\n# vCPUs via RUNPOD_CPU_COUNT.\ndataset_processes: int | None\n# The maximum number of processes to use while preprocessing your input dataset. This\n# defaults to `os.cpu_count()` if not set. For Runpod VMs, it will default to number of\n# vCPUs via RUNPOD_CPU_COUNT.\ndataset_num_proc: int | None\n\n# Deduplicates datasets and test_datasets with identical entries\ndataset_exact_deduplication: bool | None\n# Keep dataset in memory while preprocessing. Only needed if cached dataset is taking\n# too much storage\ndataset_keep_in_memory: bool | None\ndataloader_pin_memory: bool | None\ndataloader_num_workers: int | None\ndataloader_prefetch_factor: int | None\ndataloader_drop_last: bool | None\n\naccelerator_config: dict[str, Any] | None\n\nremove_unused_columns: bool | None\n\n# Push prepared dataset to hub - repo_org/repo_name\npush_dataset_to_hub: str | None\n# Whether to use hf `use_auth_token` for loading datasets. Useful for fetching private\n# datasets. Required to be true when used in combination with `push_dataset_to_hub`\nhf_use_auth_token: bool | None\n\ndevice: Any | None\n# Passed through to transformers when loading the model when launched without\n# accelerate. Use `sequential` when training w/ model parallelism to limit memory\ndevice_map: Any | None\nworld_size: int | None\n# Don't mess with this, it's here for accelerate and torchrun\nlocal_rank: int | None\nddp: bool | None\n\n# Seed for reproducibility\nseed: int | None\n# Advanced DDP Arguments - timeout\nddp_timeout: int | None\n# Advanced DDP Arguments - bucket cap in MB\nddp_bucket_cap_mb: int | None\n# Advanced DDP Arguments - broadcast buffers\nddp_broadcast_buffers: bool | None\nddp_find_unused_parameters: bool | None\n\n# Whether to run causal language model evaluation for metrics in\n# `eval_causal_lm_metrics`\ndo_causal_lm_eval: bool | None\n# HF evaluate metrics used during evaluation. Default is ['sacrebleu', 'comet', 'ter',\n# 'chrf', 'perplexity']\neval_causal_lm_metrics: list[str] | None\ndo_bench_eval: bool | None\nbench_dataset: str | None\nbench_split: str | None\nmetric_for_best_model: str | None\ngreater_is_better: bool | None\n\n# High loss value, indicating the learning has broken down (a good estimate is ~2 times\n# the loss at the start of training)\nloss_watchdog_threshold: float | None\n# Number of high-loss steps in a row before the trainer aborts (default: 3)\nloss_watchdog_patience: int | None\n\n# Run garbage collection every `gc_steps` steps. -1 will run on epoch end and before\n# evaluations. Default is 0 (disabled).\ngc_steps: int | None\n\n# Use CUDA bf16. bool or 'full' for `bf16_full_eval`, or 'auto' for automatic detection.\n# require >=ampere\nbf16: Literal['auto'] | bool | None = auto\n# Use CUDA fp16\nfp16: bool | None\n# Enable FP8 mixed precision training using TorchAO. Best used in combination with\n# torch.compile.\nfp8: bool | None\n# Enable FSDP float8 all-gather optimization for FP8 training. Can improve training\n# speed by 10-15% when FSDP is enabled.\nfp8_enable_fsdp_float8_all_gather: bool | None\n# No AMP (automatic mixed precision) - require >=ampere\nbfloat16: bool | None\n# No AMP (automatic mixed precision)\nfloat16: bool | None\n# Use CUDA tf32 - require >=ampere\ntf32: bool | None\nfloat32: bool | None\n\n# Whether to use gradient checkpointing. Available options are: true, false, 'offload',\n# 'offload_disk'.\n# https://huggingface.co/docs/transformers/v4.18.0/en/performance#gradient-checkpointing\ngradient_checkpointing: Literal['offload', 'offload_disk'] | bool | None = False\n# Additional kwargs to pass to the trainer for gradient checkpointing\ngradient_checkpointing_kwargs: dict[str, Any] | None\n# Whether to offload activations. Available options are: true, false, 'legacy', 'disk'.\nactivation_offloading: Literal['legacy', 'disk'] | bool | None = False\n\n# List of regex patterns for parameter names to keep unfrozen. All other parameters will\n# be frozen via requires_grad=False. Note: range-based patterns (e.g.\n# embed_tokens.weight$[:32000]) use gradient zeroing rather than a true freeze, so\n# weight decay will still apply to the frozen portion and optimizer states are allocated\n# for the full parameter.\nunfrozen_parameters: list[str] | None\n\n# The maximum length of an input to train with, this should typically be less than 2048\n# as most models have a token/context limit of 2048\nsequence_len: int = 512\n# What to do when a tokenized row exceeds sequence_len. 'drop' removes the row;\n# 'truncate' slices tensors to sequence_len; 'raise' raises a ValueError. Defaults to\n# 'drop' for backward compatibility.\nexcess_length_strategy: Literal['drop', 'truncate', 'raise'] | None\n# The maximum length of an input for evaluation. If not specified, defaults to\n# sequence_len\neval_sequence_len: int | None\nmin_sample_len: int | None\n# maximum prompt length for RL training\nmax_prompt_len: int | None\n# Use efficient multi-packing with block diagonal attention and per sequence\n# position_ids. Recommend set to 'true'\nsample_packing: bool | None\n# The number of samples packed at a time. Increasing the following values helps with\n# packing, but usually only slightly (<%1.)\nsample_packing_group_size: int | None = 100000\n# The number of samples which can be packed into one sequence. Increase if using a large\n# sequence_len with many short samples.\nsample_packing_bin_size: int | None = 200\n# Whether to pack samples sequentially\nsample_packing_sequentially: bool | None\n# The multiprocessing start method to use for packing. Should be 'fork', 'spawn' or\n# 'forkserver'\nsample_packing_mp_start_method: str | None\n# Set to 'false' if getting errors during eval with sample_packing on\neval_sample_packing: bool | None\n# Pad inputs so each step uses constant sized buffers. This will reduce memory\n# fragmentation and may prevent OOMs, by re-using memory more efficiently. Defaults to\n# True if `sample_packing` enabled\npad_to_sequence_len: bool | None\n# Whether to use sequential sampling for curriculum learning\ncurriculum_sampling: bool | None\nmultipack_real_batches: bool | None\n\n# Use batch flattening for speedups when not using sample_packing\nbatch_flattening: Literal['auto'] | bool | None\n\nuse_pose: bool | None\npose_split_on_token_ids: list[int] | None\npose_max_context_len: int | None\npose_num_chunks: int | None\n\npretrain_multipack_buffer_size: int | None\n# whether to prevent cross attention for packed sequences during pretraining\npretrain_multipack_attn: bool | None = True\n# whether to concatenate samples during pretraining\npretraining_sample_concatenation: bool | None\n\n# Use streaming mode for loading datasets\nstreaming: bool | None\n# Buffer size for multipack streaming datasets\nstreaming_multipack_buffer_size: int | None = 10000\n\n# Whether to use xformers attention patch https://github.com/facebookresearch/xformers\nxformers_attention: bool | None\n# Whether to use scaled-dot-product attention https://pytorch.org/docs/stable/generated/\n# torch.nn.functional.scaled_dot_product_attention.html\nsdp_attention: bool | None\n# Shifted-sparse attention (only llama) - https://arxiv.org/pdf/2309.12307.pdf\ns2_attention: bool | None\nflex_attention: bool | None\nflex_attn_compile_kwargs: dict[str, Any] | None\n# Whether to use flash attention patch https://github.com/Dao-AILab/flash-attention\nflash_attention: bool | None\n# Whether to use flash-attention cross entropy implementation - advanced use only\nflash_attn_cross_entropy: bool | None\n# Whether to use flash-attention rms norm implementation - advanced use only\nflash_attn_rms_norm: bool | None\n# Whether to fuse part of the MLP into a single operation\nflash_attn_fuse_mlp: bool | None\n# Whether to use bettertransformers\nflash_optimum: bool | None\n# Whether to use SageAttention https://github.com/thu-ml/SageAttention\nsage_attention: bool | None\n\neager_attention: bool | None\n\n# Specify a custom attention implementation, used mostly for kernels.\nattn_implementation: str | None\n\n# Which experts implementation to use for MoE models,\nexperts_implementation: str | None\n\n# Quantize MoE expert weights on load to reduce VRAM. Requires adapter (lora/qlora) with\n# load_in_4bit or load_in_8bit. Requires CUDA (not compatible with ROCm or other\n# backends). Note: total parameter count may be reported incorrectly when enabled\n# (trainable param count is correct).\nquantize_moe_experts: bool = False\n\n# Whether to use Scaled Softmax (SSMax) attention. Ref: https://arxiv.org/abs/2501.19399\nscaling_softmax: bool | None\n# Scaling factor for SSMax attention. Default is 0.43\nscaling_softmax_factor: float | None\n# Bias for SSMax attention. Default is 0.0. Note: The paper recommends bias=0 for better\n# length generalization.\nscaling_softmax_bias: float | None\n\nunsloth_cross_entropy_loss: bool | None\nunsloth_lora_mlp: bool | None\nunsloth_lora_qkv: bool | None\nunsloth_lora_o: bool | None\nunsloth_rms_norm: bool | None\nunsloth_rope: bool | None\n\n# Apply custom LoRA autograd functions and activation function Triton kernels for speed\n# and memory savings. See: https://docs.axolotl.ai/docs/lora_optims.html\nlora_mlp_kernel: bool | None\n# Apply custom LoRA autograd functions and activation function Triton kernels for speed\n# and memory savings. See: https://docs.axolotl.ai/docs/lora_optims.html\nlora_qkv_kernel: bool | None\n# Apply custom LoRA autograd functions and activation function Triton kernels for speed\n# and memory savings. See: https://docs.axolotl.ai/docs/lora_optims.html\nlora_o_kernel: bool | None\n\n# Whether to use chunked cross entropy loss for memory efficiency\nchunked_cross_entropy: bool | None\n# Number of chunks to use for chunked cross entropy loss\nchunked_cross_entropy_num_chunks: int | None\n# Enable Entropy-Aware Focal Training loss (EAFT)\nuse_eaft: bool | None\n# Exponent for entropy weighting in EAFT (default: 1.0)\neaft_alpha: float | None = 1.0\n# Number of top logits for entropy approximation (default: 20)\neaft_k: int | None = 20\n\n# Whether to use ALST tiled mlp for memory efficient long context\ntiled_mlp: bool | None\n\n# Number of shards to use for ALST tiled mlp. If unset, it will be set based on\n# seqlen/hidden_size\ntiled_mlp_num_shards: int | None\n\n# Whether to use original mlp for ALST tiled mlp. Otherwise uses a generic MLP based on\n# llama.\ntiled_mlp_use_original_mlp: bool | None = True\n\nllama4_linearized_experts: bool | None\n\n# Deepspeed config path. e.g., deepspeed_configs/zero3.json\ndeepspeed: str | dict[str, Any] | None\n# Whether to use deepcompile for faster training with deepspeed\ndeepcompile: bool | None\n# FSDP configuration\nfsdp: list[str] | None\n\n# FSDP configuration options\nfsdp_config: FSDPConfig | None\n # For FSDPConfig:\n # FSDP version\n fsdp_version: int | None\n # Enable activation checkpointing to reduce memory usage during forward passes\n activation_checkpointing: bool | None\n # Offload parameters to CPU to reduce GPU memory usage\n offload_params: bool | None\n # Synchronize module states across all processes\n sync_module_states: bool | None\n # Enable CPU RAM efficient loading to reduce memory usage during model loading\n cpu_ram_efficient_loading: bool | None\n # Disabling this enables swap memory usage for resource-constrained setups when\n # offload_params is enabled.\n cpu_offload_pin_memory: bool | None\n # Use original parameters instead of flattened parameters\n use_orig_params: bool | None\n\n # Type of state dict to use for saving/loading checkpoints\n state_dict_type: Literal['FULL_STATE_DICT', 'LOCAL_STATE_DICT', 'SHARDED_STATE_DICT'] | None\n # Final state dict type to use after training completion\n final_state_dict_type: Literal['FULL_STATE_DICT', 'LOCAL_STATE_DICT', 'SHARDED_STATE_DICT'] | None\n\n # Policy for automatically wrapping modules with FSDP\n auto_wrap_policy: Literal['TRANSFORMER_BASED_WRAP', 'SIZE_BASED_WRAP'] | None\n # Class name of transformer layers to wrap (e.g., 'LlamaDecoderLayer')\n transformer_layer_cls_to_wrap: str | None\n\n # Reshard parameters after forward pass to save memory\n reshard_after_forward: bool | None\n # Mixed precision policy for FSDP (e.g., 'fp16', 'bf16')\n mixed_precision_policy: str | None\n\n# FSDP version\nfsdp_version: int | None\nfsdp_final_state_dict_type: Literal['FULL_STATE_DICT', 'LOCAL_STATE_DICT', 'SHARDED_STATE_DICT'] | None\n\n# How much of the dataset to set aside as evaluation. 1 = 100%, 0.50 = 50%, etc. 0 for\n# no eval.\nval_set_size: float | None = 0.0\n\n# Number of devices to shard across. If not set, will use all available devices.\ndp_shard_size: int | None\n# Number of devices to replicate across.\ndp_replicate_size: int | None\n# Deprecated: use `context_parallel_size` instead\nsequence_parallel_degree: int | None\n# Set to a divisor of the number of GPUs available to split sequences into chunks of\n# equal size. Use in long context training to prevent OOM when sequences cannot fit into\n# a single GPU's VRAM. E.g., if 4 GPUs are available, set this value to 2 to split each\n# sequence into two equal-sized subsequences, or set to 4 to split into four equal-sized\n# subsequences. See https://docs.axolotl.ai/docs/sequence_parallelism.html for more\n# details.\ncontext_parallel_size: int | None\n# Optional; strides across the key dimension. Larger values use more memory but should\n# make training faster. Must evenly divide the number of KV heads in your model.\nheads_k_stride: int | None\n# One of 'varlen_llama3', 'batch_ring', 'batch_zigzag', 'batch_stripe'. Defaults to\n# 'varlen_llama3' in the sample packing case, and 'batch_ring' in the non-sample packing\n# case.\nring_attn_func: RingAttnFunc | None\n# Number of tensor parallel processes in TP group. Only supported with DeepSpeed AutoTP.\ntensor_parallel_size: int | None\n\n# Add or change special tokens. If you add tokens here, you don't need to add them to\n# the `tokens` list.\nspecial_tokens: SpecialTokensConfig | None\n # For SpecialTokensConfig:\n bos_token: str | None\n eos_token: str | None\n pad_token: str | None\n unk_token: str | None\n additional_special_tokens: list[str] | None\n\n# Add extra tokens to the tokenizer\ntokens: list[str] | None\n# Mapping token_id to new_token_string to override reserved added_tokens in the\n# tokenizer. Only works for tokens that are not part of the base vocab (aka are\n# added_tokens). Can be checked if they exist in tokenizer.json added_tokens.\nadded_tokens_overrides: dict[int, str] | None\n\n# Whether to use torch.compile and which backend to use. setting to `auto` will enable\n# torch compile when torch>=2.6.0\ntorch_compile: Literal['auto'] | bool | None\n# Backend to use for torch.compile\ntorch_compile_backend: str | None\ntorch_compile_mode: Literal['default', 'reduce-overhead', 'max-autotune'] | None\n\n# Maximum number of iterations to train for. It precedes num_epochs which means that if\n# both are set, num_epochs will not be guaranteed. e.g., when 1 epoch is 1000 steps =>\n# `num_epochs: 2` and `max_steps: 100` will train for 100 steps\nmax_steps: int | None\n# Number of warmup steps. Cannot use with warmup_ratio\nwarmup_steps: int | None\n# Warmup ratio. Cannot use with warmup_steps\nwarmup_ratio: float | None\n# Leave empty to eval at each epoch, integer for every N steps. float for fraction of\n# total steps\neval_steps: int | float | None\n# Number of times per epoch to run evals, mutually exclusive with eval_steps\nevals_per_epoch: int | None\n# Set to `no` to skip evaluation, `epoch` at end of each epoch, leave empty to infer\n# from `eval_steps`\neval_strategy: str | None\n\n# Leave empty to save at each epoch, integer for every N steps. float for fraction of\n# total steps\nsave_steps: int | float | None\n# Number of times per epoch to save a checkpoint, mutually exclusive with save_steps\nsaves_per_epoch: int | None\n# Set to `no` to skip checkpoint saves, `epoch` at end of each epoch, `best` when better\n# result is achieved, leave empty to infer from `save_steps`\nsave_strategy: str | None\n# Checkpoints saved at a time\nsave_total_limit: int | None\n# Whether to checkpoint a model after the first step of training. Defaults to False.\nsave_first_step: bool | None\n\n# Logging frequency\nlogging_steps: int | None\n# Stop training after this many evaluation losses have increased in a row. https://huggi\n# ngface.co/transformers/v4.2.2/_modules/transformers/trainer_callback.html#EarlyStoppin\n# gCallback\nearly_stopping_patience: int | None\nload_best_model_at_end: bool | None = False\n# Save only the model weights, skipping the optimizer. Using this means you can't resume\n# from checkpoints.\nsave_only_model: bool | None = False\n# Use tensorboard for logging\nuse_tensorboard: bool | None\n# Enable the pytorch profiler to capture the first N steps of training to the\n# output_dir. see https://pytorch.org/blog/understanding-gpu-memory-1/ for more\n# information. Snapshots can be visualized @ https://pytorch.org/memory_viz\nprofiler_steps: int | None\n# Which step to start the profiler at. Useful for only capturing a few steps mid-run.\nprofiler_steps_start: int | None = 0\n# bool of whether to report tokens per second at the end of training. This is not\n# supported with pre-training datasets.\ninclude_tokens_per_second: bool | None\n# bool of whether to report tokens per second per-gpu during training by measuring\n# throughput of non-padding tokens.\ninclude_tkps: bool | None = True\n# NEFT https://arxiv.org/abs/2310.05914, set this to a number (paper default is 5) to\n# add noise to embeddings. Currently only supported on Llama and Mistral\nneftune_noise_alpha: float | None\n\n# Parameter controlling the relative ratio loss weight in the ORPO loss. Passed to\n# `beta` in `ORPOConfig` due to trl mapping.\norpo_alpha: float | None\n# Weighting of NLL term in loss from RPO paper\nrpo_alpha: float | None\n# Target reward margin for the SimPO loss\nsimpo_gamma: float | None\n# Weight of the BC regularizer\ncpo_alpha: float | None\n\n# Factor for desirable loss term in KTO loss\nkto_desirable_weight: float | None\n# Factor for undesirable loss term in KTO loss\nkto_undesirable_weight: float | None\n# The beta parameter for the RL training\nrl_beta: float | None\n\n# Defines the max memory usage per gpu on the system. Passed through to transformers\n# when loading the model.\nmax_memory: dict[int | Literal['cpu', 'disk'], int | str] | None\n# Limit the memory for all available GPUs to this amount (if an integer, expressed in\n# gigabytes); default: unset\ngpu_memory_limit: int | str | None\n# Whether to use low_cpu_mem_usage\nlow_cpu_mem_usage: bool | None\n\n# The name of the chat template to use for training, following values are supported:\n# tokenizer_default: Uses the chat template that is available in the\n# tokenizer_config.json. If the chat template is not available in the tokenizer, it will\n# raise an error. This is the default value.\n# alpaca/inst/chatml/gemma/cohere/llama3/phi_3/deepseek_v2/jamba: These chat templates\n# are available in the axolotl codebase at src/axolotl/utils/chat_templates.py.\n# tokenizer_default_fallback_*: where * is the name of the chat template to fallback to.\n# E.g. tokenizer_default_fallback_chatml. This is useful when the chat template is not\n# available in the tokenizer. jinja: Uses a custom jinja template for the chat template.\n# The custom jinja template should be provided in the chat_template_jinja field. The\n# selected chat template will be saved to the tokenizer_config.json for easier\n# inferencing\nchat_template: ChatTemplate | Annotated[str, StringConstraints(pattern='^tokenizer_default_fallback_')] | None\n# Custom jinja template or path to jinja file for chat template. This will be only used\n# if chat_template is set to `jinja` or `null` (in which case chat_template is\n# automatically set to `jinja`). Default is null.\nchat_template_jinja: str | None\n# Additional kwargs to pass to the chat template. This is useful for customizing the\n# chat template. For example, you can pass `thinking=False` to add a generation prompt\n# to the chat template.\nchat_template_kwargs: dict[str, Any] | None\n# Custom EOT (End-of-Turn) tokens to mask/unmask during training. These tokens mark the\n# boundaries between conversation turns. For example: ['/INST', '</s>',\n# '[/SYSTEM_PROMPT]']. If not specified, defaults to just the model's eos_token. This is\n# useful for templates that use multiple delimiter tokens.\neot_tokens: list[str] | None\n# Changes the default system message. Currently only supports chatml.\ndefault_system_message: str | None\n\n# Token index or indices to adjust embedding weights to the mean of the other tokens.\n# This is useful when the model has untrained embeddings.\nfix_untrained_tokens: int | list[int] | None\n\nis_preprocess: bool | None\npreprocess_iterable: bool | None\n\n# Total number of tokens - internal use\ntotal_num_tokens: int | None\ntotal_supervised_tokens: int | None\n# You can set these packing optimizations AFTER starting a training at least once. The\n# trainer will provide recommended values for these values.\nsample_packing_eff_est: float | None\naxolotl_config_path: str | None\n\n# Internal use only - Used to identify which the model is based on\nis_falcon_derived_model: bool | None\n# Internal use only - Used to identify which the model is based on\nis_llama_derived_model: bool | None\n# Internal use only - Used to identify which the model is based on. Please note that if\n# you set this to true, `padding_side` will be set to 'left' by default\nis_mistral_derived_model: bool | None\n# Internal use only - Used to identify which the model is based on\nis_qwen_derived_model: bool | None\n\n# Add plugins to extend the pipeline. See `src/axolotl/integrations` for the available\n# plugins or doc below for more details.\n# https://docs.axolotl.ai/docs/custom_integrations.html\nplugins: list[str] | None\n# Enable sample generation during training for monitoring\ngenerate_samples: bool | None = False\n# Number of samples to generate at each interval\nnum_generation_samples: int | None = 3\n# Maximum new tokens to generate per sample\ngeneration_max_new_tokens: int | None = 50\n# Temperature for sample generation (0.0 = greedy)\ngeneration_temperature: float | None = 0.7\n# Nucleus sampling parameter for generation\ngeneration_top_p: float | None\n# Top-k sampling parameter for generation\ngeneration_top_k: int | None\n# Ratio of input to use as prompt (0.0-1.0)\ngeneration_prompt_ratio: float | None = 0.5\n# Whether to use sampling (vs greedy decoding)\ngeneration_do_sample: bool | None = True\n\n# This is the huggingface model that contains *.pt, *.safetensors, or *.bin files. This\n# can also be a relative path to a model on disk\nbase_model: str (required)\n# If the base_model repo on hf hub doesn't include configuration .json files, You can\n# set that here, or leave this empty to default to base_model\nbase_model_config: str | None\n# transformers config class (e.g., 'LlamaConfig', 'MistralConfig'). Defaults to\n# AutoConfig.\ncls_model_config: str | None\n# Optional tokenizer configuration path in case you want to use a different tokenizer\n# than the one defined in the base model\ntokenizer_config: str | None\n# use_fast option for tokenizer loading from_pretrained, default to True\ntokenizer_use_fast: bool | None\n# Whether to use the legacy tokenizer setting, defaults to True\ntokenizer_legacy: bool | None\n# Whether to use mistral-common tokenizer. If set to True, it will use the mistral-\n# common tokenizer.\ntokenizer_use_mistral_common: bool | None\n# Corresponding tokenizer for the model AutoTokenizer is a good choice\ntokenizer_type: str | None\n# transformers processor class\nprocessor_type: str | None\n# Whether to save jinja files for tokenizer, transformers default is True\ntokenizer_save_jinja_files: bool | None = True\n# Trust remote code for untrusted source\ntrust_remote_code: bool | None\n\n# Don't move the model to the device before sharding. Set to `false` to revert to legacy\n# behavior.\nexperimental_skip_move_to_device: bool | None = True\n\n# Use custom kernels, e.g. MegaBlocks.\nuse_kernels: bool | None\n\n# Model loading quantization config\nmodel_quantization_config: Literal['Mxfp4Config'] | None\n# kwargs for model quantization config\nmodel_quantization_config_kwargs: dict[str, Any] | None\n\n# Where to save the full-finetuned model to\noutput_dir: str = ./model-out\n# push checkpoints to hub\nhub_model_id: str | None\n# how to push checkpoints to hub\nhub_strategy: str | None\n# branch/revision to push to on hub (default: main)\nhub_revision: str | None\n# Whether to save the model using safetensors format. Defaults to True.\nsave_safetensors: bool | None = True\n\n# This will attempt to quantize the model down to 8 bits and use adam 8 bit optimizer\nload_in_8bit: bool | None = False\n# Use bitsandbytes 4 bit\nload_in_4bit: bool | None = False\n\n# If you want to use 'lora', 'qlora', or 'llama-adapter', or leave blank to train all\n# parameters in original model\nadapter: Literal['lora', 'qlora', 'llama-adapter'] | None\n# If you already have a lora model trained that you want to load, put that here. This\n# means after training, if you want to test the model, you should set this to the value\n# of `output_dir`. Note that if you merge an adapter to the base model, a new\n# subdirectory `merged` will be created under the `output_dir`.\nlora_model_dir: str | None\nlora_r: int | None\nlora_alpha: int | None\nlora_fan_in_fan_out: bool | None\nlora_target_modules: str | list[str] | None\nlora_target_parameters: str | list[str] | None\n# If true, will target all linear modules\nlora_target_linear: bool | None\n# If you added new tokens to the tokenizer, you may need to save some LoRA modules\n# because they need to know the new tokens. For LLaMA and Mistral, you need to save\n# `embed_tokens` and `lm_head`. It may vary for other models. `embed_tokens` converts\n# tokens to embeddings, and `lm_head` converts embeddings to token probabilities.\nlora_modules_to_save: list[str] | None\nlora_dropout: float | None = 0.0\n# The layer indices to transform, otherwise, apply to all layers\npeft_layers_to_transform: list[int] | None\npeft_layers_pattern: list[str] | None\n\npeft: PeftConfig | None\n # For PeftConfig:\n # Configuration options for loftq initialization for LoRA\n loftq_config: LoftQConfig | None\n # For LoftQConfig:\n # typically 4 bits\n loftq_bits: int = 4\n\n# Whether to use DoRA.\npeft_use_dora: bool | None\n# Whether to use RSLoRA.\npeft_use_rslora: bool | None\n# List of layer indices to replicate.\npeft_layer_replication: list[tuple[int, int]] | None\n# How to initialize LoRA weights. Default to True which is MS original implementation.\npeft_init_lora_weights: bool | str | None\n# A list of token indices to fine-tune on the `embed_tokens` layer. Otherwise, a dict\n# mapping an embedding layer name to its trainable token indices. See\n# https://huggingface.co/docs/peft/v0.17.0/en/developer_guides/lora#efficiently-train-\n# tokens-alongside-lora\npeft_trainable_token_indices: list[int] | dict[str, list[int]] | None\n# Whether to tie adapter weights for tied model weights. See\n# https://github.com/huggingface/peft/issues/2864\npeft_ensure_weight_tying: bool | None\n# Whether to upcast the LoRA adapter to fp32. This is enabled by default in PEFT.\npeft_autocast_adapter_dtype: bool | None\n\n# load qlora model in sharded format for FSDP using answer.ai technique.\nqlora_sharded_model_loading: bool | None = False\n# Do the LoRA/PEFT loading on CPU -- this is required if the base model is so large it\n# takes up most or all of the available GPU VRAM, e.g. during a model and LoRA merge\nlora_on_cpu: bool | None\n# Whether you are training a 4-bit GPTQ quantized model\ngptq: bool | None\n# optional overrides to the bnb 4bit quantization configuration\nbnb_config_kwargs: dict[str, Any] | None\n\n# loraplus learning rate ratio lr_B / lr_A. Recommended value is 2^4.\nloraplus_lr_ratio: float | None\n# loraplus learning rate for lora embedding layers. Default value is 1e-6.\nloraplus_lr_embedding: float | None = 1e-06\n\nmerge_lora: bool | None\n\n# Whether to use ReLoRA. Use with jagged_restart_*steps options.\nrelora: bool | None\n# threshold for optimizer magnitude when pruning\nrelora_prune_ratio: float | None\n# True to perform lora weight merges on cpu during restarts, for modest gpu memory\n# savings\nrelora_cpu_offload: bool | None\n\n# how often to reset for jagged restarts\njagged_restart_steps: int | None\n# how many warmup steps to take after reset for jagged restarts\njagged_restart_warmup_steps: int | None\n# how many anneal steps to take before reset for jagged restarts\njagged_restart_anneal_steps: int | None\n\n# If greater than 1, backpropagation will be skipped and the gradients will be\n# accumulated for the given number of steps.\ngradient_accumulation_steps: int | None = 1\n# The number of samples to include in each batch. This is the number of samples sent to\n# each GPU. Batch size per gpu = micro_batch_size * gradient_accumulation_steps\nmicro_batch_size: int | None = 1\n# Total batch size, we do not recommended setting this manually\nbatch_size: int | None\n# per gpu micro batch size for evals, defaults to value of micro_batch_size\neval_batch_size: int | None\n\n# whether to find batch size that fits in memory. Passed to underlying transformers\n# Trainer\nauto_find_batch_size: bool | None\n\n# Whether to mask out or include the human's prompt from the training labels\ntrain_on_inputs: bool | None = False\n# Group similarly sized data to minimize padding. May be slower to start, as it must\n# download and sort the entire dataset. Note that training loss may have an oscillating\n# pattern with this enabled.\ngroup_by_length: bool | None\n\nlearning_rate: str | float (required)\nembedding_lr: float | None\nembedding_lr_scale: float | None\n# Specify weight decay\nweight_decay: float | None = 0.0\n# Specify optimizer\noptimizer: OptimizerNames | CustomSupportedOptimizers | None = OptimizerNames.ADAMW_TORCH_FUSED\n# Dictionary of arguments to pass to the optimizer\noptim_args: str | dict[str, Any] | None\n# The target modules to optimize, i.e. the module names that you would like to train,\n# right now this is used only for GaLore algorithm\noptim_target_modules: list[str] | Literal['all_linear'] | None\n# Path to torch distx for optim 'adamw_anyprecision'\ntorchdistx_path: str | None\nlr_scheduler: SchedulerType | Literal['one_cycle'] | Literal['rex'] | None = SchedulerType.COSINE\n# Specify a scheduler and kwargs to use with the optimizer\nlr_scheduler_kwargs: dict[str, Any] | None\nlr_quadratic_warmup: bool | None\n# decay lr to some percentage of the peak lr, e.g. cosine_min_lr_ratio=0.1 for 10% of\n# peak lr\ncosine_min_lr_ratio: float | None\n# freeze lr at some percentage of the step, e.g. cosine_constant_lr_ratio=0.8 means\n# start cosine_min_lr at 80% of training step\ncosine_constant_lr_ratio: float | None\n# Learning rate div factor\nlr_div_factor: float | None\n\nlr_groups: list[LrGroup] | None\n # For LrGroup:\n name: str (required)\n modules: list[str] (required)\n lr: float (required)\n\n# adamw hyperparams\nadam_epsilon: float | None\n# only used for CAME Optimizer\nadam_epsilon2: float | None\n# adamw hyperparams\nadam_beta1: float | None\n# adamw hyperparams\nadam_beta2: float | None\n# only used for CAME Optimizer\nadam_beta3: float | None\n\n# Dion Optimizer learning rate\ndion_lr: float | None\n# Dion Optimizer momentum\ndion_momentum: float | None\n# Dion Optimizer: r/d fraction for low-rank approximation. Used to compute the low-rank\n# dimension.\ndion_rank_fraction: float | None = 1.0\n# Dion Optimizer: Round up the low-rank dimension to a multiple of this number. This may\n# be useful to ensure even sharding.\ndion_rank_multiple_of: int | None = 1\n\n# Gradient clipping max norm\nmax_grad_norm: float | None\nnum_epochs: float = 1.0\n\nuse_wandb: bool | None\n# Set the name of your wandb run\nwandb_name: str | None\n# Set the ID of your wandb run\nwandb_run_id: str | None\n# \"offline\" to save run metadata locally and not sync to the server, \"disabled\" to turn\n# off wandb\nwandb_mode: str | None\n# Your wandb project name\nwandb_project: str | None\n# A wandb Team name if using a Team\nwandb_entity: str | None\nwandb_watch: str | None\n# \"checkpoint\" to log model to wandb Artifacts every `save_steps` or \"end\" to log only\n# at the end of training\nwandb_log_model: str | None\n\nuse_mlflow: bool | None\n# URI to mlflow\nmlflow_tracking_uri: str | None\n# Your experiment name\nmlflow_experiment_name: str | None\n# Your run name\nmlflow_run_name: str | None\n# set to true to copy each saved checkpoint on each save to mlflow artifact registry\nhf_mlflow_log_artifacts: bool | None\n\n# Enable or disable Comet integration.\nuse_comet: bool | None\n# API key for Comet. Recommended to set via `comet login`.\ncomet_api_key: str | None\n# Workspace name in Comet. Defaults to the user's default workspace.\ncomet_workspace: str | None\n# Project name in Comet. Defaults to Uncategorized.\ncomet_project_name: str | None\n# Identifier for the experiment. Used to append data to an existing experiment or\n# control the key of new experiments. Default to a random key.\ncomet_experiment_key: str | None\n# Create a new experiment (\"create\") or log to an existing one (\"get\"). Default\n# (\"get_or_create\") auto-selects based on configuration.\ncomet_mode: str | None\n# Set to True to log data to Comet server, or False for offline storage. Default is\n# True.\ncomet_online: bool | None\n# Dictionary for additional configuration settings, see the doc for more details.\ncomet_experiment_config: dict[str, Any] | None\n\nuse_trackio: bool | None\n# Your trackio project name\ntrackio_project_name: str | None\n# Set the name of your trackio run\ntrackio_run_name: str | None\n# Hugging Face Space ID to sync dashboard to (optional, runs locally if not provided)\ntrackio_space_id: str | None\n\n# Enable OpenTelemetry metrics collection and Prometheus export\nuse_otel_metrics: bool | None = False\n# Host to bind the OpenTelemetry metrics server to\notel_metrics_host: str | None = localhost\n# Port for the Prometheus metrics HTTP server\notel_metrics_port: int | None = 8000\n\n# the number of activate layers in LISA\nlisa_n_layers: int | None\n# how often to switch layers in LISA\nlisa_step_interval: int | None\n# path under the model to access the layers\nlisa_layers_attribute: str | None = model.layers\n\ngradio_title: str | None\ngradio_share: bool | None\ngradio_server_name: str | None\ngradio_server_port: int | None\ngradio_max_new_tokens: int | None\ngradio_temperature: float | None\n\nuse_ray: bool = False\nray_run_name: str | None\nray_num_workers: int = 1\nresources_per_worker: dict\n\n# The size of the image to resize to. It can be an integer (resized into padded-square\n# image) or a tuple (width, height).If not provided, we will attempt to load from\n# preprocessor.size, otherwise, images won't be resized.\nimage_size: int | tuple[int, int] | None\n# The resampling algorithm to use for image resizing. Default is bilinear. Please refer\n# to PIL.Image.Resampling for more details.\nimage_resize_algorithm: Literal['bilinear', 'bicubic', 'lanczos'] | Resampling | None\n\n# optional overrides to the base model configuration\noverrides_of_model_config: dict[str, Any] | None\n# optional overrides the base model loading from_pretrained\noverrides_of_model_kwargs: dict[str, Any] | None\n# If you want to specify the type of model to load, AutoModelForCausalLM is a good\n# choice too\ntype_of_model: str | None\n# You can specify to choose a specific model revision from huggingface hub\nrevision_of_model: str | None\n\nmax_packed_sequence_len: int | None\nrope_scaling: Any | None\nnoisy_embedding_alpha: float | None\ndpo_beta: float | None\nevaluation_strategy: str | None\neval_table_size: int | None\neval_max_new_tokens: int | None\ndpo_use_logits_to_keep: bool | None\ndpo_generate_during_eval: bool | None", "crumbs": [ "Getting Started", "Config Reference" @@ -3558,7 +3558,7 @@ "href": "docs/custom_integrations.html#kernels-integration", "title": "Custom Integrations", "section": "Kernels Integration", - "text": "Kernels Integration\nMoE (Mixture of Experts) kernels speed up training for MoE layers and reduce VRAM costs. In transformers v5, batched_mm and grouped_mm were integrated as built-in options via the experts_implementation config kwarg:\nclass ExpertsInterface(GeneralInterface):\n _global_mapping = {\n \"batched_mm\": batched_mm_experts_forward,\n \"grouped_mm\": grouped_mm_experts_forward,\n }\nIn our custom integration, we add support for ScatterMoE, which is even more efficient and faster than grouped_mm.\n\nUsage\nAdd the following to your axolotl YAML config:\nplugins:\n - axolotl.integrations.kernels.KernelsPlugin\n\nuse_kernels: true\nuse_scattermoe: true\nImportant: Setting experts_implementation is incompatible with use_scattermoe.\n\n\nHow It Works\nThe KernelsPlugin runs before model loading and:\n\nRegisters the ScatterMoE kernel from the axolotl-ai-co/scattermoe Hub repo.\nPatches the model’s SparseMoeBlock forward method with the optimized ScatterMoE implementation.\n\nThis works for any MoE model in transformers that uses a SparseMoeBlock class (Mixtral, Qwen2-MoE, OLMoE, etc.).\n\n\nLimitations\nScatterMoE uses a softmax -> topk routing, so results may be different for some model arch as baseline (GPT-OSS, GLM_MOE_DSA).\nScatterMoE does not work for GLM4.7 Flash (glm4_moe_lite) atm.\n\n\nNote on MegaBlocks\nWe tested MegaBlocks but were unable to ensure numerical accuracy, so we did not integrate it. It was also incompatible with many newer model architectures in transformers.\nPlease see reference here", + "text": "Kernels Integration\nMoE (Mixture of Experts) kernels speed up training for MoE layers and reduce VRAM costs. In transformers v5, batched_mm and grouped_mm were integrated as built-in options via the experts_implementation config kwarg:\nclass ExpertsInterface(GeneralInterface):\n _global_mapping = {\n \"batched_mm\": batched_mm_experts_forward,\n \"grouped_mm\": grouped_mm_experts_forward,\n }\nIn our custom integration, we add support for ScatterMoE and SonicMoE, which are more efficient and faster than grouped_mm.\n\nUsage\nAdd the following to your axolotl YAML config:\nplugins:\n - axolotl.integrations.kernels.KernelsPlugin\n\nuse_kernels: true\n\nuse_scattermoe: true\nuse_sonicmoe: true\nImportant: Setting experts_implementation is incompatible with custom kernel options.\n\n\nSonicMoE installation\nPrerequisites:\n- NVIDIA Hopper (H100, H200) or Blackwell (B200, GB200) GPU\n- CUDA 12.9+ (13.0+ for B300)\n- PyTorch 2.7+ (2.9.1 recommended)\n- For B300: Triton 3.6.0\npip install --ignore-requires-python --no-deps \"sonic-moe @ git+https://github.com/Dao-AILab/sonic-moe.git@116e2df0a41874f77fa0ad269ce7df3f0cfcb956\" && pip install nvidia-cutlass-dsl==4.4.0 quack-kernels==0.2.5\nSee the SonicMoE installation guide for the latest prerequisite details.\nNote: Blackwell support is in upstream beta. On Blackwell GPUs, Axolotl automatically sets USE_QUACK_GEMM=1 to enable the Blackwell kernels.\n\n\nHow It Works\nThe KernelsPlugin runs before model loading and:\n\n\nScatterMoE\n\nRegisters the ScatterMoE kernel from the local libs/scattermoe_lora package (includes fused LoRA support via Triton kernels).\nPatches the model’s SparseMoeBlock forward method with the optimized ScatterMoE implementation.\n\n\n\nSonicMoE\n\nResolves the model’s MoE block class(es) from constants.py.\nPatches the forward method with SonicMoE’s optimized kernels and registers a weight converter for the interleaved gate/up projection format.\nSupports both softmax->topk and sigmoid->topk routing strategies.\n\nBoth paths use the shared resolve_moe_block_classes utility in constants.py for model-type-to-class resolution.\n\nSupported Models\nSee constants.py for the full list of supported model types (Qwen2-MoE, Qwen3-MoE, OLMoE, Mixtral, DeepSeek-V3, GLM-MoE, MiniMax, etc.).\n\n\n\nLimitations\nScatterMoE uses a softmax -> topk routing, so results may be different for some model architectures as baseline (GPT-OSS, etc). Incompatible with GLM_MOE_DSA (GLM 5) and GLM4_MOE_LITE (GLM 4.7 Flash) at the moment.\nSonicMoE supports both softmax->topk and sigmoid->topk routing, covering a wider range of architectures.\nScatterMoE does not work for GLM4.7 Flash (glm4_moe_lite) atm.\n\n\nNote on MegaBlocks\nWe tested MegaBlocks but were unable to ensure numerical accuracy, so we did not integrate it. It was also incompatible with many newer model architectures in transformers.\nPlease see reference here", "crumbs": [ "Advanced Features", "Custom Integrations" diff --git a/sitemap.xml b/sitemap.xml index b8e84377c..b0c9b29db 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -2,950 +2,950 @@ https://docs.axolotl.ai/examples/colab-notebooks/colab-axolotl-example.html - 2026-03-05T18:46:03.604Z + 2026-03-06T14:12:13.551Z https://docs.axolotl.ai/src/axolotl/integrations/cut_cross_entropy/ACKNOWLEDGEMENTS.html - 2026-03-05T18:46:03.625Z + 2026-03-06T14:12:13.568Z https://docs.axolotl.ai/docs/inference.html - 2026-03-05T18:46:03.598Z + 2026-03-06T14:12:13.546Z https://docs.axolotl.ai/docs/expert_quantization.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/installation.html - 2026-03-05T18:46:03.598Z + 2026-03-06T14:12:13.546Z https://docs.axolotl.ai/docs/models/ministral3/think.html - 2026-03-05T18:49:59.410Z + 2026-03-06T14:16:29.127Z https://docs.axolotl.ai/docs/models/granite4.html - 2026-03-05T18:49:59.418Z + 2026-03-06T14:16:29.133Z https://docs.axolotl.ai/docs/models/seed-oss.html - 2026-03-05T18:49:59.417Z + 2026-03-06T14:16:29.132Z https://docs.axolotl.ai/docs/models/orpheus.html - 2026-03-05T18:49:59.420Z + 2026-03-06T14:16:29.134Z https://docs.axolotl.ai/docs/models/internvl3_5.html - 2026-03-05T18:49:59.407Z + 2026-03-06T14:16:29.125Z https://docs.axolotl.ai/docs/models/magistral/vision.html - 2026-03-05T18:49:59.411Z + 2026-03-06T14:16:29.129Z https://docs.axolotl.ai/docs/models/mimo.html - 2026-03-05T18:49:59.406Z + 2026-03-06T14:16:29.124Z https://docs.axolotl.ai/docs/models/gpt-oss.html - 2026-03-05T18:49:59.416Z + 2026-03-06T14:16:29.132Z https://docs.axolotl.ai/docs/models/qwen3-next.html - 2026-03-05T18:49:59.414Z + 2026-03-06T14:16:29.131Z https://docs.axolotl.ai/docs/models/llama-2.html - 2026-03-05T18:49:59.414Z + 2026-03-06T14:16:29.130Z https://docs.axolotl.ai/docs/models/kimi-linear.html - 2026-03-05T18:49:59.406Z + 2026-03-06T14:16:29.123Z https://docs.axolotl.ai/docs/models/smolvlm2.html - 2026-03-05T18:49:59.418Z + 2026-03-06T14:16:29.132Z https://docs.axolotl.ai/docs/models/olmo3.html - 2026-03-05T18:49:59.408Z + 2026-03-06T14:16:29.125Z https://docs.axolotl.ai/docs/models/jamba.html - 2026-03-05T18:49:59.420Z + 2026-03-06T14:16:29.133Z https://docs.axolotl.ai/docs/models/mistral-small.html - 2026-03-05T18:49:59.412Z + 2026-03-06T14:16:29.129Z https://docs.axolotl.ai/docs/models/devstral.html - 2026-03-05T18:49:59.413Z + 2026-03-06T14:16:29.130Z https://docs.axolotl.ai/docs/models/index.html - 2026-03-05T18:49:59.420Z + 2026-03-06T14:16:29.134Z https://docs.axolotl.ai/docs/lora_optims.html - 2026-03-05T18:46:03.598Z + 2026-03-06T14:12:13.546Z https://docs.axolotl.ai/docs/cli.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/gradient_checkpointing.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/dataset_preprocessing.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/docker.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/attention.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/api/prompt_strategies.input_output.html - 2026-03-05T18:49:36.068Z + 2026-03-06T14:16:09.792Z https://docs.axolotl.ai/docs/api/loaders.adapter.html - 2026-03-05T18:49:35.874Z + 2026-03-06T14:16:09.619Z https://docs.axolotl.ai/docs/api/monkeypatch.btlm_attn_hijack_flash.html - 2026-03-05T18:49:36.409Z + 2026-03-06T14:16:10.095Z https://docs.axolotl.ai/docs/api/monkeypatch.llama_patch_multipack.html - 2026-03-05T18:49:36.411Z + 2026-03-06T14:16:10.096Z https://docs.axolotl.ai/docs/api/core.trainers.utils.html - 2026-03-05T18:49:35.842Z + 2026-03-06T14:16:09.591Z https://docs.axolotl.ai/docs/api/utils.data.sft.html - 2026-03-05T18:49:36.630Z + 2026-03-06T14:16:10.292Z https://docs.axolotl.ai/docs/api/utils.schemas.peft.html - 2026-03-05T18:49:36.723Z + 2026-03-06T14:16:10.377Z https://docs.axolotl.ai/docs/api/monkeypatch.lora_kernels.html - 2026-03-05T18:49:36.397Z + 2026-03-06T14:16:10.083Z https://docs.axolotl.ai/docs/api/utils.collators.mamba.html - 2026-03-05T18:49:37.028Z + 2026-03-06T14:16:10.649Z https://docs.axolotl.ai/docs/api/prompt_strategies.base.html - 2026-03-05T18:49:35.949Z + 2026-03-06T14:16:09.686Z https://docs.axolotl.ai/docs/api/loaders.processor.html - 2026-03-05T18:49:35.867Z + 2026-03-06T14:16:09.613Z https://docs.axolotl.ai/docs/api/core.training_args.html - 2026-03-05T18:49:35.446Z + 2026-03-06T14:16:09.237Z https://docs.axolotl.ai/docs/api/loaders.tokenizer.html - 2026-03-05T18:49:35.865Z + 2026-03-06T14:16:09.611Z https://docs.axolotl.ai/docs/api/prompt_strategies.completion.html - 2026-03-05T18:49:36.060Z + 2026-03-06T14:16:09.785Z https://docs.axolotl.ai/docs/api/loaders.constants.html - 2026-03-05T18:49:35.898Z + 2026-03-06T14:16:09.641Z https://docs.axolotl.ai/docs/api/cli.utils.train.html - 2026-03-05T18:49:35.759Z + 2026-03-06T14:16:09.516Z https://docs.axolotl.ai/docs/api/monkeypatch.stablelm_attn_hijack_flash.html - 2026-03-05T18:49:36.419Z + 2026-03-06T14:16:10.103Z https://docs.axolotl.ai/docs/api/utils.schemas.datasets.html - 2026-03-05T18:49:36.712Z + 2026-03-06T14:16:10.367Z https://docs.axolotl.ai/docs/api/prompt_strategies.llama2_chat.html - 2026-03-05T18:49:36.052Z + 2026-03-06T14:16:09.778Z https://docs.axolotl.ai/docs/api/monkeypatch.llama_attn_hijack_xformers.html - 2026-03-05T18:49:36.351Z + 2026-03-06T14:16:10.042Z https://docs.axolotl.ai/docs/api/common.datasets.html - 2026-03-05T18:49:36.997Z + 2026-03-06T14:16:10.621Z https://docs.axolotl.ai/docs/api/logging_config.html - 2026-03-05T18:49:35.410Z + 2026-03-06T14:16:09.205Z https://docs.axolotl.ai/docs/api/monkeypatch.unsloth_.html - 2026-03-05T18:49:36.433Z + 2026-03-06T14:16:10.115Z https://docs.axolotl.ai/docs/api/integrations.liger.args.html - 2026-03-05T18:49:36.966Z + 2026-03-06T14:16:10.593Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.llama3.html - 2026-03-05T18:49:36.123Z + 2026-03-06T14:16:09.840Z https://docs.axolotl.ai/docs/api/core.trainers.grpo.sampler.html - 2026-03-05T18:49:35.840Z + 2026-03-06T14:16:09.589Z https://docs.axolotl.ai/docs/api/core.chat.messages.html - 2026-03-05T18:49:35.475Z + 2026-03-06T14:16:09.263Z https://docs.axolotl.ai/docs/api/integrations.base.html - 2026-03-05T18:49:36.946Z + 2026-03-06T14:16:10.575Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.chat_template.html - 2026-03-05T18:49:36.110Z + 2026-03-06T14:16:09.829Z https://docs.axolotl.ai/docs/api/train.html - 2026-03-05T18:49:35.306Z + 2026-03-06T14:16:09.113Z https://docs.axolotl.ai/docs/api/utils.distributed.html - 2026-03-05T18:49:36.604Z + 2026-03-06T14:16:10.269Z https://docs.axolotl.ai/docs/api/core.builders.causal.html - 2026-03-05T18:49:35.424Z + 2026-03-06T14:16:09.217Z https://docs.axolotl.ai/docs/api/core.builders.rl.html - 2026-03-05T18:49:35.430Z + 2026-03-06T14:16:09.222Z https://docs.axolotl.ai/docs/api/utils.collators.core.html - 2026-03-05T18:49:37.000Z + 2026-03-06T14:16:10.624Z https://docs.axolotl.ai/docs/api/utils.schemas.model.html - 2026-03-05T18:49:36.680Z + 2026-03-06T14:16:10.339Z https://docs.axolotl.ai/docs/api/kernels.quantize.html - 2026-03-05T18:49:36.340Z + 2026-03-06T14:16:10.033Z https://docs.axolotl.ai/docs/api/utils.schemas.enums.html - 2026-03-05T18:49:36.765Z + 2026-03-06T14:16:10.415Z https://docs.axolotl.ai/docs/api/prompt_strategies.metharme.html - 2026-03-05T18:49:36.083Z + 2026-03-06T14:16:09.804Z https://docs.axolotl.ai/docs/api/utils.callbacks.lisa.html - 2026-03-05T18:49:37.099Z + 2026-03-06T14:16:10.712Z https://docs.axolotl.ai/docs/api/cli.preprocess.html - 2026-03-05T18:49:35.679Z + 2026-03-06T14:16:09.445Z https://docs.axolotl.ai/docs/api/loaders.model.html - 2026-03-05T18:49:35.854Z + 2026-03-06T14:16:09.602Z https://docs.axolotl.ai/docs/api/cli.merge_lora.html - 2026-03-05T18:49:35.654Z + 2026-03-06T14:16:09.423Z https://docs.axolotl.ai/docs/api/cli.utils.sweeps.html - 2026-03-05T18:49:35.744Z + 2026-03-06T14:16:09.503Z https://docs.axolotl.ai/docs/api/utils.bench.html - 2026-03-05T18:49:36.512Z + 2026-03-06T14:16:10.187Z https://docs.axolotl.ai/docs/api/core.trainers.mamba.html - 2026-03-05T18:49:35.803Z + 2026-03-06T14:16:09.555Z https://docs.axolotl.ai/docs/api/cli.vllm_serve.html - 2026-03-05T18:49:35.693Z + 2026-03-06T14:16:09.459Z https://docs.axolotl.ai/docs/api/kernels.utils.html - 2026-03-05T18:49:36.342Z + 2026-03-06T14:16:10.034Z https://docs.axolotl.ai/docs/api/prompt_strategies.chat_template.html - 2026-03-05T18:49:35.992Z + 2026-03-06T14:16:09.724Z https://docs.axolotl.ai/docs/api/utils.callbacks.perplexity.html - 2026-03-05T18:49:37.092Z + 2026-03-06T14:16:10.706Z https://docs.axolotl.ai/docs/api/utils.collators.mm_chat.html - 2026-03-05T18:49:37.034Z + 2026-03-06T14:16:10.654Z https://docs.axolotl.ai/docs/api/utils.schemas.integrations.html - 2026-03-05T18:49:36.754Z + 2026-03-06T14:16:10.405Z https://docs.axolotl.ai/docs/api/prompt_strategies.orpo.chat_template.html - 2026-03-05T18:49:36.192Z + 2026-03-06T14:16:09.901Z https://docs.axolotl.ai/docs/api/prompt_strategies.alpaca_w_system.html - 2026-03-05T18:49:36.026Z + 2026-03-06T14:16:09.754Z https://docs.axolotl.ai/docs/api/cli.evaluate.html - 2026-03-05T18:49:35.560Z + 2026-03-06T14:16:09.339Z https://docs.axolotl.ai/docs/api/core.datasets.chat.html - 2026-03-05T18:49:35.487Z + 2026-03-06T14:16:09.274Z https://docs.axolotl.ai/docs/api/prompt_strategies.alpaca_chat.html - 2026-03-05T18:49:36.009Z + 2026-03-06T14:16:09.739Z https://docs.axolotl.ai/docs/api/prompt_strategies.orcamini.html - 2026-03-05T18:49:36.087Z + 2026-03-06T14:16:09.809Z https://docs.axolotl.ai/docs/api/monkeypatch.transformers_fa_utils.html - 2026-03-05T18:49:36.431Z + 2026-03-06T14:16:10.114Z https://docs.axolotl.ai/docs/api/kernels.lora.html - 2026-03-05T18:49:36.305Z + 2026-03-06T14:16:10.002Z https://docs.axolotl.ai/docs/api/utils.callbacks.profiler.html - 2026-03-05T18:49:37.097Z + 2026-03-06T14:16:10.711Z https://docs.axolotl.ai/docs/api/utils.callbacks.mlflow_.html - 2026-03-05T18:49:37.103Z + 2026-03-06T14:16:10.716Z https://docs.axolotl.ai/docs/api/utils.freeze.html - 2026-03-05T18:49:36.522Z + 2026-03-06T14:16:10.195Z https://docs.axolotl.ai/docs/api/integrations.kd.trainer.html - 2026-03-05T18:49:36.962Z + 2026-03-06T14:16:10.589Z https://docs.axolotl.ai/docs/api/monkeypatch.gradient_checkpointing.offload_disk.html - 2026-03-05T18:49:36.483Z + 2026-03-06T14:16:10.161Z https://docs.axolotl.ai/docs/api/utils.data.streaming.html - 2026-03-05T18:49:36.622Z + 2026-03-06T14:16:10.285Z https://docs.axolotl.ai/docs/api/prompt_tokenizers.html - 2026-03-05T18:49:35.398Z + 2026-03-06T14:16:09.194Z https://docs.axolotl.ai/docs/api/core.trainers.mixins.rng_state_loader.html - 2026-03-05T18:49:35.909Z + 2026-03-06T14:16:09.651Z https://docs.axolotl.ai/docs/api/cli.cloud.modal_.html - 2026-03-05T18:49:35.706Z + 2026-03-06T14:16:09.470Z https://docs.axolotl.ai/docs/api/core.trainers.mixins.scheduler.html - 2026-03-05T18:49:35.918Z + 2026-03-06T14:16:09.659Z https://docs.axolotl.ai/docs/api/convert.html - 2026-03-05T18:49:35.345Z + 2026-03-06T14:16:09.147Z https://docs.axolotl.ai/docs/api/models.mamba.modeling_mamba.html - 2026-03-05T18:49:36.998Z + 2026-03-06T14:16:10.622Z https://docs.axolotl.ai/docs/api/cli.args.html - 2026-03-05T18:49:35.585Z + 2026-03-06T14:16:09.362Z https://docs.axolotl.ai/docs/api/core.chat.format.shared.html - 2026-03-05T18:49:35.481Z + 2026-03-06T14:16:09.268Z https://docs.axolotl.ai/docs/api/prompt_strategies.bradley_terry.llama3.html - 2026-03-05T18:49:36.196Z + 2026-03-06T14:16:09.905Z https://docs.axolotl.ai/docs/api/index.html - 2026-03-05T18:49:35.228Z + 2026-03-06T14:16:09.044Z https://docs.axolotl.ai/docs/fsdp_qlora.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/dataset-formats/stepwise_supervised.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/dataset-formats/template_free.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/dataset-formats/index.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/telemetry.html - 2026-03-05T18:46:03.600Z + 2026-03-06T14:12:13.548Z https://docs.axolotl.ai/docs/config-reference.html - 2026-03-05T18:49:58.366Z + 2026-03-06T14:16:28.169Z https://docs.axolotl.ai/docs/ray-integration.html - 2026-03-05T18:46:03.599Z + 2026-03-06T14:12:13.547Z https://docs.axolotl.ai/docs/streaming.html - 2026-03-05T18:46:03.600Z + 2026-03-06T14:12:13.548Z https://docs.axolotl.ai/docs/sequence_parallelism.html - 2026-03-05T18:46:03.600Z + 2026-03-06T14:12:13.548Z https://docs.axolotl.ai/docs/unsloth.html - 2026-03-05T18:46:03.600Z + 2026-03-06T14:12:13.548Z https://docs.axolotl.ai/docs/mixed_precision.html - 2026-03-05T18:46:03.598Z + 2026-03-06T14:12:13.546Z https://docs.axolotl.ai/docs/amd_hpc.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/lr_groups.html - 2026-03-05T18:46:03.598Z + 2026-03-06T14:12:13.546Z https://docs.axolotl.ai/docs/optimizations.html - 2026-03-05T18:46:03.599Z + 2026-03-06T14:12:13.547Z https://docs.axolotl.ai/docs/mac.html - 2026-03-05T18:46:03.598Z + 2026-03-06T14:12:13.546Z https://docs.axolotl.ai/index.html - 2026-03-05T18:46:03.620Z + 2026-03-06T14:12:13.564Z https://docs.axolotl.ai/docs/optimizers.html - 2026-03-05T18:46:03.599Z + 2026-03-06T14:12:13.547Z https://docs.axolotl.ai/docs/getting-started.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/multi-node.html - 2026-03-05T18:46:03.598Z + 2026-03-06T14:12:13.547Z https://docs.axolotl.ai/docs/input_output.html - 2026-03-05T18:46:03.598Z + 2026-03-06T14:12:13.546Z https://docs.axolotl.ai/docs/nd_parallelism.html - 2026-03-05T18:46:03.599Z + 2026-03-06T14:12:13.547Z https://docs.axolotl.ai/docs/dataset_loading.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/quantize.html - 2026-03-05T18:46:03.599Z + 2026-03-06T14:12:13.547Z https://docs.axolotl.ai/docs/rlhf.html - 2026-03-05T18:46:03.599Z + 2026-03-06T14:12:13.547Z https://docs.axolotl.ai/docs/custom_integrations.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/qat.html - 2026-03-05T18:46:03.599Z + 2026-03-06T14:12:13.547Z https://docs.axolotl.ai/docs/checkpoint_saving.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/dataset-formats/conversation.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/dataset-formats/inst_tune.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/dataset-formats/tokenized.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/dataset-formats/pretraining.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/api/cli.main.html - 2026-03-05T18:49:35.539Z + 2026-03-06T14:16:09.320Z https://docs.axolotl.ai/docs/api/utils.schemas.trl.html - 2026-03-05T18:49:36.727Z + 2026-03-06T14:16:10.381Z https://docs.axolotl.ai/docs/api/core.datasets.transforms.chat_builder.html - 2026-03-05T18:49:35.497Z + 2026-03-06T14:16:09.283Z https://docs.axolotl.ai/docs/api/common.const.html - 2026-03-05T18:49:36.978Z + 2026-03-06T14:16:10.604Z https://docs.axolotl.ai/docs/api/cli.utils.load.html - 2026-03-05T18:49:35.736Z + 2026-03-06T14:16:09.496Z https://docs.axolotl.ai/docs/api/loaders.patch_manager.html - 2026-03-05T18:49:35.896Z + 2026-03-06T14:16:09.639Z https://docs.axolotl.ai/docs/api/utils.quantization.html - 2026-03-05T18:49:36.655Z + 2026-03-06T14:16:10.314Z https://docs.axolotl.ai/docs/api/monkeypatch.utils.html - 2026-03-05T18:49:36.407Z + 2026-03-06T14:16:10.093Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.user_defined.html - 2026-03-05T18:49:36.140Z + 2026-03-06T14:16:09.856Z https://docs.axolotl.ai/docs/api/cli.quantize.html - 2026-03-05T18:49:35.685Z + 2026-03-06T14:16:09.451Z https://docs.axolotl.ai/docs/api/prompt_strategies.user_defined.html - 2026-03-05T18:49:36.036Z + 2026-03-06T14:16:09.764Z https://docs.axolotl.ai/docs/api/integrations.lm_eval.args.html - 2026-03-05T18:49:36.970Z + 2026-03-06T14:16:10.597Z https://docs.axolotl.ai/docs/api/monkeypatch.gradient_checkpointing.offload_cpu.html - 2026-03-05T18:49:36.450Z + 2026-03-06T14:16:10.131Z https://docs.axolotl.ai/docs/api/utils.schedulers.html - 2026-03-05T18:49:36.578Z + 2026-03-06T14:16:10.246Z https://docs.axolotl.ai/docs/api/kernels.geglu.html - 2026-03-05T18:49:36.318Z + 2026-03-06T14:16:10.013Z https://docs.axolotl.ai/docs/api/monkeypatch.trainer_fsdp_optim.html - 2026-03-05T18:49:36.423Z + 2026-03-06T14:16:10.107Z https://docs.axolotl.ai/docs/api/prompt_strategies.pygmalion.html - 2026-03-05T18:49:36.096Z + 2026-03-06T14:16:09.816Z https://docs.axolotl.ai/docs/api/common.architectures.html - 2026-03-05T18:49:36.976Z + 2026-03-06T14:16:10.602Z https://docs.axolotl.ai/docs/api/cli.utils.html - 2026-03-05T18:49:35.708Z + 2026-03-06T14:16:09.471Z https://docs.axolotl.ai/docs/api/monkeypatch.llama_expand_mask.html - 2026-03-05T18:49:36.361Z + 2026-03-06T14:16:10.052Z https://docs.axolotl.ai/docs/api/monkeypatch.llama_attn_hijack_flash.html - 2026-03-05T18:49:36.349Z + 2026-03-06T14:16:10.041Z https://docs.axolotl.ai/docs/api/prompt_strategies.kto.user_defined.html - 2026-03-05T18:49:36.165Z + 2026-03-06T14:16:09.878Z https://docs.axolotl.ai/docs/api/cli.checks.html - 2026-03-05T18:49:35.597Z + 2026-03-06T14:16:09.373Z https://docs.axolotl.ai/docs/api/core.trainers.trl.html - 2026-03-05T18:49:35.796Z + 2026-03-06T14:16:09.549Z https://docs.axolotl.ai/docs/api/utils.tokenization.html - 2026-03-05T18:49:36.492Z + 2026-03-06T14:16:10.169Z https://docs.axolotl.ai/docs/api/cli.utils.fetch.html - 2026-03-05T18:49:35.729Z + 2026-03-06T14:16:09.490Z https://docs.axolotl.ai/docs/api/core.builders.base.html - 2026-03-05T18:49:35.418Z + 2026-03-06T14:16:09.212Z https://docs.axolotl.ai/docs/api/monkeypatch.mistral_attn_hijack_flash.html - 2026-03-05T18:49:36.353Z + 2026-03-06T14:16:10.044Z https://docs.axolotl.ai/docs/api/utils.trainer.html - 2026-03-05T18:49:36.544Z + 2026-03-06T14:16:10.215Z https://docs.axolotl.ai/docs/api/cli.train.html - 2026-03-05T18:49:35.549Z + 2026-03-06T14:16:09.330Z https://docs.axolotl.ai/docs/api/core.chat.format.llama3x.html - 2026-03-05T18:49:35.479Z + 2026-03-06T14:16:09.266Z https://docs.axolotl.ai/docs/api/utils.lora.html - 2026-03-05T18:49:36.500Z + 2026-03-06T14:16:10.176Z https://docs.axolotl.ai/docs/api/utils.schemas.multimodal.html - 2026-03-05T18:49:36.734Z + 2026-03-06T14:16:10.387Z https://docs.axolotl.ai/docs/api/monkeypatch.mixtral.html - 2026-03-05T18:49:36.446Z + 2026-03-06T14:16:10.127Z https://docs.axolotl.ai/docs/api/core.trainers.dpo.trainer.html - 2026-03-05T18:49:35.811Z + 2026-03-06T14:16:09.563Z https://docs.axolotl.ai/docs/api/utils.samplers.multipack.html - 2026-03-05T18:49:37.084Z + 2026-03-06T14:16:10.699Z https://docs.axolotl.ai/docs/api/core.trainers.grpo.trainer.html - 2026-03-05T18:49:35.825Z + 2026-03-06T14:16:09.575Z https://docs.axolotl.ai/docs/api/utils.ctx_managers.sequence_parallel.html - 2026-03-05T18:49:35.947Z + 2026-03-06T14:16:09.685Z https://docs.axolotl.ai/docs/api/prompt_strategies.kto.chatml.html - 2026-03-05T18:49:36.163Z + 2026-03-06T14:16:09.876Z https://docs.axolotl.ai/docs/api/prompt_strategies.kto.llama3.html - 2026-03-05T18:49:36.153Z + 2026-03-06T14:16:09.867Z https://docs.axolotl.ai/docs/api/datasets.html - 2026-03-05T18:49:35.328Z + 2026-03-06T14:16:09.131Z https://docs.axolotl.ai/docs/api/integrations.grokfast.optimizer.html - 2026-03-05T18:49:36.952Z + 2026-03-06T14:16:10.580Z https://docs.axolotl.ai/docs/api/cli.art.html - 2026-03-05T18:49:35.589Z + 2026-03-06T14:16:09.366Z https://docs.axolotl.ai/docs/api/utils.callbacks.qat.html - 2026-03-05T18:49:37.116Z + 2026-03-06T14:16:10.728Z https://docs.axolotl.ai/docs/api/monkeypatch.relora.html - 2026-03-05T18:49:36.360Z + 2026-03-06T14:16:10.050Z https://docs.axolotl.ai/docs/api/prompt_strategies.messages.chat.html - 2026-03-05T18:49:36.101Z + 2026-03-06T14:16:09.821Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.passthrough.html - 2026-03-05T18:49:36.142Z + 2026-03-06T14:16:09.857Z https://docs.axolotl.ai/docs/api/core.chat.format.chatml.html - 2026-03-05T18:49:35.477Z + 2026-03-06T14:16:09.265Z https://docs.axolotl.ai/docs/api/cli.utils.args.html - 2026-03-05T18:49:35.722Z + 2026-03-06T14:16:09.484Z https://docs.axolotl.ai/docs/api/core.trainers.base.html - 2026-03-05T18:49:35.777Z + 2026-03-06T14:16:09.532Z https://docs.axolotl.ai/docs/api/utils.schemas.training.html - 2026-03-05T18:49:36.689Z + 2026-03-06T14:16:10.347Z https://docs.axolotl.ai/docs/api/evaluate.html - 2026-03-05T18:49:35.320Z + 2026-03-06T14:16:09.124Z https://docs.axolotl.ai/docs/api/cli.config.html - 2026-03-05T18:49:35.619Z + 2026-03-06T14:16:09.392Z https://docs.axolotl.ai/docs/api/integrations.spectrum.args.html - 2026-03-05T18:49:36.974Z + 2026-03-06T14:16:10.601Z https://docs.axolotl.ai/docs/api/prompt_strategies.stepwise_supervised.html - 2026-03-05T18:49:36.074Z + 2026-03-06T14:16:09.797Z https://docs.axolotl.ai/docs/api/utils.collators.batching.html - 2026-03-05T18:49:37.023Z + 2026-03-06T14:16:10.644Z https://docs.axolotl.ai/docs/api/cli.cloud.base.html - 2026-03-05T18:49:35.698Z + 2026-03-06T14:16:09.463Z https://docs.axolotl.ai/docs/api/utils.schemas.config.html - 2026-03-05T18:49:36.671Z + 2026-03-06T14:16:10.331Z https://docs.axolotl.ai/docs/api/monkeypatch.multipack.html - 2026-03-05T18:49:36.355Z + 2026-03-06T14:16:10.046Z https://docs.axolotl.ai/docs/api/utils.dict.html - 2026-03-05T18:49:36.611Z + 2026-03-06T14:16:10.275Z https://docs.axolotl.ai/docs/api/integrations.cut_cross_entropy.args.html - 2026-03-05T18:49:36.951Z + 2026-03-06T14:16:10.579Z https://docs.axolotl.ai/docs/api/cli.delinearize_llama4.html - 2026-03-05T18:49:35.626Z + 2026-03-06T14:16:09.398Z https://docs.axolotl.ai/docs/api/cli.merge_sharded_fsdp_weights.html - 2026-03-05T18:49:35.668Z + 2026-03-06T14:16:09.436Z https://docs.axolotl.ai/docs/api/utils.model_shard_quant.html - 2026-03-05T18:49:36.507Z + 2026-03-06T14:16:10.183Z https://docs.axolotl.ai/docs/api/monkeypatch.data.batch_dataset_fetcher.html - 2026-03-05T18:49:36.444Z + 2026-03-06T14:16:10.126Z https://docs.axolotl.ai/docs/api/utils.schemas.utils.html - 2026-03-05T18:49:36.772Z + 2026-03-06T14:16:10.421Z https://docs.axolotl.ai/docs/api/utils.callbacks.comet_.html - 2026-03-05T18:49:37.108Z + 2026-03-06T14:16:10.720Z https://docs.axolotl.ai/docs/api/utils.optimizers.adopt.html - 2026-03-05T18:49:36.620Z + 2026-03-06T14:16:10.284Z https://docs.axolotl.ai/docs/api/kernels.swiglu.html - 2026-03-05T18:49:36.331Z + 2026-03-06T14:16:10.025Z https://docs.axolotl.ai/docs/api/core.trainers.mixins.optimizer.html - 2026-03-05T18:49:35.905Z + 2026-03-06T14:16:09.647Z https://docs.axolotl.ai/docs/api/cli.inference.html - 2026-03-05T18:49:35.643Z + 2026-03-06T14:16:09.414Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.chatml.html - 2026-03-05T18:49:36.136Z + 2026-03-06T14:16:09.852Z https://docs.axolotl.ai/docs/api/utils.chat_templates.html - 2026-03-05T18:49:36.494Z + 2026-03-06T14:16:10.171Z https://docs.axolotl.ai/docs/api/prompt_strategies.alpaca_instruct.html - 2026-03-05T18:49:36.011Z + 2026-03-06T14:16:09.741Z https://docs.axolotl.ai/docs/api/prompt_strategies.dpo.zephyr.html - 2026-03-05T18:49:36.138Z + 2026-03-06T14:16:09.854Z https://docs.axolotl.ai/docs/multipack.html - 2026-03-05T18:46:03.599Z + 2026-03-06T14:12:13.547Z https://docs.axolotl.ai/docs/torchao.html - 2026-03-05T18:46:03.600Z + 2026-03-06T14:12:13.548Z https://docs.axolotl.ai/docs/reward_modelling.html - 2026-03-05T18:46:03.599Z + 2026-03-06T14:12:13.547Z https://docs.axolotl.ai/docs/nccl.html - 2026-03-05T18:46:03.599Z + 2026-03-06T14:12:13.547Z https://docs.axolotl.ai/docs/multi-gpu.html - 2026-03-05T18:46:03.598Z + 2026-03-06T14:12:13.547Z https://docs.axolotl.ai/docs/batch_vs_grad.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/multimodal.html - 2026-03-05T18:46:03.599Z + 2026-03-06T14:12:13.547Z https://docs.axolotl.ai/docs/models/LiquidAI.html - 2026-03-05T18:49:59.419Z + 2026-03-06T14:16:29.133Z https://docs.axolotl.ai/docs/models/mistral.html - 2026-03-05T18:49:59.413Z + 2026-03-06T14:16:29.130Z https://docs.axolotl.ai/docs/models/trinity.html - 2026-03-05T18:49:59.408Z + 2026-03-06T14:16:29.125Z https://docs.axolotl.ai/docs/models/hunyuan.html - 2026-03-05T18:49:59.419Z + 2026-03-06T14:16:29.133Z https://docs.axolotl.ai/docs/models/phi.html - 2026-03-05T18:49:59.417Z + 2026-03-06T14:16:29.132Z https://docs.axolotl.ai/docs/models/apertus.html - 2026-03-05T18:49:59.416Z + 2026-03-06T14:16:29.131Z https://docs.axolotl.ai/docs/models/plano.html - 2026-03-05T18:49:59.406Z + 2026-03-06T14:16:29.124Z https://docs.axolotl.ai/docs/models/gemma3n.html - 2026-03-05T18:49:59.415Z + 2026-03-06T14:16:29.131Z https://docs.axolotl.ai/docs/models/arcee.html - 2026-03-05T18:49:59.408Z + 2026-03-06T14:16:29.126Z https://docs.axolotl.ai/docs/models/ministral3.html - 2026-03-05T18:49:59.409Z + 2026-03-06T14:16:29.127Z https://docs.axolotl.ai/docs/models/magistral/think.html - 2026-03-05T18:49:59.411Z + 2026-03-06T14:16:29.128Z https://docs.axolotl.ai/docs/models/llama-4.html - 2026-03-05T18:49:59.414Z + 2026-03-06T14:16:29.130Z https://docs.axolotl.ai/docs/models/voxtral.html - 2026-03-05T18:49:59.413Z + 2026-03-06T14:16:29.129Z https://docs.axolotl.ai/docs/models/magistral.html - 2026-03-05T18:49:59.411Z + 2026-03-06T14:16:29.128Z https://docs.axolotl.ai/docs/models/qwen3.html - 2026-03-05T18:49:59.415Z + 2026-03-06T14:16:29.131Z https://docs.axolotl.ai/docs/models/ministral.html - 2026-03-05T18:49:59.412Z + 2026-03-06T14:16:29.129Z https://docs.axolotl.ai/docs/models/ministral3/vision.html - 2026-03-05T18:49:59.410Z + 2026-03-06T14:16:29.128Z https://docs.axolotl.ai/docs/debugging.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/docs/faq.html - 2026-03-05T18:46:03.595Z + 2026-03-06T14:12:13.543Z https://docs.axolotl.ai/src/axolotl/integrations/LICENSE.html - 2026-03-05T18:46:03.625Z + 2026-03-06T14:12:13.568Z https://docs.axolotl.ai/FAQS.html - 2026-03-05T18:46:03.593Z + 2026-03-06T14:12:13.541Z