feat: add wip fft offload config

2025-08-07 16:14:11 +07:00
parent b2a8c37a27
commit 7e83268662
2 changed files with 77 additions and 1 deletions
--- a/examples/glm4/glm4.5-qlora.yaml
+++ b/examples/glm4/glm4.5-qlora.yaml
@@ -2,6 +2,9 @@ base_model: zai-org/GLM-4.5-Air
 # Automatically upload checkpoint and final model to HF
 # hub_model_id: username/custom_model_name

+plugins:
+  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
+
 load_in_4bit: true

 datasets:
@@ -40,7 +43,7 @@ wandb_log_model:
 gradient_accumulation_steps: 2
 micro_batch_size: 2
 num_epochs: 1
-optimizer: adamw_8bit
+optimizer: adamw_torch_8bit
 lr_scheduler: cosine
 learning_rate: 0.0002