add back in reinit_weights (clobbered?); masking / pretrain fixes

2025-08-15 02:21:25 +00:00
parent 479a454ae3
commit e19be0c2d9
5 changed files with 139 additions and 111 deletions
--- a/examples/llama-3/diffusion-3.2-1b-pretrain.yaml
+++ b/examples/llama-3/diffusion-3.2-1b-pretrain.yaml
@@ -34,7 +34,7 @@ lr_scheduler: cosine
 learning_rate: 3e-4

 bf16: auto
-tf32: false
+tf32: true

 gradient_checkpointing: true
 resume_from_checkpoint:
@@ -51,8 +51,8 @@ eval_steps: 1000
 special_tokens:
  pad_token: "<|end_of_text|>"

-wandb_project: diffusion-plugin
-wandb_entity: axolotl-ai
+wandb_project:
+wandb_entity:
 wandb_watch:
 wandb_name:
 wandb_log_model: