tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

increase hyperparams_count for gradients for added normalize_topk

Browse Source
This commit is contained in:
Wing Lian
2025-05-31 08:42:06 -04:00
parent d55a51623f
commit a8e2bddd19
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/axolotl/integrations/kd/kernels/liger.py
View File

@@ -337,7 +337,7 @@ class LigerFusedLinearKLTopKLogprobFunction(LigerFusedLinearDistillationBase):
ctx.save_for_backward(grad_inputs_combined, grad_weight_acc, grad_bias_acc) ctx.save_for_backward(grad_inputs_combined, grad_weight_acc, grad_bias_acc)
# For matching None returns in backward for non-tensor/non-grad_requiring inputs # For matching None returns in backward for non-tensor/non-grad_requiring inputs
ctx.hyperparams_count = 8 # Corresponds to number of hyperparams after main tensors in fwd signature ctx.hyperparams_count = 9 # Corresponds to number of hyperparams after main tensors in fwd signature
ctx.bias_was_none = student_lm_head_bias is None ctx.bias_was_none = student_lm_head_bias is None
ctx.orig_dims = (B, N, D, K) ctx.orig_dims = (B, N, D, K)