tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

inline comment change

Browse Source
This commit is contained in:
Dan Saunders
2025-01-14 16:59:43 +00:00
parent fd8ad6fcbf
commit 28694219a5
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/axolotl/integrations/diff_transformer/diff_attn.py
View File

@@ -384,7 +384,7 @@ class LlamaDifferentialAttentionBase(nn.Module):
""" """
attn = self.subln(attn) attn = self.subln(attn)
# NOTE: this may need to be added back in, but doesn't interact well with # NOTE: this may need to be added back in, but doesn't interact well with
# `diff_attn_mix`. # `diff_attn_mix`, and doesn't allow us to preserve the original model output.
# attn = attn * self.diff_attn_mix * (1 - self.lambda_init) # attn = attn * self.diff_attn_mix * (1 - self.lambda_init)
attn = attn.transpose(1, 2).reshape(bsz, q_len, self.config.hidden_size) attn = attn.transpose(1, 2).reshape(bsz, q_len, self.config.hidden_size)