tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

is_causal fix for evals?

Browse Source
This commit is contained in:
Wing Lian
2023-08-21 10:36:26 -04:00
parent 58cf7e7fed
commit fbf49a4770
1 changed files with 2 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/axolotl/monkeypatch/llama_attn_hijack_flash.py
View File

@@ -155,6 +155,8 @@ def flashattn_forward(
# during training q,k,v always have same seqlen # during training q,k,v always have same seqlen
assert key_states.shape == query_states.shape assert key_states.shape == query_states.shape
is_causal = True is_causal = True
elif past_key_value is None:
is_causal = True
else: else:
# turn off FA causal mask after first inference autoregressive iteration # turn off FA causal mask after first inference autoregressive iteration
# only on first autoregressive step q,k,v have same seqlen # only on first autoregressive step q,k,v have same seqlen