tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix eval regression caused in 13f7efaf74

Browse Source
This commit is contained in:
Aman Karmani
2023-08-21 10:40:06 -07:00
parent fbf49a4770
commit a213d9972a
1 changed files with 1 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/axolotl/monkeypatch/llama_attn_hijack_flash.py
View File

@@ -155,12 +155,10 @@ def flashattn_forward(
# during training q,k,v always have same seqlen # during training q,k,v always have same seqlen
assert key_states.shape == query_states.shape assert key_states.shape == query_states.shape
is_causal = True is_causal = True
elif past_key_value is None:
is_causal = True
else: else:
# turn off FA causal mask after first inference autoregressive iteration # turn off FA causal mask after first inference autoregressive iteration
# only on first autoregressive step q,k,v have same seqlen # only on first autoregressive step q,k,v have same seqlen
is_causal = past_key_value is not None is_causal = key_states.shape == query_states.shape
if cu_seqlens is not None and max_seqlen is not None: if cu_seqlens is not None and max_seqlen is not None:
# special handling using sample packing # special handling using sample packing