tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

more test

Browse Source
This commit is contained in:
bursteratom
2025-02-02 00:45:30 -05:00
parent 10de67e8ea
commit 9a43a0925d
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/axolotl/monkeypatch/flex_attn.py
View File

@@ -51,7 +51,7 @@ def create_block_causal_mask(
residue_len = max_seq_len - torch.sum(seq_lens[sample_idx]) residue_len = max_seq_len - torch.sum(seq_lens[sample_idx])
block_attn_masks.append( block_attn_masks.append(
torch.tril( torch.tril(
torch.zeros( torch.ones(
residue_len, residue_len, dtype=torch.bool, device=seq_lens[sample_idx].device residue_len, residue_len, dtype=torch.bool, device=seq_lens[sample_idx].device
) )
) )