more test

2025-02-02 00:45:30 -05:00
parent 10de67e8ea
commit 9a43a0925d
1 changed files with 1 additions and 1 deletions
--- a/src/axolotl/monkeypatch/flex_attn.py
+++ b/src/axolotl/monkeypatch/flex_attn.py
@@ -51,7 +51,7 @@ def create_block_causal_mask(
        residue_len = max_seq_len - torch.sum(seq_lens[sample_idx])
        block_attn_masks.append(
            torch.tril(
-                torch.zeros(
+                torch.ones(
                    residue_len, residue_len, dtype=torch.bool, device=seq_lens[sample_idx].device
                )
            )