Merge pull request #169 from NanoCode012/feat/landmark

Feat: Add landmark attention
2023-06-10 07:26:06 +09:00
parent 4d6490bce2 2a801b001a
commit b5aa8d854c
4 changed files with 1635 additions and 8 deletions
--- a/README.md
+++ b/README.md
@@ -417,6 +417,8 @@ flash_attention:  # require a100 for llama
 # whether to use scaled-dot-product attention
 # https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html
 sdp_attention:
+# Landmark attention (only llama)
+landmark_attention:

 # resume from a specific checkpoint dir
 resume_from_checkpoint: