tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

remove landmark attn and xpos rope implementations (#1010)

Browse Source
This commit is contained in:
Wing Lian
2023-12-27 23:07:27 -06:00
committed by GitHub
parent 85dd4d525b
commit 70b46ca4f4
6 changed files with 1 additions and 1404 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/axolotl/cli/__init__.py
View File

@@ -103,14 +103,6 @@ def do_inference(
importlib.import_module("axolotl.prompters"), prompter
)
if cfg.landmark_attention:
from axolotl.monkeypatch.llama_landmark_attn import set_model_mem_id
set_model_mem_id(model, tokenizer)
model.set_mem_cache_args(
max_seq_len=255, mem_freq=50, top_k=5, max_cache_size=None
)
model = model.to(cfg.device)
while True:
@@ -176,14 +168,6 @@ def do_inference_gradio(
importlib.import_module("axolotl.prompters"), prompter
)
if cfg.landmark_attention:
from axolotl.monkeypatch.llama_landmark_attn import set_model_mem_id
set_model_mem_id(model, tokenizer)
model.set_mem_cache_args(
max_seq_len=255, mem_freq=50, top_k=5, max_cache_size=None
)
model = model.to(cfg.device)
def generate(instruction):