tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Flash attn hotfix (#951)

Browse Source 
* use previous  arg

* use eager to use legacy attention that can be patched
This commit is contained in:
Wing Lian
2023-12-13 13:42:23 -05:00
committed by GitHub
parent 450e04d3c4
commit f1f60cb5b2
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/axolotl/utils/models.py
View File

@@ -324,6 +324,10 @@ def load_model(
model_config._attn_implementation = ( # pylint: disable=protected-access model_config._attn_implementation = ( # pylint: disable=protected-access
"flash_attention_2" "flash_attention_2"
) )
else:
model_config._attn_implementation = ( # pylint: disable=protected-access
"eager"
)
try: try:
if cfg.is_llama_derived_model and not cfg.trust_remote_code and not cfg.gptq: if cfg.is_llama_derived_model and not cfg.trust_remote_code and not cfg.gptq: