tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

use wip branch for expected 4.48.1

Browse Source
This commit is contained in:
Wing Lian
2025-01-16 09:08:18 -05:00
parent 0714a49227
commit 50d4d727a0
2 changed files with 10 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
requirements.txt
View File

@@ -14,7 +14,7 @@ packaging==23.2
peft==0.14.0 peft==0.14.0
# transformers==4.48.1 # transformers==4.48.1
transformers @ git+https://github.com/huggingface/transformers.git@v4.48-release transformers @ git+https://github.com/winglian/transformers.git@pre-v4.48.1
tokenizers>=0.21.0 tokenizers>=0.21.0
accelerate==1.2.1 accelerate==1.2.1
datasets==3.2.0 datasets==3.2.0

18
src/axolotl/utils/models.py
View File

@@ -386,15 +386,15 @@ class ModelLoader:
if self.cfg.flash_attention: if self.cfg.flash_attention:
self.patch_attention() self.patch_attention()
if self.cfg.model_config_type == "llama": # if self.cfg.model_config_type == "llama":
from axolotl.monkeypatch.trainer_grad_accum import ( # patch_forward_for_ga, # from axolotl.monkeypatch.trainer_grad_accum import ( # patch_forward_for_ga,
patch_flash_attention_forward, # patch_flash_attention_forward,
patch_training_step_for_ga, # patch_training_step_for_ga,
) # )
#
patch_flash_attention_forward() # patch_flash_attention_forward()
# patch_forward_for_ga() # # patch_forward_for_ga()
patch_training_step_for_ga() # patch_training_step_for_ga()
if self.cfg.sample_packing and self.cfg.s2_attention: if self.cfg.sample_packing and self.cfg.s2_attention:
raise ValueError( raise ValueError(