tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

batch flattening with xformers too

Browse Source
This commit is contained in:
Wing Lian
2025-05-08 18:23:25 -04:00
parent 7fb01f0461
commit 5e50d1e8f0
1 changed files with 8 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/axolotl/utils/schemas/config.py
View File

@@ -475,8 +475,14 @@ class AxolotlInputConfig(
def check_batch_flattening_fa(cls, data): def check_batch_flattening_fa(cls, data):
if data.get("batch_flattening"): if data.get("batch_flattening"):
batch_flattening_auto = data.get("batch_flattening") == "auto" batch_flattening_auto = data.get("batch_flattening") == "auto"
if not data.get("flash_attention") and not batch_flattening_auto: if (
raise ValueError("batch_flattening requires flash attention") not data.get("flash_attention")
and not data.get("xformers_attention")
and not batch_flattening_auto
):
raise ValueError(
"batch_flattening requires flash attention or xformers"
)
if data.get("sample_packing") and not batch_flattening_auto: if data.get("sample_packing") and not batch_flattening_auto:
raise ValueError("batch_flattening not compatible with sample_packing") raise ValueError("batch_flattening not compatible with sample_packing")
if data.get("micro_batch_size") == 1 and not batch_flattening_auto: if data.get("micro_batch_size") == 1 and not batch_flattening_auto: