tocmo0nlord/axolotl
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: deepspeed with context parallel (#3220)

Browse Source
This commit is contained in:
NanoCode012
2025-10-20 14:06:58 +07:00
committed by GitHub
parent 87565ecc05
commit 8bb871b5cf
1 changed files with 1 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/axolotl/monkeypatch/transformers/trainer_context_parallel.py
View File

@@ -13,9 +13,7 @@ from axolotl.utils.logging import get_logger
LOG = get_logger(__name__) LOG = get_logger(__name__)
GUARD_PATTERN = 'if model.config._attn_implementation != "sdpa":' GUARD_PATTERN = 'if model.config._attn_implementation != "sdpa":'
PATCHED_GUARD = ( PATCHED_GUARD = 'if (attn_impl := (getattr(model.config, "_attn_implementation", None) or getattr(model.model.config, "_attn_implementation", None))) and attn_impl not in ("sdpa", "flash_attention_2"):'
'if model.config._attn_implementation not in ("sdpa", "flash_attention_2"):'
)
def patch_prepare_context_parallel_inputs() -> None: def patch_prepare_context_parallel_inputs() -> None: