SDPA context parallel

src/axolotl/train.py

@@ -11,11 +11,13 @@ from pathlib import Path
 from typing import Any, Dict
 
 import torch
+import torch.distributed as dist
 import transformers.modelcard
 from accelerate.utils import save_fsdp_model
 from datasets import Dataset
 from huggingface_hub.errors import OfflineModeIsEnabled
 from peft import PeftConfig, PeftModel
+from torch.distributed.tensor.experimental import _context_parallel
 from transformers import PreTrainedModel, PreTrainedTokenizer, ProcessorMixin
 from transformers.integrations.deepspeed import is_deepspeed_zero3_enabled
 from transformers.trainer import Trainer
@@ -203,6 +205,19 @@ def execute_training(
             )
 
         if cfg.sequence_parallel_degree > 1:
+            if cfg.sdp_attention:
+                world_size = dist.get_world_size()
+                mesh_shape = (
+                    world_size // cfg.sequence_parallel_degree,
+                    cfg.sequence_parallel_degree,
+                )
+                mesh = dist.DeviceMesh(
+                    "cuda",
+                    torch.tensor(list(range(world_size))).reshape(mesh_shape),
+                    mesh_dim_names=("dp", "cp"),
+                )
+                stack.enter_context(_context_parallel(seq_dim=2, mesh=mesh))
+            else:  # flash_attention
                 models = [trainer.model]
                 if hasattr(trainer, "ref_model") and trainer.ref_model:
                     models.append(trainer.ref_model)