Distributed/ND-Parallel (#2977)

2025-07-31 20:25:02 +01:00
parent 7b68dfafd7
commit 294c7fe7a6
49 changed files with 712 additions and 835 deletions
--- a/examples/alst/llama3-8b-deepspeed-alst.yaml
+++ b/examples/alst/llama3-8b-deepspeed-alst.yaml
@@ -20,7 +20,7 @@ min_sample_len: 200_000
 sample_packing: true

 tiled_mlp: true
-sequence_parallel_degree: 8
+context_parallel_size: 8
 plugins:
  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin