fsdp2 w evals fixed upstream

.github/workflows/preview-docs.yml

@@ -10,7 +10,7 @@ on:
       - '**/*.qmd'     # any Quarto file
       - '_quarto.yml'
       - docs/scripts/generate_config_docs.py
-      - src/axolotl/**/*.py
+      - src/axolotl/utils/schemas/**.py
 
 permissions:
   checks: write

TODO.md

@@ -0,0 +1,10 @@
+# todo list
+
+- [] Validation of parameters for combinations that won't work
+
+
+
+## things that are known not to work
+
+- FSDP offload and gradient_checkpointing - https://github.com/pytorch/pytorch/issues/82203
+- adamw_bnb_8bit doesn't play well with FSDP offload

requirements.txt

@@ -14,7 +14,7 @@ packaging==23.2
 
 huggingface_hub>=0.33.0
 peft==0.17.0
-transformers==4.55.0
+transformers @ git+https://github.com/vasqu/transformers@fix-fa-integration
 tokenizers>=0.21.1
 accelerate==1.10.0
 datasets==4.0.0

src/axolotl/cli/utils/train.py

@@ -74,9 +74,7 @@ def generate_config_files(config: str, sweep: str | None) -> Iterator[tuple[str,
         sweep: Sweep configuration file
 
     Yields:
-        tuple[str, bool]: (config_file, is_group)
-            - config_file: configuration file path
-            - is_group: whether this is a group of configurations
+        Tuple of configuration file name and whether this is a group of configurations
     """
 
     if not sweep:

src/axolotl/monkeypatch/ring_attn/patch.py

@@ -18,9 +18,7 @@ from torch.distributed import DeviceMesh
 try:
     from transformers.modeling_flash_attention_utils import _flash_supports_window
 except ImportError:
-    from transformers.modeling_flash_attention_utils import (
-        _flash_supports_window_size as _flash_supports_window,
-    )
+    _flash_supports_window = True
 
 from axolotl.monkeypatch.utils import get_cu_seqlens_from_pos_ids
 from axolotl.utils.logging import get_logger

tests/monkeypatch/test_trainer_loss_calc.py

@@ -3,7 +3,6 @@
 import unittest
 
 from axolotl.monkeypatch.transformers.trainer_loss_calc import (
-    check_evaluation_loop_is_fsdp2_patchable,
     check_evaluation_loop_is_patchable,
     check_maybe_log_save_evaluate_is_patchable,
 )
@@ -20,7 +19,6 @@ class TestTrainerLossCalc(unittest.TestCase):
         the patched code changes upstream.
         """
         assert check_evaluation_loop_is_patchable()
-        assert check_evaluation_loop_is_fsdp2_patchable()
         assert check_maybe_log_save_evaluate_is_patchable()