Add shifted sparse attention (#973) [skip-ci]

* Add s2_attn to hijack flash code

* Refactor code to account for s2_attn

* Add test for models utils

* Add ``s2_attention`` option to llama configs

* Add ``s2_attention`` option to README config

* Format code to appease linter

* chore: lint

* Remove xpos and llama-landmark [bad merge]

* add e2e smoke tests for shifted sparse attention

* remove stray patch from merge

* update yml with link to paper for s2_attention/longlora

* fix assertion check for full fine tune

* increase sequence len for tests and PR feedback updates

* reduce context len to 16k for tests

* reduce context len to 16k for tests

* reduce batch size for larger context len and udpate test to check message

* fix test for message

---------

Co-authored-by: joecummings <jrcummings@devvm050.nha0.facebook.com>
Co-authored-by: Wing Lian <wing.lian@gmail.com>

tests/e2e/patched/test_llama_s2_attention.py

@@ -0,0 +1,111 @@
+"""
+E2E tests for llama w/ S2 attn
+"""
+
+import logging
+import os
+import unittest
+from pathlib import Path
+
+from axolotl.cli import load_datasets
+from axolotl.common.cli import TrainerCliArgs
+from axolotl.train import train
+from axolotl.utils.config import normalize_config
+from axolotl.utils.dict import DictDefault
+
+from ..utils import with_temp_dir
+
+LOG = logging.getLogger("axolotl.tests.e2e")
+os.environ["WANDB_DISABLED"] = "true"
+
+
+class TestLlamaShiftedSparseAttention(unittest.TestCase):
+    """
+    Test case for Llama models using S2 Attn
+    """
+
+    @with_temp_dir
+    def test_lora_s2_attn(self, temp_dir):
+        # pylint: disable=duplicate-code
+        cfg = DictDefault(
+            {
+                "base_model": "JackFram/llama-68m",
+                "tokenizer_type": "LlamaTokenizer",
+                "sequence_len": 16384,
+                "sample_packing": False,
+                "flash_attention": True,
+                "s2_attention": True,
+                "load_in_8bit": True,
+                "adapter": "lora",
+                "lora_r": 32,
+                "lora_alpha": 16,
+                "lora_dropout": 0.05,
+                "lora_target_linear": True,
+                "val_set_size": 0.1,
+                "special_tokens": {},
+                "datasets": [
+                    {
+                        "path": "Yukang/LongAlpaca-12k",
+                        "type": "alpaca",
+                    },
+                ],
+                "num_epochs": 2,
+                "micro_batch_size": 1,
+                "gradient_accumulation_steps": 1,
+                "output_dir": temp_dir,
+                "learning_rate": 0.00001,
+                "optimizer": "adamw_torch",
+                "lr_scheduler": "cosine",
+                "max_steps": 10,
+                "save_steps": 5,
+                "eval_steps": 5,
+                "bf16": "auto",
+            }
+        )
+
+        normalize_config(cfg)
+        cli_args = TrainerCliArgs()
+        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
+
+        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
+        assert (Path(temp_dir) / "adapter_model.bin").exists()
+
+    @with_temp_dir
+    def test_fft_s2_attn(self, temp_dir):
+        # pylint: disable=duplicate-code
+        cfg = DictDefault(
+            {
+                "base_model": "JackFram/llama-68m",
+                "tokenizer_type": "LlamaTokenizer",
+                "sequence_len": 16384,
+                "sample_packing": False,
+                "flash_attention": True,
+                "s2_attention": True,
+                "val_set_size": 0.1,
+                "special_tokens": {},
+                "datasets": [
+                    {
+                        "path": "Yukang/LongAlpaca-12k",
+                        "type": "alpaca",
+                    },
+                ],
+                "num_epochs": 2,
+                "micro_batch_size": 1,
+                "gradient_accumulation_steps": 1,
+                "output_dir": temp_dir,
+                "learning_rate": 0.00001,
+                "optimizer": "adamw_torch",
+                "lr_scheduler": "cosine",
+                "max_steps": 10,
+                "save_steps": 5,
+                "eval_steps": 5,
+                "bf16": "auto",
+            }
+        )
+
+        normalize_config(cfg)
+        cli_args = TrainerCliArgs()
+        dataset_meta = load_datasets(cfg=cfg, cli_args=cli_args)
+
+        train(cfg=cfg, cli_args=cli_args, dataset_meta=dataset_meta)
+        assert (Path(temp_dir) / "pytorch_model.bin").exists()

tests/utils/test_models.py

@@ -0,0 +1,37 @@
+"""Module for testing models utils file."""
+
+
+import unittest
+from unittest.mock import patch
+
+import pytest
+
+from axolotl.utils.dict import DictDefault
+from axolotl.utils.models import load_model
+
+
+class ModelsUtilsTest(unittest.TestCase):
+    """Testing module for models utils."""
+
+    def test_cfg_throws_error_with_s2_attention_and_sample_packing(self):
+        cfg = DictDefault(
+            {
+                "s2_attention": True,
+                "sample_packing": True,
+                "base_model": "",
+                "model_type": "LlamaForCausalLM",
+            }
+        )
+
+        # Mock out call to HF hub
+        with patch(
+            "axolotl.utils.models.load_model_config"
+        ) as mocked_load_model_config:
+            mocked_load_model_config.return_value = {}
+            with pytest.raises(ValueError) as exc:
+                # Should error before hitting tokenizer, so we pass in an empty str
+                load_model(cfg, tokenizer="")
+            assert (
+                "shifted-sparse attention does not currently support sample packing"
+                in str(exc.value)
+            )