refactor and fixing test isolation issues

tests/e2e/integrations/convert_diff_transformer/conftest.py

@@ -4,7 +4,7 @@ import pytest
 from click.testing import CliRunner
 
 
-@pytest.fixture()
+@pytest.fixture(scope="class")
 def base_config():
     """Basic config for testing."""
     return {
@@ -26,6 +26,6 @@ def base_config():
     }
 
 
-@pytest.fixture
+@pytest.fixture(scope="class")
 def cli_runner():
     return CliRunner()

tests/e2e/integrations/convert_diff_transformer/test_convert_diff_transformer.py

@@ -15,133 +15,135 @@ from axolotl.cli.main import cli
 from axolotl.common.cli import ConvertDiffTransformerCliArgs
 
 
-def test_cli_validation(cli_runner):
-    # Test missing config file
-    result = cli_runner.invoke(cli, ["convert-diff-transformer"])
-    assert result.exit_code != 0
-    assert "Error: Missing argument 'CONFIG'." in result.output
+@pytest.mark.usefixtures("base_config", "cli_runner")
+class TestDiffTransformer:
+    """Tests for convert-diff-transformer CLI command"""
 
-    # Test non-existent config file
-    result = cli_runner.invoke(cli, ["convert-diff-transformer", "nonexistent.yml"])
-    assert result.exit_code != 0
-    assert "Error: Invalid value for 'CONFIG'" in result.output
+    def test_cli_validation(self, cli_runner):
+        # Test missing config file
+        result = cli_runner.invoke(cli, ["convert-diff-transformer"])
+        assert result.exit_code != 0
+        assert "Error: Missing argument 'CONFIG'." in result.output
 
+        # Test non-existent config file
+        result = cli_runner.invoke(cli, ["convert-diff-transformer", "nonexistent.yml"])
+        assert result.exit_code != 0
+        assert "Error: Invalid value for 'CONFIG'" in result.output
 
-def test_basic_execution(cli_runner, tmp_path: Path, base_config):
-    config_path = tmp_path / "config.yml"
-    with open(config_path, "w", encoding="utf-8") as file:
-        yaml.dump(base_config, file)
+    def test_basic_execution(self, cli_runner, tmp_path: Path, base_config):
+        config_path = tmp_path / "config.yml"
+        with open(config_path, "w", encoding="utf-8") as file:
+            yaml.dump(base_config, file)
 
-    with patch(
-        "axolotl.cli.integrations.convert_diff_transformer.do_cli"
-    ) as mock_do_cli:
-        result = cli_runner.invoke(cli, ["convert-diff-transformer", str(config_path)])
-        assert result.exit_code == 0
+        with patch(
+            "axolotl.cli.integrations.convert_diff_transformer.do_cli"
+        ) as mock_do_cli:
+            result = cli_runner.invoke(
+                cli, ["convert-diff-transformer", str(config_path)]
+            )
+            assert result.exit_code == 0
 
-        mock_do_cli.assert_called_once()
-        assert mock_do_cli.call_args.kwargs["config"] == str(config_path)
+            mock_do_cli.assert_called_once()
+            assert mock_do_cli.call_args.kwargs["config"] == str(config_path)
 
+    def test_conversion_cli_basic(self, tmp_path: Path, base_config):
+        output_dir = tmp_path / "converted"
+        base_config["output_dir"] = str(output_dir)
 
-def test_conversion_cli_basic(tmp_path: Path, base_config):
-    output_dir = tmp_path / "converted"
-    base_config["output_dir"] = str(output_dir)
+        config_path = tmp_path / "config.yml"
+        with open(config_path, "w", encoding="utf-8") as file:
+            yaml.dump(base_config, file)
 
-    config_path = tmp_path / "config.yml"
-    with open(config_path, "w", encoding="utf-8") as file:
-        yaml.dump(base_config, file)
+        cfg = load_cfg(str(config_path))
+        cli_args = ConvertDiffTransformerCliArgs()
+        _, debug_info = convert_diff_transformer(cfg, cli_args, str(config_path))
 
-    cfg = load_cfg(str(config_path))
-    cli_args = ConvertDiffTransformerCliArgs()
-    _, debug_info = convert_diff_transformer(cfg, cli_args, str(config_path))
+        assert not debug_info
+        assert (output_dir / "model.safetensors").exists()
+        assert (output_dir / "config.json").exists()
+        assert (output_dir / "axolotl_config.yml").exists()
 
-    assert not debug_info
-    assert (output_dir / "model.safetensors").exists()
-    assert (output_dir / "config.json").exists()
-    assert (output_dir / "axolotl_config.yml").exists()
+    def test_conversion_cli_debug(self, tmp_path: Path, base_config):
+        output_dir = tmp_path / "converted"
+        base_config["output_dir"] = str(output_dir)
 
+        config_path = tmp_path / "config.yml"
+        with open(config_path, "w", encoding="utf-8") as file:
+            yaml.dump(base_config, file)
 
-def test_conversion_cli_debug(tmp_path: Path, base_config):
-    output_dir = tmp_path / "converted"
-    base_config["output_dir"] = str(output_dir)
+        cfg = load_cfg(str(config_path))
+        cli_args = ConvertDiffTransformerCliArgs(debug=True)
+        _, debug_info = convert_diff_transformer(cfg, cli_args, str(config_path))
 
-    config_path = tmp_path / "config.yml"
-    with open(config_path, "w", encoding="utf-8") as file:
-        yaml.dump(base_config, file)
+        assert not debug_info["generations_match"]
+        assert not debug_info["match_expected"]
+        assert (output_dir / "model.safetensors").exists()
+        assert (output_dir / "config.json").exists()
+        assert (output_dir / "axolotl_config.yml").exists()
 
-    cfg = load_cfg(str(config_path))
-    cli_args = ConvertDiffTransformerCliArgs(debug=True)
-    _, debug_info = convert_diff_transformer(cfg, cli_args, str(config_path))
+    def test_conversion_cli_reproduce(self, tmp_path: Path, base_config):
+        output_dir = tmp_path / "converted"
+        base_config["output_dir"] = str(output_dir)
 
-    assert not debug_info["generations_match"]
-    assert not debug_info["match_expected"]
-    assert (output_dir / "model.safetensors").exists()
-    assert (output_dir / "config.json").exists()
-    assert (output_dir / "axolotl_config.yml").exists()
+        config_path = tmp_path / "config.yml"
+        with open(config_path, "w", encoding="utf-8") as file:
+            yaml.dump(base_config, file)
 
+        cfg = load_cfg(str(config_path))
+        cli_args = ConvertDiffTransformerCliArgs(
+            debug=True, zero_init=True, sublayer_norm=False
+        )
+        _, debug_info = convert_diff_transformer(cfg, cli_args, str(config_path))
 
-def test_conversion_cli_reproduce(tmp_path: Path, base_config):
-    output_dir = tmp_path / "converted"
-    base_config["output_dir"] = str(output_dir)
+        assert debug_info["generations_match"] is True
+        assert (output_dir / "model.safetensors").exists()
+        assert (output_dir / "config.json").exists()
+        assert (output_dir / "axolotl_config.yml").exists()
 
-    config_path = tmp_path / "config.yml"
-    with open(config_path, "w", encoding="utf-8") as file:
-        yaml.dump(base_config, file)
-
-    cfg = load_cfg(str(config_path))
-    cli_args = ConvertDiffTransformerCliArgs(
-        debug=True, zero_init=True, sublayer_norm=False
+    @pytest.mark.parametrize(
+        "attention", ["eager_attention", "sdp_attention", "flash_attention"]
     )
-    _, debug_info = convert_diff_transformer(cfg, cli_args, str(config_path))
+    def test_conversion_cli_repoduce_attentions(
+        self, tmp_path: Path, base_config, attention: Optional[str]
+    ):
+        output_dir = tmp_path / "converted"
+        base_config["output_dir"] = str(output_dir)
+        base_config[attention] = True
 
-    assert debug_info["generations_match"] is True
-    assert (output_dir / "model.safetensors").exists()
-    assert (output_dir / "config.json").exists()
-    assert (output_dir / "axolotl_config.yml").exists()
+        config_path = tmp_path / "config.yml"
+        with open(config_path, "w", encoding="utf-8") as file:
+            yaml.dump(base_config, file)
 
+        cfg = load_cfg(str(config_path))
+        cli_args = ConvertDiffTransformerCliArgs(
+            debug=True, zero_init=True, sublayer_norm=False
+        )
+        _, debug_info = convert_diff_transformer(cfg, cli_args, str(config_path))
 
-@pytest.mark.parametrize(
-    "attention", ["eager_attention", "sdp_attention", "flash_attention"]
-)
-def test_conversion_cli_repoduce_attentions(
-    tmp_path: Path, base_config, attention: Optional[str]
-):
-    output_dir = tmp_path / "converted"
-    base_config["output_dir"] = str(output_dir)
-    base_config[attention] = True
+        assert debug_info["generations_match"] is True
+        assert (output_dir / "model.safetensors").exists()
+        assert (output_dir / "config.json").exists()
+        assert (output_dir / "axolotl_config.yml").exists()
 
-    config_path = tmp_path / "config.yml"
-    with open(config_path, "w", encoding="utf-8") as file:
-        yaml.dump(base_config, file)
-
-    cfg = load_cfg(str(config_path))
-    cli_args = ConvertDiffTransformerCliArgs(
-        debug=True, zero_init=True, sublayer_norm=False
+    @pytest.mark.parametrize(
+        "attention", ["eager_attention", "sdp_attention", "flash_attention"]
     )
-    _, debug_info = convert_diff_transformer(cfg, cli_args, str(config_path))
+    def test_conversion_cli_split_heads(
+        self, tmp_path: Path, base_config, attention: str
+    ):
+        output_dir = tmp_path / "converted"
+        base_config["output_dir"] = str(output_dir)
+        base_config[attention] = True
 
-    assert debug_info["generations_match"] is True
-    assert (output_dir / "model.safetensors").exists()
-    assert (output_dir / "config.json").exists()
-    assert (output_dir / "axolotl_config.yml").exists()
+        config_path = tmp_path / "config.yml"
+        with open(config_path, "w", encoding="utf-8") as file:
+            yaml.dump(base_config, file)
 
+        cfg = load_cfg(str(config_path))
+        cli_args = ConvertDiffTransformerCliArgs(debug=True, split_heads=True)
+        _, debug_info = convert_diff_transformer(cfg, cli_args, str(config_path))
 
-@pytest.mark.parametrize(
-    "attention", ["eager_attention", "sdp_attention", "flash_attention"]
-)
-def test_conversion_cli_split_heads(tmp_path: Path, base_config, attention: str):
-    output_dir = tmp_path / "converted"
-    base_config["output_dir"] = str(output_dir)
-    base_config[attention] = True
-
-    config_path = tmp_path / "config.yml"
-    with open(config_path, "w", encoding="utf-8") as file:
-        yaml.dump(base_config, file)
-
-    cfg = load_cfg(str(config_path))
-    cli_args = ConvertDiffTransformerCliArgs(debug=True, split_heads=True)
-    _, debug_info = convert_diff_transformer(cfg, cli_args, str(config_path))
-
-    assert debug_info["generations_match"] is False
-    assert (output_dir / "model.safetensors").exists()
-    assert (output_dir / "config.json").exists()
-    assert (output_dir / "axolotl_config.yml").exists()
+        assert debug_info["generations_match"] is False
+        assert (output_dir / "model.safetensors").exists()
+        assert (output_dir / "config.json").exists()
+        assert (output_dir / "axolotl_config.yml").exists()