09959fac70
* feat: update mistral common * feat: add mistral3processor * fix: loading * fix: cast pixel_values to fp32 * fix: image tensor conversion * feat: add FA2 support for pixtral based models * fix: update mistral small 3.1 to use native tokenizer * fix: install tips * fix: improve info on sample dataset files * chore: move mistral configs into subfolders * fix: remove unneeded patch * fix: indent * feat: add integration tests * chore: move * feat: add magistral 2509 docs and example * fix: convert tensor to bool * feat: expand tests * chore: move tests
36 lines
1.2 KiB
Python
36 lines
1.2 KiB
Python
"""Integration tests for MistralCommonTokenizer patches."""
|
|
|
|
import pytest
|
|
|
|
|
|
class TestMistralTokenizerPatchIntegration:
|
|
"""Test MistralCommonTokenizer patch integration."""
|
|
|
|
@pytest.mark.integration
|
|
def test_mistral_tokenizer_image_patch(self):
|
|
"""Test that MistralCommonTokenizer image patch can be applied."""
|
|
try:
|
|
from transformers.tokenization_mistral_common import MistralCommonTokenizer
|
|
except ImportError:
|
|
pytest.skip("MistralCommonTokenizer not available")
|
|
|
|
from axolotl.monkeypatch.models.mistral3.mistral_common_tokenizer import (
|
|
apply_mistral_tokenizer_image_patch,
|
|
)
|
|
|
|
# Store original method
|
|
original_apply_chat_template = MistralCommonTokenizer.apply_chat_template
|
|
|
|
# Apply patch
|
|
apply_mistral_tokenizer_image_patch()
|
|
|
|
# Verify patch was applied
|
|
assert (
|
|
MistralCommonTokenizer.apply_chat_template != original_apply_chat_template
|
|
), "apply_chat_template was not patched"
|
|
|
|
# Verify the method is still callable
|
|
assert callable(MistralCommonTokenizer.apply_chat_template), (
|
|
"Patched method is not callable"
|
|
)
|