set a different triton cache for each test to avoid blocking writes to cache (#2843)

* set a different triton cache for each test to avoid blocking writes to cache

* set log level

* disable debug logging for filelock

tests/conftest.py

@@ -4,6 +4,7 @@ shared pytest fixtures
 
 import functools
 import importlib
+import logging
 import os
 import shutil
 import sys
@@ -25,6 +26,8 @@ from tests.hf_offline_utils import (
     hf_offline_context,
 )
 
+logging.getLogger("filelock").setLevel(logging.CRITICAL)
+
 
 def retry_on_request_exceptions(max_retries=3, delay=1):
     # pylint: disable=duplicate-code
@@ -420,6 +423,11 @@ def temp_dir() -> Generator[str, None, None]:
     shutil.rmtree(_temp_dir)
 
 
+@pytest.fixture(scope="function", autouse=True)
+def unique_triton_cache_dir(temp_dir):
+    os.environ["TRITON_CACHE_DIR"] = temp_dir + "/~.triton/cache"
+
+
 @pytest.fixture(scope="function", autouse=True)
 def cleanup_monkeypatches():
     from transformers import Trainer