55 lines
2.5 KiB
Python
55 lines
2.5 KiB
Python
from __future__ import annotations
|
|
import asyncio, logging, time
|
|
from .llm_types import LLMResponse, OllamaTimeoutError, OllamaUnavailableError
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class OllamaBackend:
|
|
def __init__(self, url, model, timeout=120, max_concurrent=2):
|
|
self._url = url
|
|
self._model = model
|
|
self._timeout = timeout
|
|
self._semaphore = asyncio.Semaphore(max_concurrent)
|
|
self._active = 0
|
|
|
|
async def submit(self, messages, tools=None, caller='unknown'):
|
|
import ollama
|
|
wait_start = time.monotonic()
|
|
async with self._semaphore:
|
|
wait_ms = int((time.monotonic() - wait_start) * 1000)
|
|
self._active += 1
|
|
t0 = time.monotonic()
|
|
try:
|
|
kwargs = {'model': self._model, 'messages': messages}
|
|
if tools:
|
|
kwargs['tools'] = tools
|
|
client = ollama.AsyncClient(host=self._url)
|
|
try:
|
|
response = await asyncio.wait_for(client.chat(**kwargs), timeout=self._timeout)
|
|
except asyncio.TimeoutError:
|
|
raise OllamaTimeoutError(f'Ollama timeout after {self._timeout}s caller={caller}')
|
|
except Exception as exc:
|
|
s = str(exc).lower()
|
|
if 'connect' in s or 'refused' in s or 'unreachable' in s:
|
|
raise OllamaUnavailableError(f'Ollama unreachable: {exc}') from exc
|
|
raise OllamaUnavailableError(f'Ollama error: {exc}') from exc
|
|
ms = int((time.monotonic() - t0) * 1000)
|
|
msg = response.message
|
|
tool_calls = None
|
|
if hasattr(msg, 'tool_calls') and msg.tool_calls:
|
|
tool_calls = [{'name': tc.function.name, 'arguments': tc.function.arguments}
|
|
for tc in msg.tool_calls]
|
|
tin = response.prompt_eval_count or 0
|
|
tout = response.eval_count or 0
|
|
logger.info('ollama caller=%s wait_ms=%d inf_ms=%d tin=%d tout=%d',
|
|
caller, wait_ms, ms, tin, tout)
|
|
return LLMResponse(content=msg.content or '', tool_calls=tool_calls,
|
|
backend_used='ollama', model_used=self._model,
|
|
tokens_in=tin, tokens_out=tout, latency_ms=ms)
|
|
finally:
|
|
self._active -= 1
|
|
|
|
@property
|
|
def active_count(self): return self._active
|