"""Centralized LLM API client with BYOK (bring-your-own-key) provider support. Reads provider configuration from :class:`SettingsManager` and makes OpenAI-compatible ``/chat/completions`` calls. Supports any provider that implements the OpenAI Chat Completions API surface area (OpenAI, Ollama, vLLM, LM Studio, etc.). """ from __future__ import annotations import asyncio import json import logging from typing import Any, Dict, List, Optional import aiohttp from .errors import LLMNotConfiguredError, LLMRateLimitError, LLMResponseError logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- # Model catalog sourced from opencode's maintained model registry. # maps provider_id -> list of model IDs. # --------------------------------------------------------------------------- _MODEL_CATALOG_URL = "https://models.dev/api.json" # In-memory cache: maps provider slug -> list of model ID strings. _catalog_cache: Optional[Dict[str, List[str]]] = None _CATALOG_TIMEOUT = aiohttp.ClientTimeout(total=30) async def _load_model_catalog() -> Dict[str, List[str]]: """Fetch and parse the model catalog, returning ``{provider_id: [model_id, ...]}``. The JSON at ``_MODEL_CATALOG_URL`` is a dict keyed by provider slug; each value has a ``models`` sub-dict keyed by model ID. Only the model IDs are kept. The result is cached in memory after the first successful fetch. Subsequent calls return the cached data immediately. """ global _catalog_cache if _catalog_cache is not None: return _catalog_cache try: async with aiohttp.ClientSession(timeout=_CATALOG_TIMEOUT) as session: async with session.get(_MODEL_CATALOG_URL) as resp: if resp.status != 200: logger.warning("Model catalog returned HTTP %s", resp.status) return _catalog_cache or {} data = await resp.json() except (aiohttp.ClientError, asyncio.TimeoutError, json.JSONDecodeError) as exc: logger.warning("Failed to fetch model catalog: %s", exc) return _catalog_cache or {} if not isinstance(data, dict): logger.warning("Model catalog is not a dict, got %s", type(data).__name__) return _catalog_cache or {} result: Dict[str, List[str]] = {} for provider_id, provider_info in data.items(): if not isinstance(provider_info, dict): continue models_dict = provider_info.get("models") if not isinstance(models_dict, dict): continue model_ids = [str(mid) for mid in models_dict.keys() if isinstance(mid, str)] if model_ids: result[provider_id] = model_ids _catalog_cache = result logger.info( "Loaded model catalog: %d providers, %d total models", len(result), sum(len(m) for m in result.values()), ) return result # Short timeout for Ollama's local API _OLLAMA_API_TIMEOUT = aiohttp.ClientTimeout(total=8) async def fetch_ollama_models(api_base: str) -> List[str]: """Fetch locally available models from a running Ollama instance. Uses Ollama's OpenAI-compatible ``GET {api_base}/models`` endpoint. Returns an empty list if Ollama is not reachable (not running). """ url = f"{api_base.rstrip('/')}/models" try: async with aiohttp.ClientSession(timeout=_OLLAMA_API_TIMEOUT) as session: async with session.get(url) as resp: if resp.status != 200: logger.debug("Ollama API returned HTTP %s from %s", resp.status, api_base) return [] data = await resp.json() except (aiohttp.ClientError, asyncio.TimeoutError, json.JSONDecodeError) as exc: logger.debug("Ollama not reachable at %s: %s", api_base, exc) return [] raw = data.get("data") if isinstance(data, dict) else None if not isinstance(raw, list): return [] return [ str(entry["id"]) for entry in raw if isinstance(entry, dict) and isinstance(entry.get("id"), str) ] async def get_provider_model_ids(provider_id: str) -> List[str]: """Return the list of known model IDs for *provider_id* from the catalog. The catalog is loaded on first call and cached thereafter. If the provider is not found an empty list is returned (never raises). """ catalog = await _load_model_catalog() return catalog.get(provider_id, []) async def get_all_provider_models( provider_ids: List[str], ) -> Dict[str, List[str]]: """Return model lists for a subset of providers in one call. Loads the catalog (cached) and returns only the requested providers. Handy for embedding lightweight data into the template context. """ catalog = await _load_model_catalog() return { pid: catalog.get(pid, []) for pid in provider_ids } # Provider preset definitions. # Each entry contains display metadata and defaults for the UI. # The key is the internal provider id stored in ``llm_provider``. # Models are NOT listed here — they come from the opencode model catalog at # runtime (see :func:`get_provider_model_ids`). PROVIDER_PRESETS: Dict[str, Dict[str, Any]] = { "openai": { "name": "OpenAI", "api_base": "https://api.openai.com/v1", "requires_key": True, }, "ollama": { "name": "Ollama (local)", "api_base": "http://localhost:11434/v1", "requires_key": False, }, "deepseek": { "name": "DeepSeek", "api_base": "https://api.deepseek.com/v1", "requires_key": True, }, "groq": { "name": "Groq", "api_base": "https://api.groq.com/openai/v1", "requires_key": True, }, "openrouter": { "name": "OpenRouter", "api_base": "https://openrouter.ai/api/v1", "requires_key": True, }, "opencode-go": { "name": "OpenCode Go", "api_base": "https://opencode.ai/zen/go/v1", "requires_key": True, }, # "custom" is handled specially (no preset api_base, requires user input) } # Legacy lookup derived from PROVIDER_PRESETS for backward compat. _PROVIDER_DEFAULTS: Dict[str, str] = { pid: info["api_base"] for pid, info in PROVIDER_PRESETS.items() if info.get("api_base") } # Request timeout for LLM calls (seconds) _LLM_TIMEOUT = aiohttp.ClientTimeout(total=120) class LLMService: """Centralized LLM API client. All agent skills call LLMs through this service so that BYOK config, retry logic, and error handling live in one place. """ _instance: Optional["LLMService"] = None _lock: asyncio.Lock = asyncio.Lock() def __init__(self, settings_service) -> None: self._settings = settings_service # ------------------------------------------------------------------ # Singleton access # ------------------------------------------------------------------ @classmethod async def get_instance(cls) -> "LLMService": """Return the lazily-initialised global ``LLMService`` instance.""" if cls._instance is None: async with cls._lock: if cls._instance is None: from .settings_manager import get_settings_manager cls._instance = cls(get_settings_manager()) # Start preloading the model catalog in the background so # the settings UI never blocks on it. The catalog is # cached after the first fetch (see _load_model_catalog). asyncio.create_task(_load_model_catalog()) return cls._instance @classmethod def reset_instance(cls) -> None: """Reset the cached singleton — primarily for tests.""" cls._instance = None # ------------------------------------------------------------------ # Configuration helpers # ------------------------------------------------------------------ def _get_config(self) -> Dict[str, Any]: """Read the current LLM configuration from settings.""" return { "provider": self._settings.get("llm_provider", "openai"), "api_key": self._settings.get("llm_api_key", ""), "api_base": self._settings.get("llm_api_base", ""), "model": self._settings.get("llm_model", ""), } @staticmethod def _provider_requires_key(provider: str) -> bool: """Return ``False`` when the given provider id does not need an API key.""" preset = PROVIDER_PRESETS.get(provider, {}) return bool(preset.get("requires_key", True)) def is_configured(self) -> bool: """Return ``True`` when the LLM provider is minimally configured. A provider is considered configured when ``llm_model`` is set and an API key is configured for providers that require one (e.g. Ollama does not). """ cfg = self._get_config() has_model = bool(cfg["model"]) has_key = bool(cfg["api_key"]) or not self._provider_requires_key(cfg["provider"]) return has_model and has_key def _resolve_api_base(self, provider: str, api_base: str) -> str: """Resolve the API base URL for the given provider. If ``api_base`` is explicitly set (non-empty), it takes priority. Otherwise the default from :data:`PROVIDER_PRESETS` is used. """ if api_base: return api_base.rstrip("/") return _PROVIDER_DEFAULTS.get(provider, "").rstrip("/") def _build_headers(self, api_key: str) -> Dict[str, str]: """Build HTTP headers for the LLM API request.""" headers = {"Content-Type": "application/json"} if api_key: headers["Authorization"] = f"Bearer {api_key}" return headers def _ensure_configured(self) -> Dict[str, Any]: """Validate configuration and return it, or raise. A provider is considered configured when ``llm_model`` is set and (for non-Ollama) an API key is configured. """ cfg = self._get_config() has_model = bool(cfg["model"]) needs_key = self._provider_requires_key(cfg["provider"]) has_key = bool(cfg["api_key"]) or not needs_key if not (has_model and has_key): parts = [] if not has_model: parts.append("No LLM model specified") if not has_key and needs_key: parts.append("No LLM API key configured") detail = "; ".join(parts) if parts else "LLM provider is not configured" raise LLMNotConfiguredError( f"{detail}. Configure it in Settings → AI Provider." ) return cfg # ------------------------------------------------------------------ # Core API call # ------------------------------------------------------------------ async def chat_completion( self, *, messages: List[Dict[str, str]], model: Optional[str] = None, temperature: float = 0.3, response_format: Optional[Dict[str, Any]] = None, max_tokens: Optional[int] = None, retry_on_rate_limit: bool = True, ) -> Dict[str, Any]: """Call the configured LLM provider's ``/chat/completions`` endpoint. Args: messages: OpenAI-format message list model: Override the configured model name temperature: Sampling temperature response_format: Optional ``{"type": "json_object"}`` for structured output max_tokens: Optional max output tokens retry_on_rate_limit: Retry once after a 429 with backoff Returns: Dict with ``content`` (str), ``usage`` (dict), ``model`` (str) Raises: LLMNotConfiguredError: Provider not enabled / missing config LLMRateLimitError: Rate limited and retry exhausted LLMResponseError: Non-200 response or parse failure """ cfg = self._ensure_configured() api_base = self._resolve_api_base(cfg["provider"], cfg["api_base"]) url = f"{api_base}/chat/completions" model_name = model or cfg["model"] payload: Dict[str, Any] = { "model": model_name, "messages": messages, "temperature": temperature, } if response_format is not None: payload["response_format"] = response_format if max_tokens is not None: payload["max_tokens"] = max_tokens headers = self._build_headers(cfg["api_key"]) attempt = 0 max_attempts = 2 if retry_on_rate_limit else 1 while attempt < max_attempts: attempt += 1 try: async with aiohttp.ClientSession(timeout=_LLM_TIMEOUT) as session: async with session.post( url, json=payload, headers=headers ) as resp: if resp.status == 429: if attempt < max_attempts: retry_after = float( resp.headers.get("Retry-After", "5") ) logger.warning( "LLM rate limited, retrying after %.1fs", retry_after, ) await asyncio.sleep(retry_after) continue raise LLMRateLimitError( f"LLM provider rate limited (HTTP 429)", provider=cfg["provider"], ) if resp.status != 200: body = await resp.text() raise LLMResponseError( f"LLM API returned HTTP {resp.status}: " f"{body[:500]}" ) data = await resp.json() except aiohttp.ClientError as exc: raise LLMResponseError(f"Network error calling LLM API: {exc}") from exc # Parse response try: content = data["choices"][0]["message"]["content"] usage = data.get("usage", {}) return { "content": content, "usage": usage, "model": data.get("model", model_name), } except (KeyError, IndexError) as exc: raise LLMResponseError( f"Unexpected LLM response structure: {json.dumps(data)[:500]}" ) from exc # Should not reach here, but satisfy type checker raise LLMRateLimitError("Rate limit retry exhausted", provider=cfg["provider"]) # ------------------------------------------------------------------ # Structured output convenience # ------------------------------------------------------------------ async def chat_completion_json( self, *, system_prompt: str, user_prompt: str, model: Optional[str] = None, temperature: float = 0.3, max_tokens: Optional[int] = None, ) -> Dict[str, Any]: """Call the LLM and return parsed JSON. Sends ``response_format: {"type": "json_object"}`` when the provider supports it, and parses the response content as JSON. If parsing fails, retries once with a clarifying system message. Args: system_prompt: System-level instructions user_prompt: User-level query model: Override the configured model name temperature: Sampling temperature max_tokens: Optional max output tokens Returns: Parsed JSON dict from the LLM response Raises: LLMNotConfiguredError: Provider not configured LLMRateLimitError: Rate limited LLMResponseError: JSON parse failure after retry """ messages = [ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}, ] # First attempt with JSON mode result = await self.chat_completion( messages=messages, model=model, temperature=temperature, response_format={"type": "json_object"}, max_tokens=max_tokens, ) try: return json.loads(result["content"]) except (json.JSONDecodeError, TypeError) as exc: logger.warning( "LLM JSON parse failed on first attempt: %s. Retrying.", exc ) # Retry with explicit instruction to return valid JSON retry_messages = messages + [ { "role": "assistant", "content": result["content"], }, { "role": "user", "content": ( "The previous response could not be parsed as JSON. " "Please respond with ONLY a valid JSON object, no " "markdown fences or extra text." ), }, ] result = await self.chat_completion( messages=retry_messages, model=model, temperature=0.0, # More deterministic for retry response_format={"type": "json_object"}, max_tokens=max_tokens, ) try: return json.loads(result["content"]) except (json.JSONDecodeError, TypeError) as exc: raise LLMResponseError( f"LLM response could not be parsed as JSON after retry: {exc}\n" f"Raw content: {result['content'][:500]}" ) from exc