feat(llm): multi-provider Open Responses support for user simulator

CHANGELOG.md

@@ -45,6 +45,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * **evaluator:** error-to-scenario mappings included in `evaluation.json` output
 * **evaluator:** shared evaluation constants (`SCORE_NOT_COMPUTED`, `BEHAVIOR_FAILURE_THRESHOLD`)
 * **a2a:** add tool call capture via A2A AgentExtension (`https://arksim.arklex.ai/a2a/tool-call-capture/v1`); agents declare the extension in the AgentCard and surface tool calls in `Task.artifacts[*].metadata`. See [Tool Call Capture docs](https://docs.arklex.ai/main/tool-call-capture)
+* **llm:** user simulator can now run on any Open Responses-conforming backend via `provider: responses` with configurable `base_url` and `api_key`. Works with OpenAI, Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter, LM Studio, and other consortium backers. The OpenAI provider has run on `/v1/responses` since v0.3.0; this change unlocks multi-provider reach.
+* **llm:** per-instance cache-hit telemetry on the OpenAI provider via `OpenAILLM.cache_stats()`. `run_simulation` and `run_evaluation` log a one-line summary at end of run (call count, input tokens, cached tokens, cache hit rate, output tokens) so users can verify the documented prompt-caching cost reduction without leaving arksim. Tolerates missing `usage` fields from non-OpenAI backends (Ollama, vLLM).
+* **evaluator:** add `evaluator_model`, `evaluator_provider`, `evaluator_base_url`, and `evaluator_api_key` to split simulator and evaluator backends in a single `config.yaml`. When the evaluator endpoint differs from the simulator endpoint, shared `base_url`/`api_key` are not forwarded to prevent credential leaks; specify evaluator-side credentials explicitly or rely on SDK env-var fallback.
 * **report:** scenario IDs displayed in HTML report error cards
 * **ui:** version display in web UI sidebar (next to "Arksim" title)
 * **api:** path traversal protection on filesystem and results API endpoints

arksim/evaluator/entities.py

@@ -29,6 +29,31 @@
 
 _entities_logger = logging.getLogger(__name__)
 
+
+def _norm_endpoint_value(value: str | None) -> str | None:
+    """Normalize an endpoint field for comparison.
+
+    Returns the stripped value when non-empty after stripping; otherwise
+    ``None``. This treats ``None``, ``""``, and whitespace-only as
+    equivalent ("unset"), matching the whitespace-stripping behavior of
+    the OpenAI provider so users do not silently trigger endpoint-split
+    semantics by typing trailing whitespace.
+    """
+    if value is None:
+        return None
+    stripped = value.strip()
+    return stripped if stripped else None
+
+
+# Canonical LLM kwarg names returned by ``EvaluationInput.evaluator_llm_kwargs``.
+# Adding a new field here requires also adding a matching ``evaluator_<field>``
+# attribute to ``EvaluationInput``; the same-endpoint branch of the helper
+# iterates this tuple so it picks the new field up automatically. The
+# endpoint-differs branch is intentionally explicit because ``base_url`` and
+# ``api_key`` have special "do not cross-inherit" credential semantics.
+_LLM_KWARG_FIELDS: tuple[str, ...] = ("model", "provider", "base_url", "api_key")
+
+
 _DEFAULT_METRICS_TO_RUN = [
     "faithfulness",
     "helpfulness",
@@ -57,6 +82,56 @@ class EvaluationInput(BaseModel):
     )
     model: str = Field(default=DEFAULT_MODEL, description="LLM model for evaluation")
     provider: str | None = Field(default=DEFAULT_PROVIDER, description="LLM provider")
+    base_url: str | None = Field(
+        default=None,
+        description=(
+            "Optional custom endpoint for the evaluator LLM. Use to point at any "
+            "Open Responses-conforming backend (Ollama, vLLM, NVIDIA NIM, Vercel AI "
+            "Gateway, OpenRouter, etc.). Defaults to the SDK's env var "
+            "(OPENAI_BASE_URL) or the canonical provider endpoint."
+        ),
+    )
+    api_key: str | None = Field(
+        default=None,
+        description=(
+            "Optional API key for the evaluator LLM. Pair with base_url for "
+            "self-hosted or alternate Responses backends. Defaults to the SDK's "
+            "env var (OPENAI_API_KEY)."
+        ),
+    )
+    evaluator_model: str | None = Field(
+        default=None,
+        description=(
+            "Optional override for the evaluator's LLM model. When unset, "
+            "falls back to `model`."
+        ),
+    )
+    evaluator_provider: str | None = Field(
+        default=None,
+        description=(
+            "Optional override for the evaluator's LLM provider. Useful when "
+            "the simulator runs on a backend that does not support structured "
+            "output, while the evaluator needs OpenAI-compatible `text_format` "
+            "support. When set with a value that differs from `provider`, the "
+            "shared `base_url` and `api_key` are NOT forwarded; specify "
+            "`evaluator_base_url` / `evaluator_api_key` explicitly or rely on "
+            "SDK env-var fallback."
+        ),
+    )
+    evaluator_base_url: str | None = Field(
+        default=None,
+        description=(
+            "Optional override for the evaluator LLM's custom endpoint. "
+            "When unset, falls back to `base_url`."
+        ),
+    )
+    evaluator_api_key: str | None = Field(
+        default=None,
+        description=(
+            "Optional override for the evaluator LLM's API key. When unset, "
+            "falls back to `api_key`."
+        ),
+    )
     num_workers: int | str = Field(
         default=50,
         description="Number of parallel workers (use 'auto' to default to 4)",
@@ -151,6 +226,104 @@ def validate_evaluation_input(self, info: ValidationInfo) -> Self:
 
         return self
 
+    def _evaluator_endpoint_differs(self) -> bool:
+        """Return True when the evaluator's endpoint *actually differs* from
+        the simulator's. Whitespace-only override values are treated as unset
+        to match the whitespace-stripping behavior of the OpenAI provider.
+
+        A redundant override (e.g. ``evaluator_provider`` equal to the
+        shared ``provider``) is NOT a real endpoint split, so the shared
+        ``api_key`` / ``base_url`` continue to apply. This avoids the
+        footgun where typing the same provider twice silently drops the
+        shared credentials and crashes the evaluator at run time.
+        """
+        eval_provider = _norm_endpoint_value(self.evaluator_provider)
+        eval_base_url = _norm_endpoint_value(self.evaluator_base_url)
+        if eval_provider is not None and eval_provider != _norm_endpoint_value(
+            self.provider
+        ):
+            return True
+        return eval_base_url is not None and eval_base_url != _norm_endpoint_value(
+            self.base_url
+        )
+
+    def evaluator_llm_kwargs(self) -> dict[str, str | None]:
+        """Resolve LLM kwargs for the evaluator, with safe fallback semantics.
+
+        - `evaluator_model` individually overrides `model` when set; otherwise
+          the evaluator uses the shared `model`.
+        - When the evaluator endpoint differs from the simulator endpoint
+          (`evaluator_provider` set OR `evaluator_base_url` set), the
+          credentials and base_url DO NOT cross-inherit from the shared
+          keys. They must be set explicitly via `evaluator_base_url` /
+          `evaluator_api_key`, or fall through to the SDK's env-var
+          defaults (`OPENAI_BASE_URL` / `OPENAI_API_KEY`).
+        - This prevents accidentally sending a credential intended for one
+          endpoint to another (e.g. forwarding a live OpenAI key to a
+          self-hosted Ollama endpoint).
+        - Whitespace-only override values are treated as unset.
+        """
+        if self._evaluator_endpoint_differs():
+            # Endpoint split: ``model`` and ``provider`` can fall back to the
+            # shared values (neither is credential-bearing), but ``base_url``
+            # and ``api_key`` must NOT cross-inherit. Leave them as-is so the
+            # SDK env-var fallback (e.g. ``OPENAI_API_KEY``) applies.
+            return {
+                "model": self.evaluator_model or self.model,
+                "provider": self.evaluator_provider or self.provider,
+                "base_url": self.evaluator_base_url,
+                "api_key": self.evaluator_api_key,
+            }
+
+        # Same endpoint: every field inherits from the shared key when the
+        # matching ``evaluator_<field>`` is unset (None, empty, or
+        # whitespace-only / invisible-only after normalization). Iterating
+        # ``_LLM_KWARG_FIELDS`` keeps this branch in sync with future kwarg
+        # additions automatically.
+        return {
+            field: _norm_endpoint_value(getattr(self, f"evaluator_{field}"))
+            or getattr(self, field)
+            for field in _LLM_KWARG_FIELDS
+        }
+
+    @model_validator(mode="after")
+    def _log_evaluator_endpoint_split(self) -> Self:
+        """Emit diagnostics for the two endpoint-split footguns:
+
+        1. INFO: evaluator endpoint differs from simulator but no
+           ``evaluator_api_key`` is set. Warns the user that the shared
+           ``api_key`` will NOT be forwarded; the evaluator falls back to
+           SDK env-var defaults instead.
+        2. WARNING: ``evaluator_api_key`` is set but the evaluator endpoint
+           does NOT differ from the simulator endpoint. In that case the
+           shared ``api_key`` is used and ``evaluator_api_key`` is silently
+           ignored. Warn so the user does not assume the override took effect.
+        """
+        endpoint_differs = self._evaluator_endpoint_differs()
+        if endpoint_differs and self.evaluator_api_key is None:
+            _entities_logger.info(
+                "Evaluator endpoint differs from simulator "
+                "(evaluator_provider=%s, evaluator_base_url=%s). "
+                "evaluator_api_key is unset; falling back to SDK env-var "
+                "defaults (e.g. OPENAI_API_KEY). The shared `api_key` will "
+                "NOT be forwarded to the evaluator endpoint.",
+                _norm_endpoint_value(self.evaluator_provider),
+                _norm_endpoint_value(self.evaluator_base_url),
+            )
+        if (
+            not endpoint_differs
+            and self.evaluator_api_key is not None
+            and self.evaluator_api_key.strip()
+        ):
+            _entities_logger.warning(
+                "evaluator_api_key is set but the evaluator endpoint matches "
+                "the simulator endpoint, so the shared api_key is used and "
+                "evaluator_api_key is ignored. To use a different key, also "
+                "set evaluator_provider or evaluator_base_url to point at a "
+                "different endpoint."
+            )
+        return self
+
 
 class EvaluationParams(BaseModel):
     """Parameters for the evaluation engine."""

arksim/evaluator/evaluator.py

@@ -14,6 +14,7 @@
 from tqdm import tqdm
 
 from arksim.llms.chat import LLM
+from arksim.llms.chat.utils import log_cache_stats
 from arksim.scenario import Scenarios
 from arksim.scenario.entities import AssertionType, ExpectedToolCall
 
@@ -938,10 +939,13 @@ def run_evaluation(
                 "scenario context will be unavailable"
             )
 
-    llm = LLM(
-        model=settings.model,
-        provider=settings.provider,
-    )
+    # See EvaluationInput.evaluator_llm_kwargs() for the per-group fallback
+    # rule. When evaluator endpoint differs from simulator, base_url and
+    # api_key do not cross-inherit from shared keys; they must be set
+    # explicitly or fall back to SDK env vars. This prevents credential
+    # leaks across endpoints.
+    evaluator_kwargs = settings.evaluator_llm_kwargs()
+    llm = LLM(**evaluator_kwargs)
     all_quant, all_qual = _load_custom_metrics(
         settings.custom_metrics_file_paths, llm=llm
     )
@@ -961,6 +965,8 @@ def run_evaluation(
     evaluator_output = evaluator.evaluate(simulation, on_progress=on_progress)
     evaluator.save_results()
     evaluator.display_evaluation_summary()
+    # Surface cache-hit telemetry from the evaluator LLM.
+    log_cache_stats(llm, phase="evaluation")
 
     if settings.generate_html_report:
         from arksim.utils.html_report.generate_html_report import (
@@ -986,8 +992,8 @@ def run_evaluation(
             metric_descriptions=metric_descriptions,
             metric_ranges=metric_ranges,
             qual_label_colors=qual_label_colors,
-            evaluation_model=settings.model,
-            evaluation_provider=settings.provider,
+            evaluation_model=evaluator_kwargs["model"],
+            evaluation_provider=evaluator_kwargs["provider"],
         )
         generate_html_report(report_params)
 

arksim/llms/chat/llm.py

@@ -23,7 +23,10 @@ def __new__(cls, model: str, **kwargs: object) -> BaseLLM:
 
     @classmethod
     def _get_provider(cls, provider: str) -> type:
-        if provider == "openai":
+        if provider in ("openai", "responses"):
+            # `responses` is a documentation-only alias signaling multi-provider
+            # intent (Open Responses spec). Both names resolve to OpenAILLM.
+            # See docs/main/user-simulator-on-open-responses.mdx.
             from arksim.llms.chat.providers.openai import OpenAILLM
 
             return OpenAILLM

arksim/llms/chat/providers/openai.py

@@ -3,7 +3,7 @@
 
 from typing import Any, TypeVar, overload
 
-from openai import AsyncOpenAI, OpenAI
+from openai import AsyncOpenAI, OpenAI, OpenAIError
 from pydantic import BaseModel
 
 from arksim.llms.chat.base.base_llm import BaseLLM
@@ -19,11 +19,80 @@ def __init__(
         model: str,
         provider: str | None = None,
         temperature: float | None = None,
+        base_url: str | None = None,
+        api_key: str | None = None,
         **kwargs: object,
     ) -> None:
         super().__init__(model, provider, temperature, **kwargs)
-        self.client = OpenAI()
-        self.async_client = AsyncOpenAI()
+        # Coerce empty or whitespace-only strings to omitted, then fall back
+        # to OPENAI_BASE_URL / OPENAI_API_KEY env vars. YAML configs commonly
+        # produce "" for unset fields; whitespace-only values would otherwise
+        # be URL-encoded by the SDK and cause hard-to-diagnose hangs.
+        client_kwargs: dict[str, str] = {}
+        if base_url and base_url.strip():
+            client_kwargs["base_url"] = base_url.strip()
+        if api_key and api_key.strip():
+            client_kwargs["api_key"] = api_key.strip()
+        try:
+            self.client = OpenAI(**client_kwargs)
+            self.async_client = AsyncOpenAI(**client_kwargs)
+        except OpenAIError as exc:
+            raise ValueError(
+                "OpenAI SDK requires an api_key. Set 'api_key' in your "
+                "config.yaml (alongside 'model' and 'provider') or set the "
+                "OPENAI_API_KEY environment variable. "
+                "See https://docs.arklex.ai/main/user-simulator-on-open-responses"
+            ) from exc
+
+        # Per-instance cumulative usage counters. Read via `cache_stats()`.
+        # Surfaces OpenAI's automatic prompt-cache hits so users can verify
+        # the cost-reduction story without leaving arksim.
+        self._cumulative_input_tokens: int = 0
+        self._cumulative_cached_input_tokens: int = 0
+        self._cumulative_output_tokens: int = 0
+        self._call_count: int = 0
+
+    def _record_usage(self, response: object) -> None:
+        """Accumulate token-usage counters from a Responses API response.
+
+        Tolerates missing fields: older SDK versions or non-OpenAI backends
+        (Ollama, vLLM) may omit `cached_tokens` or even the full `usage`
+        object. We never raise from telemetry.
+        """
+        usage = getattr(response, "usage", None)
+        if usage is None:
+            return
+        self._call_count += 1
+        self._cumulative_input_tokens += getattr(usage, "input_tokens", 0) or 0
+        self._cumulative_output_tokens += getattr(usage, "output_tokens", 0) or 0
+        details = getattr(usage, "input_tokens_details", None)
+        if details is not None:
+            self._cumulative_cached_input_tokens += (
+                getattr(details, "cached_tokens", 0) or 0
+            )
+
+    def cache_stats(self) -> dict[str, int | float]:
+        """Return cumulative cache-hit statistics across all calls.
+
+        Keys:
+        - ``call_count``: total responses.parse calls observed
+        - ``input_tokens``: total input tokens billed
+        - ``cached_input_tokens``: subset of input_tokens that hit cache
+        - ``output_tokens``: total output tokens
+        - ``cache_hit_rate``: cached_input_tokens / input_tokens (0.0 if no calls)
+        """
+        rate = (
+            self._cumulative_cached_input_tokens / self._cumulative_input_tokens
+            if self._cumulative_input_tokens > 0
+            else 0.0
+        )
+        return {
+            "call_count": self._call_count,
+            "input_tokens": self._cumulative_input_tokens,
+            "cached_input_tokens": self._cumulative_cached_input_tokens,
+            "output_tokens": self._cumulative_output_tokens,
+            "cache_hit_rate": rate,
+        }
 
     def _prepare_params(
         self,
@@ -68,6 +137,7 @@ def call(
     ) -> T | str:
         params = self._prepare_params(messages, schema=schema)
         response = self.client.responses.parse(**params)
+        self._record_usage(response)
         # For structured output, return the parsed output
         if schema:
             return response.output_parsed
@@ -93,6 +163,7 @@ async def call_async(
     ) -> T | str:
         params = self._prepare_params(messages, schema=schema)
         response = await self.async_client.responses.parse(**params)
+        self._record_usage(response)
         # For structured output, return the parsed output
         if schema:
             return response.output_parsed