diff --git a/CHANGELOG.md b/CHANGELOG.md index 85e4d2c5..262b8877 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * **evaluator:** error-to-scenario mappings included in `evaluation.json` output * **evaluator:** shared evaluation constants (`SCORE_NOT_COMPUTED`, `BEHAVIOR_FAILURE_THRESHOLD`) * **a2a:** add tool call capture via A2A AgentExtension (`https://arksim.arklex.ai/a2a/tool-call-capture/v1`); agents declare the extension in the AgentCard and surface tool calls in `Task.artifacts[*].metadata`. See [Tool Call Capture docs](https://docs.arklex.ai/main/tool-call-capture) +* **llm:** user simulator can now run on any Open Responses-conforming backend via `provider: responses` with configurable `base_url` and `api_key`. Works with OpenAI, Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter, LM Studio, and other consortium backers. The OpenAI provider has run on `/v1/responses` since v0.3.0; this change unlocks multi-provider reach. +* **llm:** per-instance cache-hit telemetry on the OpenAI provider via `OpenAILLM.cache_stats()`. `run_simulation` and `run_evaluation` log a one-line summary at end of run (call count, input tokens, cached tokens, cache hit rate, output tokens) so users can verify the documented prompt-caching cost reduction without leaving arksim. Tolerates missing `usage` fields from non-OpenAI backends (Ollama, vLLM). +* **evaluator:** add `evaluator_model`, `evaluator_provider`, `evaluator_base_url`, and `evaluator_api_key` to split simulator and evaluator backends in a single `config.yaml`. When the evaluator endpoint differs from the simulator endpoint, shared `base_url`/`api_key` are not forwarded to prevent credential leaks; specify evaluator-side credentials explicitly or rely on SDK env-var fallback. * **report:** scenario IDs displayed in HTML report error cards * **ui:** version display in web UI sidebar (next to "Arksim" title) * **api:** path traversal protection on filesystem and results API endpoints diff --git a/arksim/evaluator/entities.py b/arksim/evaluator/entities.py index ff3af7db..cf5e0550 100644 --- a/arksim/evaluator/entities.py +++ b/arksim/evaluator/entities.py @@ -29,6 +29,31 @@ _entities_logger = logging.getLogger(__name__) + +def _norm_endpoint_value(value: str | None) -> str | None: + """Normalize an endpoint field for comparison. + + Returns the stripped value when non-empty after stripping; otherwise + ``None``. This treats ``None``, ``""``, and whitespace-only as + equivalent ("unset"), matching the whitespace-stripping behavior of + the OpenAI provider so users do not silently trigger endpoint-split + semantics by typing trailing whitespace. + """ + if value is None: + return None + stripped = value.strip() + return stripped if stripped else None + + +# Canonical LLM kwarg names returned by ``EvaluationInput.evaluator_llm_kwargs``. +# Adding a new field here requires also adding a matching ``evaluator_`` +# attribute to ``EvaluationInput``; the same-endpoint branch of the helper +# iterates this tuple so it picks the new field up automatically. The +# endpoint-differs branch is intentionally explicit because ``base_url`` and +# ``api_key`` have special "do not cross-inherit" credential semantics. +_LLM_KWARG_FIELDS: tuple[str, ...] = ("model", "provider", "base_url", "api_key") + + _DEFAULT_METRICS_TO_RUN = [ "faithfulness", "helpfulness", @@ -57,6 +82,56 @@ class EvaluationInput(BaseModel): ) model: str = Field(default=DEFAULT_MODEL, description="LLM model for evaluation") provider: str | None = Field(default=DEFAULT_PROVIDER, description="LLM provider") + base_url: str | None = Field( + default=None, + description=( + "Optional custom endpoint for the evaluator LLM. Use to point at any " + "Open Responses-conforming backend (Ollama, vLLM, NVIDIA NIM, Vercel AI " + "Gateway, OpenRouter, etc.). Defaults to the SDK's env var " + "(OPENAI_BASE_URL) or the canonical provider endpoint." + ), + ) + api_key: str | None = Field( + default=None, + description=( + "Optional API key for the evaluator LLM. Pair with base_url for " + "self-hosted or alternate Responses backends. Defaults to the SDK's " + "env var (OPENAI_API_KEY)." + ), + ) + evaluator_model: str | None = Field( + default=None, + description=( + "Optional override for the evaluator's LLM model. When unset, " + "falls back to `model`." + ), + ) + evaluator_provider: str | None = Field( + default=None, + description=( + "Optional override for the evaluator's LLM provider. Useful when " + "the simulator runs on a backend that does not support structured " + "output, while the evaluator needs OpenAI-compatible `text_format` " + "support. When set with a value that differs from `provider`, the " + "shared `base_url` and `api_key` are NOT forwarded; specify " + "`evaluator_base_url` / `evaluator_api_key` explicitly or rely on " + "SDK env-var fallback." + ), + ) + evaluator_base_url: str | None = Field( + default=None, + description=( + "Optional override for the evaluator LLM's custom endpoint. " + "When unset, falls back to `base_url`." + ), + ) + evaluator_api_key: str | None = Field( + default=None, + description=( + "Optional override for the evaluator LLM's API key. When unset, " + "falls back to `api_key`." + ), + ) num_workers: int | str = Field( default=50, description="Number of parallel workers (use 'auto' to default to 4)", @@ -151,6 +226,104 @@ def validate_evaluation_input(self, info: ValidationInfo) -> Self: return self + def _evaluator_endpoint_differs(self) -> bool: + """Return True when the evaluator's endpoint *actually differs* from + the simulator's. Whitespace-only override values are treated as unset + to match the whitespace-stripping behavior of the OpenAI provider. + + A redundant override (e.g. ``evaluator_provider`` equal to the + shared ``provider``) is NOT a real endpoint split, so the shared + ``api_key`` / ``base_url`` continue to apply. This avoids the + footgun where typing the same provider twice silently drops the + shared credentials and crashes the evaluator at run time. + """ + eval_provider = _norm_endpoint_value(self.evaluator_provider) + eval_base_url = _norm_endpoint_value(self.evaluator_base_url) + if eval_provider is not None and eval_provider != _norm_endpoint_value( + self.provider + ): + return True + return eval_base_url is not None and eval_base_url != _norm_endpoint_value( + self.base_url + ) + + def evaluator_llm_kwargs(self) -> dict[str, str | None]: + """Resolve LLM kwargs for the evaluator, with safe fallback semantics. + + - `evaluator_model` individually overrides `model` when set; otherwise + the evaluator uses the shared `model`. + - When the evaluator endpoint differs from the simulator endpoint + (`evaluator_provider` set OR `evaluator_base_url` set), the + credentials and base_url DO NOT cross-inherit from the shared + keys. They must be set explicitly via `evaluator_base_url` / + `evaluator_api_key`, or fall through to the SDK's env-var + defaults (`OPENAI_BASE_URL` / `OPENAI_API_KEY`). + - This prevents accidentally sending a credential intended for one + endpoint to another (e.g. forwarding a live OpenAI key to a + self-hosted Ollama endpoint). + - Whitespace-only override values are treated as unset. + """ + if self._evaluator_endpoint_differs(): + # Endpoint split: ``model`` and ``provider`` can fall back to the + # shared values (neither is credential-bearing), but ``base_url`` + # and ``api_key`` must NOT cross-inherit. Leave them as-is so the + # SDK env-var fallback (e.g. ``OPENAI_API_KEY``) applies. + return { + "model": self.evaluator_model or self.model, + "provider": self.evaluator_provider or self.provider, + "base_url": self.evaluator_base_url, + "api_key": self.evaluator_api_key, + } + + # Same endpoint: every field inherits from the shared key when the + # matching ``evaluator_`` is unset (None, empty, or + # whitespace-only / invisible-only after normalization). Iterating + # ``_LLM_KWARG_FIELDS`` keeps this branch in sync with future kwarg + # additions automatically. + return { + field: _norm_endpoint_value(getattr(self, f"evaluator_{field}")) + or getattr(self, field) + for field in _LLM_KWARG_FIELDS + } + + @model_validator(mode="after") + def _log_evaluator_endpoint_split(self) -> Self: + """Emit diagnostics for the two endpoint-split footguns: + + 1. INFO: evaluator endpoint differs from simulator but no + ``evaluator_api_key`` is set. Warns the user that the shared + ``api_key`` will NOT be forwarded; the evaluator falls back to + SDK env-var defaults instead. + 2. WARNING: ``evaluator_api_key`` is set but the evaluator endpoint + does NOT differ from the simulator endpoint. In that case the + shared ``api_key`` is used and ``evaluator_api_key`` is silently + ignored. Warn so the user does not assume the override took effect. + """ + endpoint_differs = self._evaluator_endpoint_differs() + if endpoint_differs and self.evaluator_api_key is None: + _entities_logger.info( + "Evaluator endpoint differs from simulator " + "(evaluator_provider=%s, evaluator_base_url=%s). " + "evaluator_api_key is unset; falling back to SDK env-var " + "defaults (e.g. OPENAI_API_KEY). The shared `api_key` will " + "NOT be forwarded to the evaluator endpoint.", + _norm_endpoint_value(self.evaluator_provider), + _norm_endpoint_value(self.evaluator_base_url), + ) + if ( + not endpoint_differs + and self.evaluator_api_key is not None + and self.evaluator_api_key.strip() + ): + _entities_logger.warning( + "evaluator_api_key is set but the evaluator endpoint matches " + "the simulator endpoint, so the shared api_key is used and " + "evaluator_api_key is ignored. To use a different key, also " + "set evaluator_provider or evaluator_base_url to point at a " + "different endpoint." + ) + return self + class EvaluationParams(BaseModel): """Parameters for the evaluation engine.""" diff --git a/arksim/evaluator/evaluator.py b/arksim/evaluator/evaluator.py index 87d7a687..86d094a3 100644 --- a/arksim/evaluator/evaluator.py +++ b/arksim/evaluator/evaluator.py @@ -14,6 +14,7 @@ from tqdm import tqdm from arksim.llms.chat import LLM +from arksim.llms.chat.utils import log_cache_stats from arksim.scenario import Scenarios from arksim.scenario.entities import AssertionType, ExpectedToolCall @@ -938,10 +939,13 @@ def run_evaluation( "scenario context will be unavailable" ) - llm = LLM( - model=settings.model, - provider=settings.provider, - ) + # See EvaluationInput.evaluator_llm_kwargs() for the per-group fallback + # rule. When evaluator endpoint differs from simulator, base_url and + # api_key do not cross-inherit from shared keys; they must be set + # explicitly or fall back to SDK env vars. This prevents credential + # leaks across endpoints. + evaluator_kwargs = settings.evaluator_llm_kwargs() + llm = LLM(**evaluator_kwargs) all_quant, all_qual = _load_custom_metrics( settings.custom_metrics_file_paths, llm=llm ) @@ -961,6 +965,8 @@ def run_evaluation( evaluator_output = evaluator.evaluate(simulation, on_progress=on_progress) evaluator.save_results() evaluator.display_evaluation_summary() + # Surface cache-hit telemetry from the evaluator LLM. + log_cache_stats(llm, phase="evaluation") if settings.generate_html_report: from arksim.utils.html_report.generate_html_report import ( @@ -986,8 +992,8 @@ def run_evaluation( metric_descriptions=metric_descriptions, metric_ranges=metric_ranges, qual_label_colors=qual_label_colors, - evaluation_model=settings.model, - evaluation_provider=settings.provider, + evaluation_model=evaluator_kwargs["model"], + evaluation_provider=evaluator_kwargs["provider"], ) generate_html_report(report_params) diff --git a/arksim/llms/chat/llm.py b/arksim/llms/chat/llm.py index edcf51d1..175354ae 100644 --- a/arksim/llms/chat/llm.py +++ b/arksim/llms/chat/llm.py @@ -23,7 +23,10 @@ def __new__(cls, model: str, **kwargs: object) -> BaseLLM: @classmethod def _get_provider(cls, provider: str) -> type: - if provider == "openai": + if provider in ("openai", "responses"): + # `responses` is a documentation-only alias signaling multi-provider + # intent (Open Responses spec). Both names resolve to OpenAILLM. + # See docs/main/user-simulator-on-open-responses.mdx. from arksim.llms.chat.providers.openai import OpenAILLM return OpenAILLM diff --git a/arksim/llms/chat/providers/openai.py b/arksim/llms/chat/providers/openai.py index 6ab74688..39ac73f3 100644 --- a/arksim/llms/chat/providers/openai.py +++ b/arksim/llms/chat/providers/openai.py @@ -3,7 +3,7 @@ from typing import Any, TypeVar, overload -from openai import AsyncOpenAI, OpenAI +from openai import AsyncOpenAI, OpenAI, OpenAIError from pydantic import BaseModel from arksim.llms.chat.base.base_llm import BaseLLM @@ -19,11 +19,80 @@ def __init__( model: str, provider: str | None = None, temperature: float | None = None, + base_url: str | None = None, + api_key: str | None = None, **kwargs: object, ) -> None: super().__init__(model, provider, temperature, **kwargs) - self.client = OpenAI() - self.async_client = AsyncOpenAI() + # Coerce empty or whitespace-only strings to omitted, then fall back + # to OPENAI_BASE_URL / OPENAI_API_KEY env vars. YAML configs commonly + # produce "" for unset fields; whitespace-only values would otherwise + # be URL-encoded by the SDK and cause hard-to-diagnose hangs. + client_kwargs: dict[str, str] = {} + if base_url and base_url.strip(): + client_kwargs["base_url"] = base_url.strip() + if api_key and api_key.strip(): + client_kwargs["api_key"] = api_key.strip() + try: + self.client = OpenAI(**client_kwargs) + self.async_client = AsyncOpenAI(**client_kwargs) + except OpenAIError as exc: + raise ValueError( + "OpenAI SDK requires an api_key. Set 'api_key' in your " + "config.yaml (alongside 'model' and 'provider') or set the " + "OPENAI_API_KEY environment variable. " + "See https://docs.arklex.ai/main/user-simulator-on-open-responses" + ) from exc + + # Per-instance cumulative usage counters. Read via `cache_stats()`. + # Surfaces OpenAI's automatic prompt-cache hits so users can verify + # the cost-reduction story without leaving arksim. + self._cumulative_input_tokens: int = 0 + self._cumulative_cached_input_tokens: int = 0 + self._cumulative_output_tokens: int = 0 + self._call_count: int = 0 + + def _record_usage(self, response: object) -> None: + """Accumulate token-usage counters from a Responses API response. + + Tolerates missing fields: older SDK versions or non-OpenAI backends + (Ollama, vLLM) may omit `cached_tokens` or even the full `usage` + object. We never raise from telemetry. + """ + usage = getattr(response, "usage", None) + if usage is None: + return + self._call_count += 1 + self._cumulative_input_tokens += getattr(usage, "input_tokens", 0) or 0 + self._cumulative_output_tokens += getattr(usage, "output_tokens", 0) or 0 + details = getattr(usage, "input_tokens_details", None) + if details is not None: + self._cumulative_cached_input_tokens += ( + getattr(details, "cached_tokens", 0) or 0 + ) + + def cache_stats(self) -> dict[str, int | float]: + """Return cumulative cache-hit statistics across all calls. + + Keys: + - ``call_count``: total responses.parse calls observed + - ``input_tokens``: total input tokens billed + - ``cached_input_tokens``: subset of input_tokens that hit cache + - ``output_tokens``: total output tokens + - ``cache_hit_rate``: cached_input_tokens / input_tokens (0.0 if no calls) + """ + rate = ( + self._cumulative_cached_input_tokens / self._cumulative_input_tokens + if self._cumulative_input_tokens > 0 + else 0.0 + ) + return { + "call_count": self._call_count, + "input_tokens": self._cumulative_input_tokens, + "cached_input_tokens": self._cumulative_cached_input_tokens, + "output_tokens": self._cumulative_output_tokens, + "cache_hit_rate": rate, + } def _prepare_params( self, @@ -68,6 +137,7 @@ def call( ) -> T | str: params = self._prepare_params(messages, schema=schema) response = self.client.responses.parse(**params) + self._record_usage(response) # For structured output, return the parsed output if schema: return response.output_parsed @@ -93,6 +163,7 @@ async def call_async( ) -> T | str: params = self._prepare_params(messages, schema=schema) response = await self.async_client.responses.parse(**params) + self._record_usage(response) # For structured output, return the parsed output if schema: return response.output_parsed diff --git a/arksim/llms/chat/utils.py b/arksim/llms/chat/utils.py index 0ac13f20..a71bea4c 100644 --- a/arksim/llms/chat/utils.py +++ b/arksim/llms/chat/utils.py @@ -79,3 +79,30 @@ def sync_wrapper(*args: object, **kwargs: object) -> object: return sync_wrapper return decorator + + +def log_cache_stats(llm: object, phase: str) -> None: + """Log a one-line summary of LLM cache-hit stats if available. + + Tolerates LLM instances that do not implement ``cache_stats`` (e.g. + Anthropic, Google providers) and never raises from telemetry. + """ + stats_fn = getattr(llm, "cache_stats", None) + if stats_fn is None: + return + try: + stats = stats_fn() + except Exception: + return + if stats.get("call_count", 0) == 0: + return + logger.info( + "%s LLM cache stats: %d calls, %d input tokens, " + "%d cached (%.1f%% hit rate), %d output tokens", + phase, + stats["call_count"], + stats["input_tokens"], + stats["cached_input_tokens"], + stats["cache_hit_rate"] * 100, + stats["output_tokens"], + ) diff --git a/arksim/simulation_engine/entities.py b/arksim/simulation_engine/entities.py index 22ac0fda..bf45e193 100644 --- a/arksim/simulation_engine/entities.py +++ b/arksim/simulation_engine/entities.py @@ -41,6 +41,23 @@ class SimulationInput(BaseModel): ) model: str = Field(default=DEFAULT_MODEL, description="LLM model for simulation") provider: str | None = Field(default=DEFAULT_PROVIDER, description="LLM provider") + base_url: str | None = Field( + default=None, + description=( + "Optional custom endpoint for the simulator LLM. Use to point at any " + "Open Responses-conforming backend (Ollama, vLLM, NVIDIA NIM, Vercel AI " + "Gateway, OpenRouter, etc.). Defaults to the SDK's env var " + "(OPENAI_BASE_URL) or the canonical provider endpoint." + ), + ) + api_key: str | None = Field( + default=None, + description=( + "Optional API key for the simulator LLM. Pair with base_url for " + "self-hosted or alternate Responses backends. Defaults to the SDK's " + "env var (OPENAI_API_KEY)." + ), + ) num_conversations_per_scenario: int = Field( default=5, description="Number of conversations per scenario to simulate" ) diff --git a/arksim/simulation_engine/simulator.py b/arksim/simulation_engine/simulator.py index 206799bc..096fc2f8 100644 --- a/arksim/simulation_engine/simulator.py +++ b/arksim/simulation_engine/simulator.py @@ -14,6 +14,7 @@ from arksim.config import AgentConfig, AgentType from arksim.llms.chat import LLM +from arksim.llms.chat.utils import log_cache_stats from arksim.scenario import ( KnowledgeItem, Scenarios, @@ -486,6 +487,8 @@ async def run_simulation( llm = LLM( model=settings.model, provider=settings.provider, + base_url=settings.base_url, + api_key=settings.api_key, ) simulation_params = SimulationParams( @@ -532,6 +535,9 @@ async def run_simulation( ) await simulator.save() + # Surface cache-hit telemetry from the simulator LLM. Helps users + # verify the documented prompt-caching benefit without leaving arksim. + log_cache_stats(llm, phase="simulation") return simulation_output finally: if trace_receiver is not None: diff --git a/docs/docs.json b/docs/docs.json index 42969ec3..b2633bf0 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -109,6 +109,7 @@ "pages": [ "main/build-scenario", "main/simulate-conversation", + "main/user-simulator-on-open-responses", "main/evaluate-conversation", "main/tool-call-capture", "main/schema-reference" diff --git a/docs/main/simulate-conversation.mdx b/docs/main/simulate-conversation.mdx index 0f442b0b..9771ceb9 100644 --- a/docs/main/simulate-conversation.mdx +++ b/docs/main/simulate-conversation.mdx @@ -64,6 +64,10 @@ provider: openai # simulated_user_prompt_template: null ``` + +The user simulator can run against any [Open Responses](https://www.openresponses.org/)-conforming backend (Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter, and others) via `provider: responses`. See [User simulator on Open Responses](./user-simulator-on-open-responses) for the configuration and the prompt-caching benefit on OpenAI. + + The default `num_workers` is `50`. Set `num_workers: auto` to automatically parallelize across all conversations, or specify a fixed number to control load on your agent endpoint. diff --git a/docs/main/user-simulator-on-open-responses.mdx b/docs/main/user-simulator-on-open-responses.mdx new file mode 100644 index 00000000..7b7a6ea8 --- /dev/null +++ b/docs/main/user-simulator-on-open-responses.mdx @@ -0,0 +1,130 @@ +--- +title: "User simulator on Open Responses" +description: "Run the user-simulator LLM against any Open Responses-conforming backend." +--- + +## What is Open Responses + +The [Open Responses spec](https://www.openresponses.org/) is a multi-vendor open standard for the `/v1/responses` API shape. It is essentially OpenAI's Responses API formalized as a shared spec, backed by a multi-vendor consortium including NVIDIA, AWS, Databricks, Hugging Face, Vercel, OpenRouter, Ollama, vLLM, LM Studio, Llama Stack, Red Hat, and OpenAI. See [openresponses.org](https://www.openresponses.org/) for the current backer list. + +ArkSim's OpenAI provider has run on OpenAI's Responses API endpoint (`client.responses.parse`) since v0.3.0. The new `responses` provider alias lets you point the user-simulator LLM at any conforming backend by setting `base_url` and `api_key` in your config. Explicit `base_url` and `api_key` values in config take precedence over the `OPENAI_BASE_URL` and `OPENAI_API_KEY` environment variables; leave them unset (or empty) to fall back to env. + +These keys go at the top level of your `config.yaml`, alongside other simulation settings. + +## Why it matters + +ArkSim's user simulator sends a stable system prompt and scenario context to the LLM on every turn of every scenario. OpenAI's automatic prompt cache hits this stable prefix and discounts the cached portion by 50 to 75% depending on the model. Depending on prompt-cache hit rate and the output share of total cost, typical multi-turn runs see roughly 20 to 40% total cost reduction. The benefit was already active for OpenAI users; this page documents how to extend the same shape to self-hosted or alternate backends. + +## Splitting simulator and evaluator LLMs + +The evaluator passes `text_format` (OpenAI structured output) on every metric call. Self-hosted backends like Ollama do not currently accept `text_format`; vLLM accepts it only with `--guided-decoding-backend` enabled. + +To run the simulator on a self-hosted backend while keeping the evaluator on OpenAI, set the shared LLM keys for the simulator and override the evaluator-specific keys: + +```yaml +# Simulator runs on Ollama +provider: responses +model: llama3.1 +base_url: http://localhost:11434/v1 +api_key: ollama + +# Evaluator overrides to keep using OpenAI +evaluator_provider: openai +evaluator_model: gpt-4o-mini +``` + +Because the evaluator endpoint differs from the simulator endpoint (`evaluator_provider` is set), arksim does not forward the shared `api_key: ollama` to the evaluator. The evaluator's OpenAI SDK falls back to `OPENAI_API_KEY` in your environment. Set `OPENAI_API_KEY` before running, or specify `evaluator_api_key: ` explicitly. + +Make sure Ollama is running (`ollama serve`) and the model is pulled (`ollama pull llama3.1`). Requires Ollama v0.13.3 or newer for `/v1/responses` support; see the Ollama section below for details. + +For full explicit control without relying on env vars: + +```yaml +# Simulator on Ollama +provider: responses +model: llama3.1 +base_url: http://localhost:11434/v1 +api_key: ollama + +# Evaluator fully specified +evaluator_provider: openai +evaluator_model: gpt-4o-mini +evaluator_base_url: https://api.openai.com/v1 +evaluator_api_key: +``` + +If you omit the `evaluator_*` keys, the evaluator uses the same backend as the simulator. Available override fields: `evaluator_model`, `evaluator_provider`, `evaluator_base_url`, `evaluator_api_key`. + +## Configuration + +### OpenAI (default) + +```yaml +provider: openai +model: gpt-4o-mini +``` + +`api.openai.com` is the default. No `base_url` needed. + +Requires `OPENAI_API_KEY` in your environment. + +### Ollama (local) + +```yaml +provider: responses +model: llama3.1 +base_url: http://localhost:11434/v1 +api_key: ollama +``` + +Start Ollama (`ollama serve`), pull the model (`ollama pull llama3.1`), then run as usual. The `api_key` value is a placeholder; Ollama ignores it. + +Requires Ollama v0.13.3 or newer for `/v1/responses` support. The endpoint is currently experimental in Ollama and only the non-stateful flavor is implemented. + +### vLLM + +```yaml +provider: responses +model: meta-llama/Llama-3.1-8B-Instruct +base_url: http://my-vllm-host:8000/v1 +api_key: +``` + +arksim does not expand `${VAR}` substitutions in YAML. Paste the literal key, or omit the `api_key` field and set `OPENAI_API_KEY` in your shell environment. + +Start vLLM with `vllm serve --port 8000`, then point `base_url` at the running host. + +### NVIDIA NIM, Vercel AI Gateway, OpenRouter, LM Studio, Llama Stack + +Same shape, different `base_url`. Consult each provider's documentation for the exact endpoint path and authentication header. + +## When to use `openai` vs `responses` + +- Use `provider: openai` when you are hitting OpenAI itself with default endpoints. Most users land here. +- Use `provider: responses` when you set a custom `base_url`. The alias makes the multi-provider intent explicit in your config. + +Both names resolve to the same Python class under the hood. The distinction is documentation, not behavior. + +## Verifying caching is working + +For OpenAI-hosted models, the response payload exposes `usage.input_tokens_details.cached_tokens` on every call. ArkSim logs cache-hit statistics at the end of each simulation and evaluation run at INFO level. Look for lines like: + +``` +simulation LLM cache stats: 12 calls, 18420 input tokens, 9210 cached (50.0% hit rate), 1840 output tokens +``` + +For deeper detail, the [OpenAI usage dashboard](https://platform.openai.com/usage) exposes `cached_input_tokens` per request. + +Self-hosted backends do not have OpenAI-equivalent automatic prefix caching out of the box. Ollama maintains a KV cache within a single process but does not persist it across HTTP calls in a way that benefits cross-turn ArkSim runs. vLLM enables prefix caching by default on the V1 engine (the default since v0.8.1, April 2025). Older deployments still on V0 must opt in with `--enable-prefix-caching`. + +## Caveats + +- The OpenAI prompt cache requires prompts of at least 1024 tokens to activate. Toy scenarios with tiny user profiles and no knowledge base will not hit it. +- Cache TTL is roughly 5 to 10 minutes. ArkSim runs fast enough that this is rarely a constraint, but a paused or debugged run can lose its cached prefix. +- ArkSim does not stream Responses API output today; the user simulator uses non-streaming `responses.parse()`. +- ArkSim uses Responses in stateless mode: it replays the full conversation history each turn rather than threading via `previous_response_id`. Stateful threading is a planned follow-up that would further improve cache utilization. + +## Limitations + +- The `responses` provider currently forwards `model`, `temperature`, `base_url`, and `api_key`. Provider-specific kwargs (custom headers, organization IDs) need explicit support added. +- Anthropic and Google providers in ArkSim do not speak Open Responses; they use their native APIs. diff --git a/examples/customer-service/README.md b/examples/customer-service/README.md index a5f7733e..81477f28 100644 --- a/examples/customer-service/README.md +++ b/examples/customer-service/README.md @@ -127,3 +127,37 @@ The SQLite database (`store.db`) is created automatically on first run with samp - 4 orders (shipped, processing, delivered, cancelled) - 6 products (laptops, headphones, accessories) - 2 verification codes (one per customer) + +### Running the user simulator on a self-hosted backend + +The user-simulator LLM defaults to OpenAI. To run the simulator on any [Open Responses](https://www.openresponses.org/)-conforming backend (Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter) while keeping the evaluator on OpenAI, replace the LLM configuration in `config.yaml`: + +```yaml +# Simulator on Ollama +provider: responses +model: llama3.1 +base_url: http://localhost:11434/v1 +api_key: ollama + +# Evaluator stays on OpenAI for structured-output support +evaluator_provider: openai +evaluator_model: gpt-4o-mini +``` + +The evaluator endpoint differs from the simulator endpoint, so arksim does not forward `api_key: ollama` to OpenAI. The evaluator falls back to `OPENAI_API_KEY` in your environment. + +Before running, start the local server and set the evaluator key. For Ollama: + +```bash +ollama serve +ollama pull llama3.1 # requires Ollama v0.13.3 or newer +export OPENAI_API_KEY=sk-... +``` + +Then run: + +```bash +arksim simulate-evaluate config.yaml +``` + +See [User simulator on Open Responses](https://docs.arklex.ai/main/user-simulator-on-open-responses) for vLLM, NIM, and other backends. diff --git a/tests/integration/test_responses_provider.py b/tests/integration/test_responses_provider.py new file mode 100644 index 00000000..192ef767 --- /dev/null +++ b/tests/integration/test_responses_provider.py @@ -0,0 +1,309 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Opt-in integration smokes for the responses provider against api.openai.com. + +Skipped unless OPENAI_API_KEY is set in the environment. Two short calls +verify (1) alias wiring through the default constructor and (2) explicit +base_url + api_key end-to-end through the OpenAI SDK. + +Override the model via ARKSIM_SMOKE_MODEL when gpt-4.1-mini is deprecated. + +Cost: pennies per run on the happy path. OpenAILLM.call is wrapped in +`@retry(max_retries=5)`; auth or model-deprecation failures may make up +to 6 attempts before surfacing the error. +""" + +from __future__ import annotations + +import os +import time + +import pytest + +from arksim.llms.chat.llm import LLM + +_SMOKE_MODEL = os.environ.get("ARKSIM_SMOKE_MODEL", "gpt-4.1-mini") +_CACHE_TEST_MODEL = os.environ.get("ARKSIM_CACHE_TEST_MODEL", "gpt-5.1") + +# Per-model cache verification matrix. Order matters only for readability: +# the arksim DEFAULT_MODEL (`arksim/constants.py`) leads, family siblings +# follow, then known-working baseline, then known-broken regression sentinel. +# +# Names that no longer resolve (``gpt-5.1-mini``, ``gpt-5.1-nano``) skip +# at construction time; we keep them in the list so a future OpenAI release +# of those SKUs is automatically picked up without a test edit. +_CACHE_MATRIX_MODELS = ( + "gpt-5.1", # arksim DEFAULT_MODEL + "gpt-5.1-mini", + "gpt-5.1-nano", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-4o-mini", # historical baseline (cache routing variable) + "gpt-4.1-mini", # slow-warm regression sentinel +) + +# Models that empirically return ``cached_tokens=0`` across all four calls +# even with a stable >=1024-token prefix. The matrix test treats hits on +# these as a regression notice (remove them from the set) rather than a +# failure of arksim. Empty today, retained as the documented escape hatch. +_CACHE_MATRIX_KNOWN_BROKEN: frozenset[str] = frozenset() + + +@pytest.mark.integration +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), + reason="OPENAI_API_KEY not set; opt-in smoke test skipped", +) +def test_responses_provider_smoke_against_openai() -> None: + llm = LLM(model=_SMOKE_MODEL, provider="responses") + out = llm.call("Say only the word 'hello' and nothing else.") + assert isinstance(out, str) + assert "hello" in out.lower() + + +@pytest.mark.integration +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), + reason="OPENAI_API_KEY not set; opt-in smoke test skipped", +) +def test_responses_provider_smoke_with_explicit_base_url() -> None: + """Exercise the new base_url + api_key path against the real endpoint. + + Without an explicit base_url the provider falls back to env vars and + we would not be testing the new code path. This call forces both + the base_url and api_key plumbing to be exercised end-to-end. + """ + llm = LLM( + model=_SMOKE_MODEL, + provider="responses", + base_url="https://api.openai.com/v1", + api_key=os.environ["OPENAI_API_KEY"], + ) + out = llm.call("Say only the word 'hello' and nothing else.") + assert isinstance(out, str) + assert "hello" in out.lower() + + +@pytest.mark.integration +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), + reason="OPENAI_API_KEY not set; opt-in cache test skipped", +) +def test_responses_provider_cache_hit_on_repeat_prefix() -> None: + """Empirically verify OpenAI's prompt cache fires for repeat prefixes. + + Per OpenAI's prompt-caching docs, prompts >= 1024 tokens with stable + prefixes are cached automatically. This test sends several calls with + the same large prefix and asserts at least one observes cached_tokens + > 0. Multiple calls are needed because OpenAI load-balances across + servers and a fresh prefix may not hit a warm cache on the first + retry. The cumulative ``cached_input_tokens`` counter on the LLM + captures any hit across the batch. + + Why this exists: PR #170 documents a 20-40% cost reduction from + automatic prompt caching. Without this test, the claim is unverified. + + Model pin: this test defaults to arksim's DEFAULT_MODEL (gpt-5.1) + regardless of ``ARKSIM_SMOKE_MODEL`` so the verified benefit + tracks the model arksim users actually run. Override via + ``ARKSIM_CACHE_TEST_MODEL`` if your account routes a different + model for caching tests. + """ + llm = LLM(model=_CACHE_TEST_MODEL, provider="responses") + + # Build a stable prefix comfortably above the 1024-token cache floor. + # Empirically the paragraph below tokenizes to ~35 tokens, so 60 + # repetitions yields ~2100 tokens. + paragraph = ( + "You are a friendly customer service simulator. The user is " + "trying to resolve a billing issue with their subscription. " + "Be patient, ask clarifying questions, and stay in character. " + ) + stable_prefix = paragraph * 60 # ~2100 tokens, comfortably above 1024 + + # Send several calls with a small inter-call delay. The first primes + # the cache; subsequent calls should hit it. OpenAI load-balances + # across cache shards, so we look for at least one hit across the + # batch rather than asserting per-call. + num_calls = 4 + for i in range(num_calls): + llm.call(stable_prefix + f"Reply with only the number {i}.") + if i < num_calls - 1: + time.sleep(2) + + stats = llm.cache_stats() + assert stats["cached_input_tokens"] > 0, ( + f"Expected at least one cache hit across {num_calls} calls with " + f"a ~2100-token stable prefix on model {_CACHE_TEST_MODEL!r}, " + f"got cached_input_tokens=0. Full stats: {stats}. " + f"Possible causes: model does not support prompt caching, " + f"OpenAI cache shard miss across all retries, " + f"or the cached_tokens field is not populated by this backend." + ) + + +@pytest.mark.integration +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), + reason="OPENAI_API_KEY not set; opt-in cache matrix skipped", +) +@pytest.mark.parametrize("model", _CACHE_MATRIX_MODELS) +def test_responses_provider_cache_matrix(model: str) -> None: + """Per-model verification that OpenAI's prompt cache fires. + + Status per model is dynamic (depends on OpenAI's server-side cache + routing for each model and shard warmth). The matrix exists to + (1) prove arksim's DEFAULT_MODEL actually benefits from caching, + (2) flag regressions when OpenAI changes model behavior, and + (3) give PR reviewers an at-a-glance answer for which models + deliver the documented benefit. + + Shape mirrors ``test_responses_provider_cache_hit_on_repeat_prefix``: + four calls with a ~2100-token stable prefix, asserting cumulative + ``cached_input_tokens > 0``. A two-call shape is unreliable because + OpenAI load-balances across cache shards and a fresh prefix often + misses the first repeat. + + Cost: roughly $0.01-0.05 per model per run. + """ + try: + llm = LLM(model=model, provider="responses") + except Exception as exc: + pytest.skip(f"Could not construct LLM for {model}: {exc}") + + paragraph = ( + "You are a friendly customer service simulator. The user is " + "trying to resolve a billing issue with their subscription. " + "Be patient, ask clarifying questions, and stay in character. " + ) + stable_prefix = paragraph * 60 # ~2100 tokens, comfortably above 1024 + + num_calls = 4 + for i in range(num_calls): + try: + llm.call(stable_prefix + f"Reply with only the number {i}.") + except Exception as exc: + pytest.skip(f"Model {model} unavailable or call failed: {exc}") + if i < num_calls - 1: + time.sleep(2) + + stats = llm.cache_stats() + cached = stats["cached_input_tokens"] + + if model in _CACHE_MATRIX_KNOWN_BROKEN: + if cached > 0: + pytest.fail( + f"Model {model} was known-broken (cached_tokens always 0 " + f"across {num_calls} calls) but now shows {cached} cached " + f"tokens. Remove {model!r} from _CACHE_MATRIX_KNOWN_BROKEN." + ) + pytest.skip( + f"Model {model} known to return cached_tokens=0 (server-side " + f"cache-routing anomaly). Observed cached={cached}." + ) + + assert cached > 0, ( + f"Expected at least one cache hit across {num_calls} calls to " + f"{model} with a ~2100-token stable prefix. Got cached={cached}. " + f"Full stats: {stats}. If this is a known model-side regression, " + f"add {model!r} to _CACHE_MATRIX_KNOWN_BROKEN." + ) + + +@pytest.mark.integration +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), + reason="OPENAI_API_KEY not set; opt-in growth test skipped", +) +def test_cache_hits_grow_across_simulator_turns() -> None: + """Cache fires across multi-turn simulator calls (arksim's real shape). + + arksim's user simulator sends [system + scenario + growing history + + trigger] on every turn. The stable prefix is reused across turns, so + OpenAI's prompt cache fires from turn 2 onward and the cumulative hit + rate climbs sharply across a multi-turn scenario. + + Observed cold-cache shape on gpt-5.1 (turn-1 starts cold, turns 2-5 + hit 85-95%): per-turn rates [0%, 86%, 92%, 89%, 95%], cumulative 74%. + + Distinct from the matrix test, which sends the SAME prefix twice + (idealized retry shape). This test exercises the real arksim call + pattern where each turn's prompt differs but shares a stable prefix. + """ + llm = LLM(model=_CACHE_TEST_MODEL, provider="responses") + + # OpenAI's prompt cache only fires for prefixes >= 1024 tokens. Size the + # stable portion (system + scenario) to ~1250 tokens so the cache is + # eligible from turn 1. Real arksim user-simulator prompts in + # production tend to land in the 1500-3000 token range once the + # scenario goal and persona are populated. + system_prompt = ( + "You are a simulated user testing a customer-service AI agent. " + "Stay in character, respond naturally, and explore the agent's " + "capabilities. Do not break character or reveal that you are a " + "simulator. Always end your response with a follow-up question " + "or a clarification request when appropriate. " + ) * 10 # ~530 tokens + + scenario_context = ( + "Scenario: You are a long-time customer of XYZ Bank. Your goal " + "is to dispute a charge on your credit card statement from last " + "month. You believe the charge of $89.99 from 'Acme Subscriptions' " + "is unauthorized; you never signed up for this service. You are " + "polite but firm. You have your account number ready: 4532-XXXX. " + ) * 10 # ~720 tokens + + history: list[str] = [] + hit_rates: list[float] = [] + cached_per_turn: list[int] = [] + + for turn in range(5): + previous_cached = llm.cache_stats()["cached_input_tokens"] + previous_input = llm.cache_stats()["input_tokens"] + + message = ( + system_prompt + + "\n\n" + + scenario_context + + "\n\nConversation so far:\n" + + "\n".join(history) + + f"\n\nIt is turn {turn + 1}. Respond as the user " + "with one short sentence." + ) + response = llm.call(message) + history.append(f"user (turn {turn + 1}): {response}") + history.append( + f"agent (turn {turn + 1}): I understand. Let me look into that for you." + ) + + new_cached = llm.cache_stats()["cached_input_tokens"] - previous_cached + new_input = llm.cache_stats()["input_tokens"] - previous_input + rate = new_cached / new_input if new_input > 0 else 0.0 + hit_rates.append(rate) + cached_per_turn.append(new_cached) + + final_stats = llm.cache_stats() + + # Headline assertion: cumulative cache hit rate must reflect the + # growing prefix benefit. After 5 turns of a scenario, at least 30% + # of input tokens should be cached. This is a conservative floor. + overall_rate = final_stats["cache_hit_rate"] + assert overall_rate >= 0.30, ( + f"Expected >= 30% cumulative cache hit rate after 5 turns, " + f"got {overall_rate:.1%}. Per-turn: {hit_rates}. " + f"Cached tokens per turn: {cached_per_turn}. " + f"This is the realistic shape arksim users hit; if this drops " + f"below 30% the documented cost-reduction claim is wrong." + ) + + # Cache must keep firing as the conversation grows. OpenAI's prompt + # cache rounds to 128-token blocks and load-balances across shards, + # so we cannot assert strict monotonic growth (cold shards may flake + # a single turn back to 0). We assert that the cache fires on the + # majority of turns and that the cached-token count never collapses. + turns_with_hits = sum(1 for c in cached_per_turn if c > 0) + assert turns_with_hits >= 3, ( + f"Cache must fire on at least 3 of 5 turns for the growth claim " + f"to hold. Got {turns_with_hits} turns with hits. " + f"Per-turn cached tokens: {cached_per_turn}." + ) diff --git a/tests/unit/test_evaluation_input.py b/tests/unit/test_evaluation_input.py index 60a16ccf..69d10809 100644 --- a/tests/unit/test_evaluation_input.py +++ b/tests/unit/test_evaluation_input.py @@ -1,12 +1,166 @@ # SPDX-License-Identifier: Apache-2.0 """Tests for arksim.evaluator.entities.EvaluationInput validator.""" +from __future__ import annotations + +import logging from pathlib import Path from typing import Any +import pytest + from arksim.evaluator.entities import EvaluationInput +class TestEvaluationInputFields: + """Tests for newly added optional LLM transport fields.""" + + def test_evaluation_input_accepts_base_url_and_api_key(self) -> None: + settings = EvaluationInput( + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + ) + assert settings.base_url == "http://localhost:11434/v1" + assert settings.api_key == "ollama" + + def test_evaluation_input_defaults_base_url_and_api_key_to_none(self) -> None: + settings = EvaluationInput() + assert settings.base_url is None + assert settings.api_key is None + + +class TestEvaluationInputEvaluatorOverrides: + """Tests for the evaluator_* LLM override fields.""" + + def test_evaluator_overrides_default_to_none(self) -> None: + settings = EvaluationInput() + assert settings.evaluator_model is None + assert settings.evaluator_provider is None + assert settings.evaluator_base_url is None + assert settings.evaluator_api_key is None + + def test_evaluator_overrides_accept_values(self) -> None: + settings = EvaluationInput( + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + evaluator_model="gpt-4o-mini", + evaluator_provider="openai", + evaluator_base_url="https://api.openai.com/v1", + evaluator_api_key="sk-test", + ) + assert settings.evaluator_model == "gpt-4o-mini" + assert settings.evaluator_provider == "openai" + assert settings.evaluator_base_url == "https://api.openai.com/v1" + assert settings.evaluator_api_key == "sk-test" + + def test_evaluator_overrides_can_be_partial(self) -> None: + """A user can override just provider/base_url and keep shared + model/api_key, or any other partial pattern. + """ + settings = EvaluationInput( + provider="responses", + base_url="http://localhost:11434/v1", + evaluator_provider="openai", + ) + assert settings.provider == "responses" + assert settings.evaluator_provider == "openai" + assert settings.evaluator_base_url is None + assert settings.evaluator_api_key is None + + def test_evaluator_kwargs_treats_whitespace_evaluator_provider_as_unset( + self, + ) -> None: + """Whitespace-only evaluator_provider should not trigger endpoint- + split semantics. The helper strips whitespace before deciding + whether the endpoint differs; this matches the existing whitespace- + stripping behavior in the OpenAI provider. The shared api_key + therefore flows through to the evaluator. + """ + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + api_key="sk-shared", + evaluator_provider=" ", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["api_key"] == "sk-shared" + assert kwargs["provider"] == "openai" + + def test_evaluator_kwargs_treats_whitespace_evaluator_base_url_as_unset( + self, + ) -> None: + """Whitespace-only evaluator_base_url should not trigger endpoint- + split semantics; the shared api_key still flows through. + """ + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + api_key="sk-shared", + evaluator_base_url=" ", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["api_key"] == "sk-shared" + + def test_endpoint_does_not_differ_when_evaluator_provider_matches_shared( + self, + ) -> None: + """Setting evaluator_provider to the same value as provider must not + trigger endpoint-split (the shared api_key must NOT be dropped). + A redundant override is not a real endpoint change. + """ + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + api_key="sk-shared", + evaluator_provider="openai", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["api_key"] == "sk-shared", ( + "Redundant evaluator_provider must not strip shared credentials" + ) + assert kwargs["provider"] == "openai" + + def test_endpoint_does_not_differ_when_evaluator_base_url_matches_shared( + self, + ) -> None: + """Same as above for base_url: a redundant override that matches the + shared value must not trigger endpoint-split semantics. + """ + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + base_url="https://api.openai.com/v1", + api_key="sk-shared", + evaluator_base_url="https://api.openai.com/v1", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["api_key"] == "sk-shared" + assert kwargs["base_url"] == "https://api.openai.com/v1" + + def test_warns_when_evaluator_api_key_set_alone( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Setting evaluator_api_key alone (no endpoint split) silently + discards the override. Warn the user so they do not assume the + override took effect. + """ + with caplog.at_level(logging.WARNING, logger="arksim.evaluator.entities"): + EvaluationInput( + model="gpt-4o-mini", + provider="openai", + api_key="sk-shared", + evaluator_api_key="sk-eval-IGNORED", + ) + assert any( + "evaluator_api_key is set" in rec.message and "ignored" in rec.message + for rec in caplog.records + ) + + class TestEvaluationInputPathResolution: """Tests for config-relative path resolution in EvaluationInput.""" diff --git a/tests/unit/test_llm_factory.py b/tests/unit/test_llm_factory.py index 0176d1c8..34287000 100644 --- a/tests/unit/test_llm_factory.py +++ b/tests/unit/test_llm_factory.py @@ -48,3 +48,7 @@ def test_google_provider(self) -> None: def test_unknown_provider_raises(self) -> None: with pytest.raises(ValueError, match="not supported"): LLM._get_provider("unknown") + + def test_responses_provider_resolves_to_openai_llm(self) -> None: + cls = LLM._get_provider("responses") + assert cls.__name__ == "OpenAILLM" diff --git a/tests/unit/test_openai_llm_provider.py b/tests/unit/test_openai_llm_provider.py new file mode 100644 index 00000000..e1f9c6d9 --- /dev/null +++ b/tests/unit/test_openai_llm_provider.py @@ -0,0 +1,217 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tests for arksim.llms.chat.providers.openai (OpenAILLM constructor).""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest +from openai import OpenAIError + +from arksim.llms.chat.providers.openai import OpenAILLM + + +class TestOpenAILLMConstructor: + def test_defaults_no_base_url_or_api_key(self) -> None: + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM(model="gpt-4.1-mini") + mock_sync.assert_called_once_with() + mock_async.assert_called_once_with() + + def test_base_url_passthrough(self) -> None: + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM(model="llama3.1", base_url="http://localhost:11434/v1") + mock_sync.assert_called_once_with(base_url="http://localhost:11434/v1") + mock_async.assert_called_once_with(base_url="http://localhost:11434/v1") + + def test_api_key_passthrough(self) -> None: + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM(model="gpt-4.1-mini", api_key="sk-test-123") + mock_sync.assert_called_once_with(api_key="sk-test-123") + mock_async.assert_called_once_with(api_key="sk-test-123") + + def test_base_url_and_api_key_both_passthrough(self) -> None: + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM( + model="meta-llama/Llama-3.1-8B-Instruct", + base_url="http://my-vllm:8000/v1", + api_key="vllm-token", + ) + mock_sync.assert_called_once_with( + base_url="http://my-vllm:8000/v1", api_key="vllm-token" + ) + mock_async.assert_called_once_with( + base_url="http://my-vllm:8000/v1", api_key="vllm-token" + ) + + def test_explicit_none_treated_as_omitted(self) -> None: + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM(model="gpt-4.1-mini", base_url=None, api_key=None) + mock_sync.assert_called_once_with() + mock_async.assert_called_once_with() + + def test_empty_string_treated_as_omitted(self) -> None: + """YAML configs can produce empty strings; treat them like None. + + Avoids confusing httpx/auth errors when a user leaves + `base_url:` or `api_key:` blank in their config. + """ + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM(model="gpt-4.1-mini", base_url="", api_key="") + mock_sync.assert_called_once_with() + mock_async.assert_called_once_with() + + def test_temperature_flows_through_with_base_url(self) -> None: + """Adding base_url/api_key must not swallow temperature via kwargs.""" + with ( + patch("arksim.llms.chat.providers.openai.OpenAI"), + patch("arksim.llms.chat.providers.openai.AsyncOpenAI"), + ): + llm = OpenAILLM( + model="gpt-4.1-mini", + temperature=0.7, + base_url="http://localhost:11434/v1", + ) + assert llm.temperature == 0.7 + assert llm.model == "gpt-4.1-mini" + + def test_whitespace_base_url_and_api_key_treated_as_omitted(self) -> None: + """Whitespace-only values must not reach the SDK; they would be + URL-encoded into `%20...` and cause slow connection failures. + """ + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM(model="gpt-4o-mini", base_url=" ", api_key="\t\n") + mock_sync.assert_called_once_with() + mock_async.assert_called_once_with() + + def test_base_url_and_api_key_are_stripped(self) -> None: + """Leading and trailing whitespace is stripped before passing to SDK.""" + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM( + model="llama3.1", + base_url=" http://localhost:11434/v1 ", + api_key=" ollama ", + ) + mock_sync.assert_called_once_with( + base_url="http://localhost:11434/v1", api_key="ollama" + ) + mock_async.assert_called_once_with( + base_url="http://localhost:11434/v1", api_key="ollama" + ) + + def test_missing_credentials_raises_actionable_error(self) -> None: + """When the OpenAI SDK rejects no-credentials construction, wrap the + error with an arksim-specific message pointing at the YAML field + and docs URL. The user should not see a bare OpenAIError stack trace. + """ + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI"), + ): + mock_sync.side_effect = OpenAIError( + "The api_key client option must be set..." + ) + with pytest.raises(ValueError, match="OpenAI SDK requires an api_key"): + OpenAILLM(model="gpt-4o-mini") + + +class TestOpenAILLMCacheStats: + def test_cache_stats_starts_at_zero(self) -> None: + with ( + patch("arksim.llms.chat.providers.openai.OpenAI"), + patch("arksim.llms.chat.providers.openai.AsyncOpenAI"), + ): + llm = OpenAILLM(model="gpt-4o-mini") + stats = llm.cache_stats() + assert stats == { + "call_count": 0, + "input_tokens": 0, + "cached_input_tokens": 0, + "output_tokens": 0, + "cache_hit_rate": 0.0, + } + + def test_record_usage_accumulates(self) -> None: + from types import SimpleNamespace + + with ( + patch("arksim.llms.chat.providers.openai.OpenAI"), + patch("arksim.llms.chat.providers.openai.AsyncOpenAI"), + ): + llm = OpenAILLM(model="gpt-4o-mini") + + # First call: no cache hits + response1 = SimpleNamespace( + usage=SimpleNamespace( + input_tokens=1024, + output_tokens=50, + input_tokens_details=SimpleNamespace(cached_tokens=0), + ) + ) + llm._record_usage(response1) + + # Second call: prefix hits cache + response2 = SimpleNamespace( + usage=SimpleNamespace( + input_tokens=1100, + output_tokens=60, + input_tokens_details=SimpleNamespace(cached_tokens=900), + ) + ) + llm._record_usage(response2) + + stats = llm.cache_stats() + assert stats["call_count"] == 2 + assert stats["input_tokens"] == 2124 + assert stats["cached_input_tokens"] == 900 + assert stats["output_tokens"] == 110 + assert stats["cache_hit_rate"] == 900 / 2124 + + def test_record_usage_tolerates_missing_usage_field(self) -> None: + """Self-hosted backends (Ollama, vLLM) may omit usage entirely. + Telemetry must never raise. + """ + from types import SimpleNamespace + + with ( + patch("arksim.llms.chat.providers.openai.OpenAI"), + patch("arksim.llms.chat.providers.openai.AsyncOpenAI"), + ): + llm = OpenAILLM(model="llama3.1") + + # Response without usage attribute at all + response = SimpleNamespace() + llm._record_usage(response) + assert llm.cache_stats()["call_count"] == 0 + + # Response with usage but no input_tokens_details (e.g. older SDK) + response = SimpleNamespace( + usage=SimpleNamespace(input_tokens=500, output_tokens=20) + ) + llm._record_usage(response) + assert llm.cache_stats()["call_count"] == 1 + assert llm.cache_stats()["cached_input_tokens"] == 0 diff --git a/tests/unit/test_responses_yaml_plumbing.py b/tests/unit/test_responses_yaml_plumbing.py new file mode 100644 index 00000000..edbb644f --- /dev/null +++ b/tests/unit/test_responses_yaml_plumbing.py @@ -0,0 +1,215 @@ +# SPDX-License-Identifier: Apache-2.0 +"""End-to-end plumbing test: YAML Pydantic -> LLM(...) -> OpenAI SDK kwargs. + +Verifies that base_url and api_key set on SimulationInput / EvaluationInput +travel through model_dump() and LLM(...) construction down to the underlying +OpenAI / AsyncOpenAI client constructors. This guards against the regression +where the fields were silently dropped by Pydantic's default extra='ignore'. +""" + +from __future__ import annotations + +from unittest.mock import patch + +from arksim.evaluator.entities import EvaluationInput +from arksim.llms.chat.llm import LLM +from arksim.simulation_engine.entities import SimulationInput + + +class TestResponsesYamlPlumbing: + """SimulationInput / EvaluationInput -> LLM -> OpenAI SDK forwarding.""" + + def test_simulation_input_forwards_base_url_and_api_key_to_openai(self) -> None: + settings = SimulationInput( + agent_config_file_path="agent.json", + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + ) + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + LLM( + model=settings.model, + provider=settings.provider, + base_url=settings.base_url, + api_key=settings.api_key, + ) + mock_sync.assert_called_once_with( + base_url="http://localhost:11434/v1", api_key="ollama" + ) + mock_async.assert_called_once_with( + base_url="http://localhost:11434/v1", api_key="ollama" + ) + + def test_evaluation_input_forwards_base_url_and_api_key_to_openai(self) -> None: + settings = EvaluationInput( + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + ) + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + LLM( + model=settings.model, + provider=settings.provider, + base_url=settings.base_url, + api_key=settings.api_key, + ) + mock_sync.assert_called_once_with( + base_url="http://localhost:11434/v1", api_key="ollama" + ) + mock_async.assert_called_once_with( + base_url="http://localhost:11434/v1", api_key="ollama" + ) + + def test_simulation_input_defaults_omit_base_url_and_api_key(self) -> None: + settings = SimulationInput( + agent_config_file_path="agent.json", provider="responses" + ) + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + LLM( + model=settings.model, + provider=settings.provider, + base_url=settings.base_url, + api_key=settings.api_key, + ) + # base_url=None and api_key=None must not be forwarded to the SDK; + # the SDK falls back to OPENAI_BASE_URL / OPENAI_API_KEY env vars. + mock_sync.assert_called_once_with() + mock_async.assert_called_once_with() + + def test_evaluator_kwargs_inherit_when_no_overrides(self) -> None: + """When no evaluator_* fields are set, evaluator inherits all + simulator-side LLM keys. + """ + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + base_url=None, + api_key="sk-shared", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs == { + "model": "gpt-4o-mini", + "provider": "openai", + "base_url": None, + "api_key": "sk-shared", + } + + def test_evaluator_kwargs_model_only_override_keeps_shared_endpoint( + self, + ) -> None: + """evaluator_model alone overrides only the model; other shared + keys still apply because the endpoint did not change. + """ + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + api_key="sk-shared", + evaluator_model="gpt-4o", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["model"] == "gpt-4o" + assert kwargs["provider"] == "openai" + assert kwargs["api_key"] == "sk-shared" + + def test_evaluator_kwargs_provider_split_does_not_inherit_credentials( + self, + ) -> None: + """When evaluator_provider differs from provider, the shared + api_key and base_url MUST NOT be forwarded. This is the bug fix + for the cross-endpoint credential leak. + """ + settings = EvaluationInput( + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + evaluator_provider="openai", + evaluator_model="gpt-4o-mini", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["model"] == "gpt-4o-mini" + assert kwargs["provider"] == "openai" + assert kwargs["base_url"] is None, ( + "Shared base_url must not leak into evaluator when provider differs" + ) + assert kwargs["api_key"] is None, ( + "Shared api_key must not leak into evaluator when provider differs" + ) + + def test_evaluator_kwargs_base_url_split_does_not_inherit_credentials( + self, + ) -> None: + """When evaluator_base_url differs from base_url (endpoint split), + api_key must NOT inherit from shared. Catches the credential-leak + scenario where a live OpenAI key would otherwise be sent to a + self-hosted endpoint. + """ + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + api_key="sk-LIVE-OPENAI-KEY", + evaluator_base_url="http://localhost:11434/v1", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["api_key"] is None, ( + "Live api_key must not be forwarded to a different base_url" + ) + assert kwargs["base_url"] == "http://localhost:11434/v1" + + def test_evaluator_kwargs_explicit_overrides_used_verbatim(self) -> None: + """When evaluator_* fields are fully specified, they pass through + verbatim and shared keys are ignored. + """ + settings = EvaluationInput( + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + evaluator_model="gpt-4o-mini", + evaluator_provider="openai", + evaluator_base_url="https://api.openai.com/v1", + evaluator_api_key="sk-eval-explicit", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs == { + "model": "gpt-4o-mini", + "provider": "openai", + "base_url": "https://api.openai.com/v1", + "api_key": "sk-eval-explicit", + } + + def test_evaluator_split_reaches_openai_sdk_without_shared_credentials( + self, + ) -> None: + """End-to-end: simulator on Ollama, evaluator on OpenAI. The OpenAI + SDK constructor must not see Ollama's base_url or api_key. + """ + settings = EvaluationInput( + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + evaluator_provider="openai", + evaluator_model="gpt-4o-mini", + ) + + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + LLM(**settings.evaluator_llm_kwargs()) + # Critical: NO base_url, NO api_key kwargs. SDK must fall back + # to OPENAI_BASE_URL / OPENAI_API_KEY env vars. + mock_sync.assert_called_once_with() + mock_async.assert_called_once_with() diff --git a/tests/unit/test_simulation_input.py b/tests/unit/test_simulation_input.py index 818fc35b..ec61ac56 100644 --- a/tests/unit/test_simulation_input.py +++ b/tests/unit/test_simulation_input.py @@ -29,6 +29,22 @@ def test_invalid_num_workers_string(self) -> None: with pytest.raises(ValidationError, match="num_workers"): SimulationInput(agent_config_file_path="a.json", num_workers="fast") + def test_simulation_input_accepts_base_url_and_api_key(self) -> None: + settings = SimulationInput( + agent_config_file_path="agent.json", + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + ) + assert settings.base_url == "http://localhost:11434/v1" + assert settings.api_key == "ollama" + + def test_simulation_input_defaults_base_url_and_api_key_to_none(self) -> None: + settings = SimulationInput(agent_config_file_path="agent.json") + assert settings.base_url is None + assert settings.api_key is None + class TestSimulationInputPathResolution: """Tests for config-relative path resolution in SimulationInput."""