From 46bdfc65945524b22c10cdfae02b3ca2794305e4 Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Fri, 15 May 2026 10:58:17 -0700 Subject: [PATCH 01/23] feat(llm): allow base_url and api_key on OpenAILLM constructor --- arksim/llms/chat/providers/openai.py | 15 ++++- tests/unit/test_openai_llm_provider.py | 91 ++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_openai_llm_provider.py diff --git a/arksim/llms/chat/providers/openai.py b/arksim/llms/chat/providers/openai.py index 6ab74688..5dad62e8 100644 --- a/arksim/llms/chat/providers/openai.py +++ b/arksim/llms/chat/providers/openai.py @@ -19,11 +19,22 @@ def __init__( model: str, provider: str | None = None, temperature: float | None = None, + base_url: str | None = None, + api_key: str | None = None, **kwargs: object, ) -> None: super().__init__(model, provider, temperature, **kwargs) - self.client = OpenAI() - self.async_client = AsyncOpenAI() + # Coerce empty strings to omitted: YAML configs commonly produce "" + # for unset fields, and an empty base_url or api_key would otherwise + # confuse the SDK rather than falling back to OPENAI_BASE_URL / + # OPENAI_API_KEY env vars. + client_kwargs: dict[str, str] = {} + if base_url: + client_kwargs["base_url"] = base_url + if api_key: + client_kwargs["api_key"] = api_key + self.client = OpenAI(**client_kwargs) + self.async_client = AsyncOpenAI(**client_kwargs) def _prepare_params( self, diff --git a/tests/unit/test_openai_llm_provider.py b/tests/unit/test_openai_llm_provider.py new file mode 100644 index 00000000..a48623af --- /dev/null +++ b/tests/unit/test_openai_llm_provider.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Tests for arksim.llms.chat.providers.openai (OpenAILLM constructor).""" + +from __future__ import annotations + +from unittest.mock import patch + +from arksim.llms.chat.providers.openai import OpenAILLM + + +class TestOpenAILLMConstructor: + def test_defaults_no_base_url_or_api_key(self) -> None: + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM(model="gpt-4.1-mini") + mock_sync.assert_called_once_with() + mock_async.assert_called_once_with() + + def test_base_url_passthrough(self) -> None: + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM(model="llama3.1", base_url="http://localhost:11434/v1") + mock_sync.assert_called_once_with(base_url="http://localhost:11434/v1") + mock_async.assert_called_once_with(base_url="http://localhost:11434/v1") + + def test_api_key_passthrough(self) -> None: + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM(model="gpt-4.1-mini", api_key="sk-test-123") + mock_sync.assert_called_once_with(api_key="sk-test-123") + mock_async.assert_called_once_with(api_key="sk-test-123") + + def test_base_url_and_api_key_both_passthrough(self) -> None: + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM( + model="meta-llama/Llama-3.1-8B-Instruct", + base_url="http://my-vllm:8000/v1", + api_key="vllm-token", + ) + mock_sync.assert_called_once_with( + base_url="http://my-vllm:8000/v1", api_key="vllm-token" + ) + mock_async.assert_called_once_with( + base_url="http://my-vllm:8000/v1", api_key="vllm-token" + ) + + def test_explicit_none_treated_as_omitted(self) -> None: + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM(model="gpt-4.1-mini", base_url=None, api_key=None) + mock_sync.assert_called_once_with() + mock_async.assert_called_once_with() + + def test_empty_string_treated_as_omitted(self) -> None: + """YAML configs can produce empty strings; treat them like None. + + Avoids confusing httpx/auth errors when a user leaves + `base_url:` or `api_key:` blank in their config. + """ + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM(model="gpt-4.1-mini", base_url="", api_key="") + mock_sync.assert_called_once_with() + mock_async.assert_called_once_with() + + def test_temperature_flows_through_with_base_url(self) -> None: + """Adding base_url/api_key must not swallow temperature via kwargs.""" + with ( + patch("arksim.llms.chat.providers.openai.OpenAI"), + patch("arksim.llms.chat.providers.openai.AsyncOpenAI"), + ): + llm = OpenAILLM( + model="gpt-4.1-mini", + temperature=0.7, + base_url="http://localhost:11434/v1", + ) + assert llm.temperature == 0.7 + assert llm.model == "gpt-4.1-mini" From 68874cb9161763e88171fb17e23b0a622043dd5c Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Fri, 15 May 2026 11:04:55 -0700 Subject: [PATCH 02/23] feat(llm): register responses/open_responses provider aliases --- arksim/llms/chat/llm.py | 7 ++++++- tests/unit/test_llm_factory.py | 26 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/arksim/llms/chat/llm.py b/arksim/llms/chat/llm.py index edcf51d1..a18ef1b2 100644 --- a/arksim/llms/chat/llm.py +++ b/arksim/llms/chat/llm.py @@ -23,7 +23,12 @@ def __new__(cls, model: str, **kwargs: object) -> BaseLLM: @classmethod def _get_provider(cls, provider: str) -> type: - if provider == "openai": + if provider in ("openai", "responses", "open_responses"): + if provider == "open_responses": + logger.info( + "'open_responses' provider alias resolves to the same " + "Responses-API client as 'responses'." + ) from arksim.llms.chat.providers.openai import OpenAILLM return OpenAILLM diff --git a/tests/unit/test_llm_factory.py b/tests/unit/test_llm_factory.py index 0176d1c8..d9e2de20 100644 --- a/tests/unit/test_llm_factory.py +++ b/tests/unit/test_llm_factory.py @@ -48,3 +48,29 @@ def test_google_provider(self) -> None: def test_unknown_provider_raises(self) -> None: with pytest.raises(ValueError, match="not supported"): LLM._get_provider("unknown") + + def test_responses_provider_resolves_to_openai_llm(self) -> None: + cls = LLM._get_provider("responses") + assert cls.__name__ == "OpenAILLM" + + def test_open_responses_provider_resolves_to_openai_llm(self) -> None: + cls = LLM._get_provider("open_responses") + assert cls.__name__ == "OpenAILLM" + + def test_open_responses_provider_emits_info_log( + self, caplog: pytest.LogCaptureFixture + ) -> None: + import logging + + with caplog.at_level(logging.INFO, logger="arksim.llms.chat.llm"): + LLM._get_provider("open_responses") + assert any("open_responses" in rec.message for rec in caplog.records) + + def test_openai_provider_does_not_emit_info_log( + self, caplog: pytest.LogCaptureFixture + ) -> None: + import logging + + with caplog.at_level(logging.INFO, logger="arksim.llms.chat.llm"): + LLM._get_provider("openai") + assert not any("open_responses" in rec.message for rec in caplog.records) From 200f372e771156e7c0f0eeca5e20a118bd72b23a Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Fri, 15 May 2026 11:11:50 -0700 Subject: [PATCH 03/23] refactor(llm): drop open_responses alias, keep responses --- arksim/llms/chat/llm.py | 7 +------ tests/unit/test_llm_factory.py | 22 ---------------------- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/arksim/llms/chat/llm.py b/arksim/llms/chat/llm.py index a18ef1b2..3922245b 100644 --- a/arksim/llms/chat/llm.py +++ b/arksim/llms/chat/llm.py @@ -23,12 +23,7 @@ def __new__(cls, model: str, **kwargs: object) -> BaseLLM: @classmethod def _get_provider(cls, provider: str) -> type: - if provider in ("openai", "responses", "open_responses"): - if provider == "open_responses": - logger.info( - "'open_responses' provider alias resolves to the same " - "Responses-API client as 'responses'." - ) + if provider in ("openai", "responses"): from arksim.llms.chat.providers.openai import OpenAILLM return OpenAILLM diff --git a/tests/unit/test_llm_factory.py b/tests/unit/test_llm_factory.py index d9e2de20..34287000 100644 --- a/tests/unit/test_llm_factory.py +++ b/tests/unit/test_llm_factory.py @@ -52,25 +52,3 @@ def test_unknown_provider_raises(self) -> None: def test_responses_provider_resolves_to_openai_llm(self) -> None: cls = LLM._get_provider("responses") assert cls.__name__ == "OpenAILLM" - - def test_open_responses_provider_resolves_to_openai_llm(self) -> None: - cls = LLM._get_provider("open_responses") - assert cls.__name__ == "OpenAILLM" - - def test_open_responses_provider_emits_info_log( - self, caplog: pytest.LogCaptureFixture - ) -> None: - import logging - - with caplog.at_level(logging.INFO, logger="arksim.llms.chat.llm"): - LLM._get_provider("open_responses") - assert any("open_responses" in rec.message for rec in caplog.records) - - def test_openai_provider_does_not_emit_info_log( - self, caplog: pytest.LogCaptureFixture - ) -> None: - import logging - - with caplog.at_level(logging.INFO, logger="arksim.llms.chat.llm"): - LLM._get_provider("openai") - assert not any("open_responses" in rec.message for rec in caplog.records) From e4f6722cded32e2f7942c034709bc83556b2ddd6 Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Fri, 15 May 2026 11:14:34 -0700 Subject: [PATCH 04/23] test(integration): add opt-in Open Responses smoke against OpenAI --- tests/integration/test_responses_provider.py | 50 ++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/integration/test_responses_provider.py diff --git a/tests/integration/test_responses_provider.py b/tests/integration/test_responses_provider.py new file mode 100644 index 00000000..c58b71e2 --- /dev/null +++ b/tests/integration/test_responses_provider.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Opt-in integration smoke for the responses provider against api.openai.com. + +Skipped unless OPENAI_API_KEY is set in the environment. One short call +verifies the alias wiring, the SDK call shape, and that we get a non-empty +response back. Cost: pennies per run. +""" + +from __future__ import annotations + +import os + +import pytest + +from arksim.llms.chat.llm import LLM + + +@pytest.mark.integration +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), + reason="OPENAI_API_KEY not set; opt-in smoke test skipped", +) +def test_responses_provider_smoke_against_openai() -> None: + llm = LLM(model="gpt-4.1-mini", provider="responses") + out = llm.call("Say only the word 'hello' and nothing else.") + assert isinstance(out, str) + assert "hello" in out.lower() + + +@pytest.mark.integration +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), + reason="OPENAI_API_KEY not set; opt-in smoke test skipped", +) +def test_responses_provider_smoke_with_explicit_base_url() -> None: + """Exercise the new base_url + api_key path against the real endpoint. + + Without an explicit base_url the provider falls back to env vars and + we would not be testing the new code path. This call forces both + the base_url and api_key plumbing to be exercised end-to-end. + """ + llm = LLM( + model="gpt-4.1-mini", + provider="responses", + base_url="https://api.openai.com/v1", + api_key=os.environ["OPENAI_API_KEY"], + ) + out = llm.call("Say only the word 'hello' and nothing else.") + assert isinstance(out, str) + assert "hello" in out.lower() From 478ca8e48fbd0e1245e8ed36ea945320630f8b14 Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Fri, 15 May 2026 11:18:35 -0700 Subject: [PATCH 05/23] test(integration): override smoke model via env, document retry cost --- tests/integration/test_responses_provider.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_responses_provider.py b/tests/integration/test_responses_provider.py index c58b71e2..4123c6fa 100644 --- a/tests/integration/test_responses_provider.py +++ b/tests/integration/test_responses_provider.py @@ -1,9 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 -"""Opt-in integration smoke for the responses provider against api.openai.com. +"""Opt-in integration smokes for the responses provider against api.openai.com. -Skipped unless OPENAI_API_KEY is set in the environment. One short call -verifies the alias wiring, the SDK call shape, and that we get a non-empty -response back. Cost: pennies per run. +Skipped unless OPENAI_API_KEY is set in the environment. Two short calls +verify (1) alias wiring through the default constructor and (2) explicit +base_url + api_key end-to-end through the OpenAI SDK. + +Override the model via ARKSIM_SMOKE_MODEL when gpt-4.1-mini is deprecated. + +Cost: pennies per run on the happy path. OpenAILLM.call is wrapped in +`@retry(max_retries=5)`; auth or model-deprecation failures may make up +to 6 attempts before surfacing the error. """ from __future__ import annotations @@ -14,6 +20,8 @@ from arksim.llms.chat.llm import LLM +_SMOKE_MODEL = os.environ.get("ARKSIM_SMOKE_MODEL", "gpt-4.1-mini") + @pytest.mark.integration @pytest.mark.skipif( @@ -21,7 +29,7 @@ reason="OPENAI_API_KEY not set; opt-in smoke test skipped", ) def test_responses_provider_smoke_against_openai() -> None: - llm = LLM(model="gpt-4.1-mini", provider="responses") + llm = LLM(model=_SMOKE_MODEL, provider="responses") out = llm.call("Say only the word 'hello' and nothing else.") assert isinstance(out, str) assert "hello" in out.lower() @@ -40,7 +48,7 @@ def test_responses_provider_smoke_with_explicit_base_url() -> None: the base_url and api_key plumbing to be exercised end-to-end. """ llm = LLM( - model="gpt-4.1-mini", + model=_SMOKE_MODEL, provider="responses", base_url="https://api.openai.com/v1", api_key=os.environ["OPENAI_API_KEY"], From e62bb206cee28d128abf7602999cb98991f45947 Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Fri, 15 May 2026 11:20:17 -0700 Subject: [PATCH 06/23] docs: add user-simulator-on-open-responses page --- .../main/user-simulator-on-open-responses.mdx | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 docs/main/user-simulator-on-open-responses.mdx diff --git a/docs/main/user-simulator-on-open-responses.mdx b/docs/main/user-simulator-on-open-responses.mdx new file mode 100644 index 00000000..420cf481 --- /dev/null +++ b/docs/main/user-simulator-on-open-responses.mdx @@ -0,0 +1,79 @@ +--- +title: 'User simulator on Open Responses' +description: 'Run the user-simulator LLM against any Open Responses-conforming backend.' +--- + +## What is Open Responses + +The [Open Responses spec](https://www.openresponses.org/) is a multi-vendor open standard for the `/v1/responses` API shape. It is essentially OpenAI's Responses API formalized as a shared spec, with backing from a consortium that includes OpenAI, NVIDIA, Vercel, Ollama, vLLM, and others. See [openresponses.org](https://www.openresponses.org/) for the current backer list. + +ArkSim's OpenAI provider has run on OpenAI's Responses API endpoint (`client.responses.parse`) since v0.3.0. The new `responses` provider alias lets you point the user-simulator LLM at any conforming backend by setting `base_url` and `api_key` in your config. Explicit `base_url` and `api_key` values in config take precedence over the `OPENAI_BASE_URL` and `OPENAI_API_KEY` environment variables; leave them unset (or empty) to fall back to env. + +## Why it matters + +ArkSim's user simulator sends a stable system prompt and scenario context to the LLM on every turn of every scenario. OpenAI's automatic prompt cache hits this stable prefix and discounts the cached portion by 50-75% depending on the model. For typical multi-turn runs, expect roughly 25-40% total cost reduction. The benefit was already active for OpenAI users; this page documents how to extend the same shape to self-hosted or alternate backends. + +## Configuration + +### OpenAI (default) + +```yaml +simulator: + llm: + provider: openai + model: gpt-4.1-mini +``` + +`api.openai.com` is the default. No `base_url` needed. + +### Ollama (local) + +```yaml +simulator: + llm: + provider: responses + model: llama3.1 + base_url: http://localhost:11434/v1 + api_key: ollama +``` + +Start Ollama (`ollama serve`), pull the model (`ollama pull llama3.1`), then run as usual. The `api_key` value is a placeholder; Ollama ignores it. + +### vLLM + +```yaml +simulator: + llm: + provider: responses + model: meta-llama/Llama-3.1-8B-Instruct + base_url: http://my-vllm-host:8000/v1 + api_key: ${VLLM_API_KEY} +``` + +### NVIDIA NIM, Vercel AI Gateway, OpenRouter, LM Studio, Llama Stack + +Same shape, different `base_url`. Consult each provider's documentation for the exact endpoint path and authentication header. + +## When to use `openai` vs `responses` + +- Use `provider: openai` when you are hitting OpenAI itself with default endpoints. Most users land here. +- Use `provider: responses` when you set a custom `base_url`. The alias makes the multi-provider intent explicit in your config. + +Both names resolve to the same Python class under the hood. The distinction is documentation, not behavior. + +## Verifying caching is working + +For OpenAI-hosted models, the response payload exposes `usage.input_tokens_details.cached_tokens` on every call. ArkSim does not surface this metric into reports yet (planned follow-up). For now, check the [OpenAI usage dashboard](https://platform.openai.com/usage) and look for `cached_input_tokens` on your ArkSim runs. You should see the cached fraction climb across turns of a single scenario. + +Self-hosted backends do not have OpenAI-equivalent automatic prefix caching out of the box. Ollama maintains a KV cache within a single process but does not persist it across HTTP calls in a way that benefits cross-turn arksim runs. vLLM has prefix caching but it is configuration-gated; consult vLLM's docs to enable it. + +## Caveats + +- The OpenAI prompt cache requires prompts of at least 1024 tokens to activate. Toy scenarios with tiny user profiles and no knowledge base will not hit it. +- Cache TTL is roughly 5 to 10 minutes. ArkSim runs fast enough that this is rarely a constraint, but a paused or debugged run can lose its cached prefix. +- ArkSim does not stream Responses API output today; the user simulator uses non-streaming `responses.parse()`. + +## Limitations + +- The `responses` provider currently forwards `model`, `temperature`, `base_url`, and `api_key`. Provider-specific kwargs (custom headers, organization IDs) need explicit support added. +- Anthropic and Google providers in ArkSim do not speak Open Responses; they use their native APIs. From 7b8d7e84135161599f25c6e0c0cbb39a198e4097 Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Fri, 15 May 2026 11:25:26 -0700 Subject: [PATCH 07/23] docs: polish user-simulator-on-open-responses page --- docs/main/user-simulator-on-open-responses.mdx | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/main/user-simulator-on-open-responses.mdx b/docs/main/user-simulator-on-open-responses.mdx index 420cf481..89c28f01 100644 --- a/docs/main/user-simulator-on-open-responses.mdx +++ b/docs/main/user-simulator-on-open-responses.mdx @@ -1,17 +1,17 @@ --- -title: 'User simulator on Open Responses' -description: 'Run the user-simulator LLM against any Open Responses-conforming backend.' +title: "User simulator on Open Responses" +description: "Run the user-simulator LLM against any Open Responses-conforming backend." --- ## What is Open Responses -The [Open Responses spec](https://www.openresponses.org/) is a multi-vendor open standard for the `/v1/responses` API shape. It is essentially OpenAI's Responses API formalized as a shared spec, with backing from a consortium that includes OpenAI, NVIDIA, Vercel, Ollama, vLLM, and others. See [openresponses.org](https://www.openresponses.org/) for the current backer list. +The [Open Responses spec](https://www.openresponses.org/) is a multi-vendor open standard for the `/v1/responses` API shape. It is essentially OpenAI's Responses API formalized as a shared spec, backed by a consortium that includes OpenAI and NVIDIA among others. See [openresponses.org](https://www.openresponses.org/) for the current backer list. ArkSim's OpenAI provider has run on OpenAI's Responses API endpoint (`client.responses.parse`) since v0.3.0. The new `responses` provider alias lets you point the user-simulator LLM at any conforming backend by setting `base_url` and `api_key` in your config. Explicit `base_url` and `api_key` values in config take precedence over the `OPENAI_BASE_URL` and `OPENAI_API_KEY` environment variables; leave them unset (or empty) to fall back to env. ## Why it matters -ArkSim's user simulator sends a stable system prompt and scenario context to the LLM on every turn of every scenario. OpenAI's automatic prompt cache hits this stable prefix and discounts the cached portion by 50-75% depending on the model. For typical multi-turn runs, expect roughly 25-40% total cost reduction. The benefit was already active for OpenAI users; this page documents how to extend the same shape to self-hosted or alternate backends. +ArkSim's user simulator sends a stable system prompt and scenario context to the LLM on every turn of every scenario. OpenAI's automatic prompt cache hits this stable prefix and discounts the cached portion by 50 to 75% depending on the model. Depending on prompt-cache hit rate and the output share of total cost, typical multi-turn runs see roughly 20 to 40% total cost reduction. The benefit was already active for OpenAI users; this page documents how to extend the same shape to self-hosted or alternate backends. ## Configuration @@ -26,6 +26,8 @@ simulator: `api.openai.com` is the default. No `base_url` needed. +Requires `OPENAI_API_KEY` in your environment. + ### Ollama (local) ```yaml @@ -50,6 +52,8 @@ simulator: api_key: ${VLLM_API_KEY} ``` +Start vLLM with `vllm serve --port 8000`, then point `base_url` at the running host. + ### NVIDIA NIM, Vercel AI Gateway, OpenRouter, LM Studio, Llama Stack Same shape, different `base_url`. Consult each provider's documentation for the exact endpoint path and authentication header. @@ -63,9 +67,9 @@ Both names resolve to the same Python class under the hood. The distinction is d ## Verifying caching is working -For OpenAI-hosted models, the response payload exposes `usage.input_tokens_details.cached_tokens` on every call. ArkSim does not surface this metric into reports yet (planned follow-up). For now, check the [OpenAI usage dashboard](https://platform.openai.com/usage) and look for `cached_input_tokens` on your ArkSim runs. You should see the cached fraction climb across turns of a single scenario. +For OpenAI-hosted models, the response payload exposes `usage.input_tokens_details.cached_tokens` on every call. ArkSim does not surface this metric into reports yet. For now, check the [OpenAI usage dashboard](https://platform.openai.com/usage) and look for `cached_input_tokens` on your ArkSim runs. You should see the cached fraction climb across turns of a single scenario. -Self-hosted backends do not have OpenAI-equivalent automatic prefix caching out of the box. Ollama maintains a KV cache within a single process but does not persist it across HTTP calls in a way that benefits cross-turn arksim runs. vLLM has prefix caching but it is configuration-gated; consult vLLM's docs to enable it. +Self-hosted backends do not have OpenAI-equivalent automatic prefix caching out of the box. Ollama maintains a KV cache within a single process but does not persist it across HTTP calls in a way that benefits cross-turn ArkSim runs. vLLM has prefix caching but it is configuration-gated; consult vLLM's docs to enable it. ## Caveats From 99bae64db33f3f54efbfcd4a2c4ba8251c950c69 Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Fri, 15 May 2026 11:27:48 -0700 Subject: [PATCH 08/23] docs: link Open Responses user-simulator page into nav --- docs/docs.json | 1 + docs/main/simulate-conversation.mdx | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/docs/docs.json b/docs/docs.json index 42969ec3..b2633bf0 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -109,6 +109,7 @@ "pages": [ "main/build-scenario", "main/simulate-conversation", + "main/user-simulator-on-open-responses", "main/evaluate-conversation", "main/tool-call-capture", "main/schema-reference" diff --git a/docs/main/simulate-conversation.mdx b/docs/main/simulate-conversation.mdx index 0f442b0b..b92f63ab 100644 --- a/docs/main/simulate-conversation.mdx +++ b/docs/main/simulate-conversation.mdx @@ -64,6 +64,10 @@ provider: openai # simulated_user_prompt_template: null ``` + +The user simulator can run against any [Open Responses](https://www.openresponses.org/)-conforming backend (Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter, and others) via `provider: responses`. See [User simulator on Open Responses](/main/user-simulator-on-open-responses) for the configuration and the prompt-caching benefit on OpenAI. + + The default `num_workers` is `50`. Set `num_workers: auto` to automatically parallelize across all conversations, or specify a fixed number to control load on your agent endpoint. From 0ed8ca71a38b73b6849462da8b256afba47e2d51 Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Fri, 15 May 2026 11:30:17 -0700 Subject: [PATCH 09/23] docs: use relative link for user-simulator-on-open-responses --- docs/main/simulate-conversation.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/main/simulate-conversation.mdx b/docs/main/simulate-conversation.mdx index b92f63ab..9771ceb9 100644 --- a/docs/main/simulate-conversation.mdx +++ b/docs/main/simulate-conversation.mdx @@ -65,7 +65,7 @@ provider: openai ``` -The user simulator can run against any [Open Responses](https://www.openresponses.org/)-conforming backend (Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter, and others) via `provider: responses`. See [User simulator on Open Responses](/main/user-simulator-on-open-responses) for the configuration and the prompt-caching benefit on OpenAI. +The user simulator can run against any [Open Responses](https://www.openresponses.org/)-conforming backend (Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter, and others) via `provider: responses`. See [User simulator on Open Responses](./user-simulator-on-open-responses) for the configuration and the prompt-caching benefit on OpenAI. From ce2b8ea488e5d3139175a6071c59d91fd9d6315c Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Fri, 15 May 2026 11:31:33 -0700 Subject: [PATCH 10/23] docs(examples): show Ollama swap for user simulator --- examples/customer-service/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/examples/customer-service/README.md b/examples/customer-service/README.md index a5f7733e..a19889df 100644 --- a/examples/customer-service/README.md +++ b/examples/customer-service/README.md @@ -127,3 +127,18 @@ The SQLite database (`store.db`) is created automatically on first run with samp - 4 orders (shipped, processing, delivered, cancelled) - 6 products (laptops, headphones, accessories) - 2 verification codes (one per customer) + +### Running the user simulator on a self-hosted backend + +The user-simulator LLM defaults to OpenAI. To swap it for any [Open Responses](https://www.openresponses.org/)-conforming backend (Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter), replace the `simulator.llm` block in `config.yaml` with: + +```yaml +simulator: + llm: + provider: responses + model: llama3.1 + base_url: http://localhost:11434/v1 + api_key: ollama +``` + +Then start the local server. For Ollama: `ollama serve` and `ollama pull llama3.1`. See [User simulator on Open Responses](https://docs.arklex.ai/main/user-simulator-on-open-responses) for vLLM, NIM, and other backends. From 47d8d77bdee59e7aa179d882594c022605c8a494 Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Fri, 15 May 2026 11:33:43 -0700 Subject: [PATCH 11/23] docs: changelog entry for Open Responses LLM provider --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85e4d2c5..07bd6069 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * **evaluator:** error-to-scenario mappings included in `evaluation.json` output * **evaluator:** shared evaluation constants (`SCORE_NOT_COMPUTED`, `BEHAVIOR_FAILURE_THRESHOLD`) * **a2a:** add tool call capture via A2A AgentExtension (`https://arksim.arklex.ai/a2a/tool-call-capture/v1`); agents declare the extension in the AgentCard and surface tool calls in `Task.artifacts[*].metadata`. See [Tool Call Capture docs](https://docs.arklex.ai/main/tool-call-capture) +* **llm:** user simulator can now run on any Open Responses-conforming backend via `provider: responses` with configurable `base_url` and `api_key`. Works with OpenAI, Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter, LM Studio, and other consortium backers. The OpenAI provider has run on `/v1/responses` since v0.3.0; this change unlocks multi-provider reach. * **report:** scenario IDs displayed in HTML report error cards * **ui:** version display in web UI sidebar (next to "Arksim" title) * **api:** path traversal protection on filesystem and results API endpoints From ef7c89b856691385c96e2c6eb1a0eac66e03bf50 Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Fri, 15 May 2026 13:54:02 -0700 Subject: [PATCH 12/23] feat(llm): plumb base_url/api_key from YAML to LLM construction --- arksim/evaluator/entities.py | 17 +++++ arksim/evaluator/evaluator.py | 2 + arksim/simulation_engine/entities.py | 17 +++++ arksim/simulation_engine/simulator.py | 2 + tests/unit/test_evaluation_input.py | 21 ++++++ tests/unit/test_responses_yaml_plumbing.py | 88 ++++++++++++++++++++++ tests/unit/test_simulation_input.py | 16 ++++ 7 files changed, 163 insertions(+) create mode 100644 tests/unit/test_responses_yaml_plumbing.py diff --git a/arksim/evaluator/entities.py b/arksim/evaluator/entities.py index ff3af7db..e9384dd7 100644 --- a/arksim/evaluator/entities.py +++ b/arksim/evaluator/entities.py @@ -57,6 +57,23 @@ class EvaluationInput(BaseModel): ) model: str = Field(default=DEFAULT_MODEL, description="LLM model for evaluation") provider: str | None = Field(default=DEFAULT_PROVIDER, description="LLM provider") + base_url: str | None = Field( + default=None, + description=( + "Optional custom endpoint for the evaluator LLM. Use to point at any " + "Open Responses-conforming backend (Ollama, vLLM, NVIDIA NIM, Vercel AI " + "Gateway, OpenRouter, etc.). Defaults to the SDK's env var " + "(OPENAI_BASE_URL) or the canonical provider endpoint." + ), + ) + api_key: str | None = Field( + default=None, + description=( + "Optional API key for the evaluator LLM. Pair with base_url for " + "self-hosted or alternate Responses backends. Defaults to the SDK's " + "env var (OPENAI_API_KEY)." + ), + ) num_workers: int | str = Field( default=50, description="Number of parallel workers (use 'auto' to default to 4)", diff --git a/arksim/evaluator/evaluator.py b/arksim/evaluator/evaluator.py index 87d7a687..d222274a 100644 --- a/arksim/evaluator/evaluator.py +++ b/arksim/evaluator/evaluator.py @@ -941,6 +941,8 @@ def run_evaluation( llm = LLM( model=settings.model, provider=settings.provider, + base_url=settings.base_url, + api_key=settings.api_key, ) all_quant, all_qual = _load_custom_metrics( settings.custom_metrics_file_paths, llm=llm diff --git a/arksim/simulation_engine/entities.py b/arksim/simulation_engine/entities.py index 22ac0fda..bf45e193 100644 --- a/arksim/simulation_engine/entities.py +++ b/arksim/simulation_engine/entities.py @@ -41,6 +41,23 @@ class SimulationInput(BaseModel): ) model: str = Field(default=DEFAULT_MODEL, description="LLM model for simulation") provider: str | None = Field(default=DEFAULT_PROVIDER, description="LLM provider") + base_url: str | None = Field( + default=None, + description=( + "Optional custom endpoint for the simulator LLM. Use to point at any " + "Open Responses-conforming backend (Ollama, vLLM, NVIDIA NIM, Vercel AI " + "Gateway, OpenRouter, etc.). Defaults to the SDK's env var " + "(OPENAI_BASE_URL) or the canonical provider endpoint." + ), + ) + api_key: str | None = Field( + default=None, + description=( + "Optional API key for the simulator LLM. Pair with base_url for " + "self-hosted or alternate Responses backends. Defaults to the SDK's " + "env var (OPENAI_API_KEY)." + ), + ) num_conversations_per_scenario: int = Field( default=5, description="Number of conversations per scenario to simulate" ) diff --git a/arksim/simulation_engine/simulator.py b/arksim/simulation_engine/simulator.py index 206799bc..952d677c 100644 --- a/arksim/simulation_engine/simulator.py +++ b/arksim/simulation_engine/simulator.py @@ -486,6 +486,8 @@ async def run_simulation( llm = LLM( model=settings.model, provider=settings.provider, + base_url=settings.base_url, + api_key=settings.api_key, ) simulation_params = SimulationParams( diff --git a/tests/unit/test_evaluation_input.py b/tests/unit/test_evaluation_input.py index 60a16ccf..a80f0741 100644 --- a/tests/unit/test_evaluation_input.py +++ b/tests/unit/test_evaluation_input.py @@ -1,12 +1,33 @@ # SPDX-License-Identifier: Apache-2.0 """Tests for arksim.evaluator.entities.EvaluationInput validator.""" +from __future__ import annotations + from pathlib import Path from typing import Any from arksim.evaluator.entities import EvaluationInput +class TestEvaluationInputFields: + """Tests for newly added optional LLM transport fields.""" + + def test_evaluation_input_accepts_base_url_and_api_key(self) -> None: + settings = EvaluationInput( + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + ) + assert settings.base_url == "http://localhost:11434/v1" + assert settings.api_key == "ollama" + + def test_evaluation_input_defaults_base_url_and_api_key_to_none(self) -> None: + settings = EvaluationInput() + assert settings.base_url is None + assert settings.api_key is None + + class TestEvaluationInputPathResolution: """Tests for config-relative path resolution in EvaluationInput.""" diff --git a/tests/unit/test_responses_yaml_plumbing.py b/tests/unit/test_responses_yaml_plumbing.py new file mode 100644 index 00000000..bbbd6390 --- /dev/null +++ b/tests/unit/test_responses_yaml_plumbing.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +"""End-to-end plumbing test: YAML Pydantic -> LLM(...) -> OpenAI SDK kwargs. + +Verifies that base_url and api_key set on SimulationInput / EvaluationInput +travel through model_dump() and LLM(...) construction down to the underlying +OpenAI / AsyncOpenAI client constructors. This guards against the regression +where the fields were silently dropped by Pydantic's default extra='ignore'. +""" + +from __future__ import annotations + +from unittest.mock import patch + +from arksim.evaluator.entities import EvaluationInput +from arksim.llms.chat.llm import LLM +from arksim.simulation_engine.entities import SimulationInput + + +class TestResponsesYamlPlumbing: + """SimulationInput / EvaluationInput -> LLM -> OpenAI SDK forwarding.""" + + def test_simulation_input_forwards_base_url_and_api_key_to_openai(self) -> None: + settings = SimulationInput( + agent_config_file_path="agent.json", + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + ) + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + LLM( + model=settings.model, + provider=settings.provider, + base_url=settings.base_url, + api_key=settings.api_key, + ) + mock_sync.assert_called_once_with( + base_url="http://localhost:11434/v1", api_key="ollama" + ) + mock_async.assert_called_once_with( + base_url="http://localhost:11434/v1", api_key="ollama" + ) + + def test_evaluation_input_forwards_base_url_and_api_key_to_openai(self) -> None: + settings = EvaluationInput( + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + ) + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + LLM( + model=settings.model, + provider=settings.provider, + base_url=settings.base_url, + api_key=settings.api_key, + ) + mock_sync.assert_called_once_with( + base_url="http://localhost:11434/v1", api_key="ollama" + ) + mock_async.assert_called_once_with( + base_url="http://localhost:11434/v1", api_key="ollama" + ) + + def test_simulation_input_defaults_omit_base_url_and_api_key(self) -> None: + settings = SimulationInput( + agent_config_file_path="agent.json", provider="responses" + ) + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + LLM( + model=settings.model, + provider=settings.provider, + base_url=settings.base_url, + api_key=settings.api_key, + ) + # base_url=None and api_key=None must not be forwarded to the SDK; + # the SDK falls back to OPENAI_BASE_URL / OPENAI_API_KEY env vars. + mock_sync.assert_called_once_with() + mock_async.assert_called_once_with() diff --git a/tests/unit/test_simulation_input.py b/tests/unit/test_simulation_input.py index 818fc35b..ec61ac56 100644 --- a/tests/unit/test_simulation_input.py +++ b/tests/unit/test_simulation_input.py @@ -29,6 +29,22 @@ def test_invalid_num_workers_string(self) -> None: with pytest.raises(ValidationError, match="num_workers"): SimulationInput(agent_config_file_path="a.json", num_workers="fast") + def test_simulation_input_accepts_base_url_and_api_key(self) -> None: + settings = SimulationInput( + agent_config_file_path="agent.json", + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + ) + assert settings.base_url == "http://localhost:11434/v1" + assert settings.api_key == "ollama" + + def test_simulation_input_defaults_base_url_and_api_key_to_none(self) -> None: + settings = SimulationInput(agent_config_file_path="agent.json") + assert settings.base_url is None + assert settings.api_key is None + class TestSimulationInputPathResolution: """Tests for config-relative path resolution in SimulationInput.""" From 09217345b6dcfa74c57514916c9036ae27215d80 Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Mon, 18 May 2026 06:34:22 -0700 Subject: [PATCH 13/23] docs: align Open Responses examples with flat YAML schema --- .../main/user-simulator-on-open-responses.mdx | 28 ++++++++----------- examples/customer-service/README.md | 12 ++++---- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/docs/main/user-simulator-on-open-responses.mdx b/docs/main/user-simulator-on-open-responses.mdx index 89c28f01..313feb34 100644 --- a/docs/main/user-simulator-on-open-responses.mdx +++ b/docs/main/user-simulator-on-open-responses.mdx @@ -9,6 +9,8 @@ The [Open Responses spec](https://www.openresponses.org/) is a multi-vendor open ArkSim's OpenAI provider has run on OpenAI's Responses API endpoint (`client.responses.parse`) since v0.3.0. The new `responses` provider alias lets you point the user-simulator LLM at any conforming backend by setting `base_url` and `api_key` in your config. Explicit `base_url` and `api_key` values in config take precedence over the `OPENAI_BASE_URL` and `OPENAI_API_KEY` environment variables; leave them unset (or empty) to fall back to env. +These keys go at the top level of your `config.yaml`, alongside other simulation settings. + ## Why it matters ArkSim's user simulator sends a stable system prompt and scenario context to the LLM on every turn of every scenario. OpenAI's automatic prompt cache hits this stable prefix and discounts the cached portion by 50 to 75% depending on the model. Depending on prompt-cache hit rate and the output share of total cost, typical multi-turn runs see roughly 20 to 40% total cost reduction. The benefit was already active for OpenAI users; this page documents how to extend the same shape to self-hosted or alternate backends. @@ -18,10 +20,8 @@ ArkSim's user simulator sends a stable system prompt and scenario context to the ### OpenAI (default) ```yaml -simulator: - llm: - provider: openai - model: gpt-4.1-mini +provider: openai +model: gpt-4.1-mini ``` `api.openai.com` is the default. No `base_url` needed. @@ -31,12 +31,10 @@ Requires `OPENAI_API_KEY` in your environment. ### Ollama (local) ```yaml -simulator: - llm: - provider: responses - model: llama3.1 - base_url: http://localhost:11434/v1 - api_key: ollama +provider: responses +model: llama3.1 +base_url: http://localhost:11434/v1 +api_key: ollama ``` Start Ollama (`ollama serve`), pull the model (`ollama pull llama3.1`), then run as usual. The `api_key` value is a placeholder; Ollama ignores it. @@ -44,12 +42,10 @@ Start Ollama (`ollama serve`), pull the model (`ollama pull llama3.1`), then run ### vLLM ```yaml -simulator: - llm: - provider: responses - model: meta-llama/Llama-3.1-8B-Instruct - base_url: http://my-vllm-host:8000/v1 - api_key: ${VLLM_API_KEY} +provider: responses +model: meta-llama/Llama-3.1-8B-Instruct +base_url: http://my-vllm-host:8000/v1 +api_key: ${VLLM_API_KEY} ``` Start vLLM with `vllm serve --port 8000`, then point `base_url` at the running host. diff --git a/examples/customer-service/README.md b/examples/customer-service/README.md index a19889df..09d2a5b5 100644 --- a/examples/customer-service/README.md +++ b/examples/customer-service/README.md @@ -130,15 +130,13 @@ The SQLite database (`store.db`) is created automatically on first run with samp ### Running the user simulator on a self-hosted backend -The user-simulator LLM defaults to OpenAI. To swap it for any [Open Responses](https://www.openresponses.org/)-conforming backend (Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter), replace the `simulator.llm` block in `config.yaml` with: +The user-simulator LLM defaults to OpenAI. To swap it for any [Open Responses](https://www.openresponses.org/)-conforming backend (Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter), replace the LLM configuration in `config.yaml` (the `model` and `provider` keys near the top) with: ```yaml -simulator: - llm: - provider: responses - model: llama3.1 - base_url: http://localhost:11434/v1 - api_key: ollama +provider: responses +model: llama3.1 +base_url: http://localhost:11434/v1 +api_key: ollama ``` Then start the local server. For Ollama: `ollama serve` and `ollama pull llama3.1`. See [User simulator on Open Responses](https://docs.arklex.ai/main/user-simulator-on-open-responses) for vLLM, NIM, and other backends. From 99c5957f4282dd41e41dac38b6e113fb9ee6afe1 Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Mon, 18 May 2026 10:54:25 -0700 Subject: [PATCH 14/23] fix(llm): strip whitespace, wrap missing-credentials error --- arksim/llms/chat/llm.py | 3 ++ arksim/llms/chat/providers/openai.py | 30 ++++++++++------ tests/unit/test_openai_llm_provider.py | 48 ++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 11 deletions(-) diff --git a/arksim/llms/chat/llm.py b/arksim/llms/chat/llm.py index 3922245b..175354ae 100644 --- a/arksim/llms/chat/llm.py +++ b/arksim/llms/chat/llm.py @@ -24,6 +24,9 @@ def __new__(cls, model: str, **kwargs: object) -> BaseLLM: @classmethod def _get_provider(cls, provider: str) -> type: if provider in ("openai", "responses"): + # `responses` is a documentation-only alias signaling multi-provider + # intent (Open Responses spec). Both names resolve to OpenAILLM. + # See docs/main/user-simulator-on-open-responses.mdx. from arksim.llms.chat.providers.openai import OpenAILLM return OpenAILLM diff --git a/arksim/llms/chat/providers/openai.py b/arksim/llms/chat/providers/openai.py index 5dad62e8..9e1c7bd2 100644 --- a/arksim/llms/chat/providers/openai.py +++ b/arksim/llms/chat/providers/openai.py @@ -3,7 +3,7 @@ from typing import Any, TypeVar, overload -from openai import AsyncOpenAI, OpenAI +from openai import AsyncOpenAI, OpenAI, OpenAIError from pydantic import BaseModel from arksim.llms.chat.base.base_llm import BaseLLM @@ -24,17 +24,25 @@ def __init__( **kwargs: object, ) -> None: super().__init__(model, provider, temperature, **kwargs) - # Coerce empty strings to omitted: YAML configs commonly produce "" - # for unset fields, and an empty base_url or api_key would otherwise - # confuse the SDK rather than falling back to OPENAI_BASE_URL / - # OPENAI_API_KEY env vars. + # Coerce empty or whitespace-only strings to omitted, then fall back + # to OPENAI_BASE_URL / OPENAI_API_KEY env vars. YAML configs commonly + # produce "" for unset fields; whitespace-only values would otherwise + # be URL-encoded by the SDK and cause hard-to-diagnose hangs. client_kwargs: dict[str, str] = {} - if base_url: - client_kwargs["base_url"] = base_url - if api_key: - client_kwargs["api_key"] = api_key - self.client = OpenAI(**client_kwargs) - self.async_client = AsyncOpenAI(**client_kwargs) + if base_url and base_url.strip(): + client_kwargs["base_url"] = base_url.strip() + if api_key and api_key.strip(): + client_kwargs["api_key"] = api_key.strip() + try: + self.client = OpenAI(**client_kwargs) + self.async_client = AsyncOpenAI(**client_kwargs) + except OpenAIError as exc: + raise ValueError( + "OpenAI SDK requires an api_key. Set 'api_key' in your " + "config.yaml (alongside 'model' and 'provider') or set the " + "OPENAI_API_KEY environment variable. " + "See https://docs.arklex.ai/main/user-simulator-on-open-responses" + ) from exc def _prepare_params( self, diff --git a/tests/unit/test_openai_llm_provider.py b/tests/unit/test_openai_llm_provider.py index a48623af..38143cde 100644 --- a/tests/unit/test_openai_llm_provider.py +++ b/tests/unit/test_openai_llm_provider.py @@ -5,6 +5,9 @@ from unittest.mock import patch +import pytest +from openai import OpenAIError + from arksim.llms.chat.providers.openai import OpenAILLM @@ -89,3 +92,48 @@ def test_temperature_flows_through_with_base_url(self) -> None: ) assert llm.temperature == 0.7 assert llm.model == "gpt-4.1-mini" + + def test_whitespace_base_url_and_api_key_treated_as_omitted(self) -> None: + """Whitespace-only values must not reach the SDK; they would be + URL-encoded into `%20...` and cause slow connection failures. + """ + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM(model="gpt-4o-mini", base_url=" ", api_key="\t\n") + mock_sync.assert_called_once_with() + mock_async.assert_called_once_with() + + def test_base_url_and_api_key_are_stripped(self) -> None: + """Leading and trailing whitespace is stripped before passing to SDK.""" + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, + ): + OpenAILLM( + model="llama3.1", + base_url=" http://localhost:11434/v1 ", + api_key=" ollama ", + ) + mock_sync.assert_called_once_with( + base_url="http://localhost:11434/v1", api_key="ollama" + ) + mock_async.assert_called_once_with( + base_url="http://localhost:11434/v1", api_key="ollama" + ) + + def test_missing_credentials_raises_actionable_error(self) -> None: + """When the OpenAI SDK rejects no-credentials construction, wrap the + error with an arksim-specific message pointing at the YAML field + and docs URL. The user should not see a bare OpenAIError stack trace. + """ + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI"), + ): + mock_sync.side_effect = OpenAIError( + "The api_key client option must be set..." + ) + with pytest.raises(ValueError, match="OpenAI SDK requires an api_key"): + OpenAILLM(model="gpt-4o-mini") From 670f937b4b38d652e9b09f9eb05592777f451b3d Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Mon, 18 May 2026 10:54:31 -0700 Subject: [PATCH 15/23] docs: scope Open Responses to simulator, fix vendor accuracy --- docs/main/user-simulator-on-open-responses.mdx | 16 +++++++++++++--- examples/customer-service/README.md | 4 +++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/main/user-simulator-on-open-responses.mdx b/docs/main/user-simulator-on-open-responses.mdx index 313feb34..e5edad23 100644 --- a/docs/main/user-simulator-on-open-responses.mdx +++ b/docs/main/user-simulator-on-open-responses.mdx @@ -5,7 +5,7 @@ description: "Run the user-simulator LLM against any Open Responses-conforming b ## What is Open Responses -The [Open Responses spec](https://www.openresponses.org/) is a multi-vendor open standard for the `/v1/responses` API shape. It is essentially OpenAI's Responses API formalized as a shared spec, backed by a consortium that includes OpenAI and NVIDIA among others. See [openresponses.org](https://www.openresponses.org/) for the current backer list. +The [Open Responses spec](https://www.openresponses.org/) is a multi-vendor open standard for the `/v1/responses` API shape. It is essentially OpenAI's Responses API formalized as a shared spec, backed by a multi-vendor consortium including NVIDIA, AWS, Databricks, Hugging Face, Vercel, OpenRouter, Ollama, vLLM, LM Studio, Llama Stack, Red Hat, and OpenAI. See [openresponses.org](https://www.openresponses.org/) for the current backer list. ArkSim's OpenAI provider has run on OpenAI's Responses API endpoint (`client.responses.parse`) since v0.3.0. The new `responses` provider alias lets you point the user-simulator LLM at any conforming backend by setting `base_url` and `api_key` in your config. Explicit `base_url` and `api_key` values in config take precedence over the `OPENAI_BASE_URL` and `OPENAI_API_KEY` environment variables; leave them unset (or empty) to fall back to env. @@ -15,6 +15,12 @@ These keys go at the top level of your `config.yaml`, alongside other simulation ArkSim's user simulator sends a stable system prompt and scenario context to the LLM on every turn of every scenario. OpenAI's automatic prompt cache hits this stable prefix and discounts the cached portion by 50 to 75% depending on the model. Depending on prompt-cache hit rate and the output share of total cost, typical multi-turn runs see roughly 20 to 40% total cost reduction. The benefit was already active for OpenAI users; this page documents how to extend the same shape to self-hosted or alternate backends. +## Scope: user simulator only + +The `responses` provider works for the user-simulator LLM. The evaluator LLM also reads these keys, but the evaluator passes `text_format` (OpenAI structured output) on every metric call. Self-hosted backends like Ollama do not currently accept `text_format`; vLLM accepts it only with `--guided-decoding-backend` enabled. + +For evaluation, keep `provider: openai` (or another fully-compatible backend with structured-output support) unless you have explicitly verified your backend handles `text_format`. A safe pattern is to set `provider: responses` and a custom `base_url` for simulation, then either accept the schema risk on the evaluator or run evaluation as a separate step with an OpenAI-compatible LLM. + ## Configuration ### OpenAI (default) @@ -39,15 +45,19 @@ api_key: ollama Start Ollama (`ollama serve`), pull the model (`ollama pull llama3.1`), then run as usual. The `api_key` value is a placeholder; Ollama ignores it. +Requires Ollama v0.13.3 or newer for `/v1/responses` support. The endpoint is currently experimental in Ollama and only the non-stateful flavor is implemented. + ### vLLM ```yaml provider: responses model: meta-llama/Llama-3.1-8B-Instruct base_url: http://my-vllm-host:8000/v1 -api_key: ${VLLM_API_KEY} +api_key: ``` +arksim does not expand `${VAR}` substitutions in YAML. Paste the literal key, or omit the `api_key` field and set `OPENAI_API_KEY` in your shell environment. + Start vLLM with `vllm serve --port 8000`, then point `base_url` at the running host. ### NVIDIA NIM, Vercel AI Gateway, OpenRouter, LM Studio, Llama Stack @@ -65,7 +75,7 @@ Both names resolve to the same Python class under the hood. The distinction is d For OpenAI-hosted models, the response payload exposes `usage.input_tokens_details.cached_tokens` on every call. ArkSim does not surface this metric into reports yet. For now, check the [OpenAI usage dashboard](https://platform.openai.com/usage) and look for `cached_input_tokens` on your ArkSim runs. You should see the cached fraction climb across turns of a single scenario. -Self-hosted backends do not have OpenAI-equivalent automatic prefix caching out of the box. Ollama maintains a KV cache within a single process but does not persist it across HTTP calls in a way that benefits cross-turn ArkSim runs. vLLM has prefix caching but it is configuration-gated; consult vLLM's docs to enable it. +Self-hosted backends do not have OpenAI-equivalent automatic prefix caching out of the box. Ollama maintains a KV cache within a single process but does not persist it across HTTP calls in a way that benefits cross-turn ArkSim runs. vLLM enables prefix caching by default on the V1 engine (the default since v0.8.1, April 2025). Older deployments still on V0 must opt in with `--enable-prefix-caching`. ## Caveats diff --git a/examples/customer-service/README.md b/examples/customer-service/README.md index 09d2a5b5..d764db37 100644 --- a/examples/customer-service/README.md +++ b/examples/customer-service/README.md @@ -139,4 +139,6 @@ base_url: http://localhost:11434/v1 api_key: ollama ``` -Then start the local server. For Ollama: `ollama serve` and `ollama pull llama3.1`. See [User simulator on Open Responses](https://docs.arklex.ai/main/user-simulator-on-open-responses) for vLLM, NIM, and other backends. +Then start the local server. For Ollama: `ollama serve` and `ollama pull llama3.1` (requires Ollama v0.13.3 or newer). + +**Scope note:** these keys also configure the evaluator LLM, which passes structured-output (`text_format`) calls that Ollama does not currently accept. Run simulation against Ollama, then run evaluation separately with `provider: openai` (or set the env var override) until evaluator-side multi-provider support lands. See [User simulator on Open Responses](https://docs.arklex.ai/main/user-simulator-on-open-responses) for vLLM, NIM, and other backends. From c0bf468f1d18cedc83562b65ceb66c47eedf2211 Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Tue, 19 May 2026 05:47:02 -0700 Subject: [PATCH 16/23] feat(evaluator): allow simulator/evaluator LLM split via evaluator_* overrides --- CHANGELOG.md | 2 +- arksim/evaluator/entities.py | 30 +++++++++++ arksim/evaluator/evaluator.py | 8 +-- .../main/user-simulator-on-open-responses.mdx | 20 +++++-- examples/customer-service/README.md | 15 ++++-- tests/unit/test_evaluation_input.py | 41 ++++++++++++++ tests/unit/test_responses_yaml_plumbing.py | 54 +++++++++++++++++++ 7 files changed, 159 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07bd6069..d8049a59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * **evaluator:** error-to-scenario mappings included in `evaluation.json` output * **evaluator:** shared evaluation constants (`SCORE_NOT_COMPUTED`, `BEHAVIOR_FAILURE_THRESHOLD`) * **a2a:** add tool call capture via A2A AgentExtension (`https://arksim.arklex.ai/a2a/tool-call-capture/v1`); agents declare the extension in the AgentCard and surface tool calls in `Task.artifacts[*].metadata`. See [Tool Call Capture docs](https://docs.arklex.ai/main/tool-call-capture) -* **llm:** user simulator can now run on any Open Responses-conforming backend via `provider: responses` with configurable `base_url` and `api_key`. Works with OpenAI, Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter, LM Studio, and other consortium backers. The OpenAI provider has run on `/v1/responses` since v0.3.0; this change unlocks multi-provider reach. +* **llm:** user simulator can now run on any Open Responses-conforming backend via `provider: responses` with configurable `base_url` and `api_key`. Works with OpenAI, Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter, LM Studio, and other consortium backers. The OpenAI provider has run on `/v1/responses` since v0.3.0; this change unlocks multi-provider reach. Add `evaluator_model`, `evaluator_provider`, `evaluator_base_url`, and `evaluator_api_key` for split simulator/evaluator backends (e.g. Ollama simulator + OpenAI evaluator in a single `config.yaml`). * **report:** scenario IDs displayed in HTML report error cards * **ui:** version display in web UI sidebar (next to "Arksim" title) * **api:** path traversal protection on filesystem and results API endpoints diff --git a/arksim/evaluator/entities.py b/arksim/evaluator/entities.py index e9384dd7..08fad82e 100644 --- a/arksim/evaluator/entities.py +++ b/arksim/evaluator/entities.py @@ -74,6 +74,36 @@ class EvaluationInput(BaseModel): "env var (OPENAI_API_KEY)." ), ) + evaluator_model: str | None = Field( + default=None, + description=( + "Optional override for the evaluator's LLM model. When unset, " + "falls back to `model`." + ), + ) + evaluator_provider: str | None = Field( + default=None, + description=( + "Optional override for the evaluator's LLM provider. Useful when " + "the simulator runs on a self-hosted backend (e.g. Ollama) that " + "does not support structured output, while the evaluator needs " + "OpenAI-compatible `text_format` support." + ), + ) + evaluator_base_url: str | None = Field( + default=None, + description=( + "Optional override for the evaluator LLM's custom endpoint. " + "When unset, falls back to `base_url`." + ), + ) + evaluator_api_key: str | None = Field( + default=None, + description=( + "Optional override for the evaluator LLM's API key. When unset, " + "falls back to `api_key`." + ), + ) num_workers: int | str = Field( default=50, description="Number of parallel workers (use 'auto' to default to 4)", diff --git a/arksim/evaluator/evaluator.py b/arksim/evaluator/evaluator.py index d222274a..cdf3c546 100644 --- a/arksim/evaluator/evaluator.py +++ b/arksim/evaluator/evaluator.py @@ -939,10 +939,10 @@ def run_evaluation( ) llm = LLM( - model=settings.model, - provider=settings.provider, - base_url=settings.base_url, - api_key=settings.api_key, + model=settings.evaluator_model or settings.model, + provider=settings.evaluator_provider or settings.provider, + base_url=settings.evaluator_base_url or settings.base_url, + api_key=settings.evaluator_api_key or settings.api_key, ) all_quant, all_qual = _load_custom_metrics( settings.custom_metrics_file_paths, llm=llm diff --git a/docs/main/user-simulator-on-open-responses.mdx b/docs/main/user-simulator-on-open-responses.mdx index e5edad23..85a14559 100644 --- a/docs/main/user-simulator-on-open-responses.mdx +++ b/docs/main/user-simulator-on-open-responses.mdx @@ -15,11 +15,25 @@ These keys go at the top level of your `config.yaml`, alongside other simulation ArkSim's user simulator sends a stable system prompt and scenario context to the LLM on every turn of every scenario. OpenAI's automatic prompt cache hits this stable prefix and discounts the cached portion by 50 to 75% depending on the model. Depending on prompt-cache hit rate and the output share of total cost, typical multi-turn runs see roughly 20 to 40% total cost reduction. The benefit was already active for OpenAI users; this page documents how to extend the same shape to self-hosted or alternate backends. -## Scope: user simulator only +## Splitting simulator and evaluator LLMs -The `responses` provider works for the user-simulator LLM. The evaluator LLM also reads these keys, but the evaluator passes `text_format` (OpenAI structured output) on every metric call. Self-hosted backends like Ollama do not currently accept `text_format`; vLLM accepts it only with `--guided-decoding-backend` enabled. +The evaluator passes `text_format` (OpenAI structured output) on every metric call. Self-hosted backends like Ollama do not currently accept `text_format`; vLLM accepts it only with `--guided-decoding-backend` enabled. -For evaluation, keep `provider: openai` (or another fully-compatible backend with structured-output support) unless you have explicitly verified your backend handles `text_format`. A safe pattern is to set `provider: responses` and a custom `base_url` for simulation, then either accept the schema risk on the evaluator or run evaluation as a separate step with an OpenAI-compatible LLM. +To run the simulator on a self-hosted backend while keeping the evaluator on OpenAI, set the shared LLM keys for the simulator and override the evaluator-specific keys: + +```yaml +# Simulator runs on Ollama +provider: responses +model: llama3.1 +base_url: http://localhost:11434/v1 +api_key: ollama + +# Evaluator overrides to keep using OpenAI +evaluator_provider: openai +evaluator_model: gpt-4o-mini +``` + +If you omit the `evaluator_*` keys, the evaluator uses the same backend as the simulator. Available override fields: `evaluator_model`, `evaluator_provider`, `evaluator_base_url`, `evaluator_api_key`. ## Configuration diff --git a/examples/customer-service/README.md b/examples/customer-service/README.md index d764db37..8fa5ffc0 100644 --- a/examples/customer-service/README.md +++ b/examples/customer-service/README.md @@ -130,15 +130,24 @@ The SQLite database (`store.db`) is created automatically on first run with samp ### Running the user simulator on a self-hosted backend -The user-simulator LLM defaults to OpenAI. To swap it for any [Open Responses](https://www.openresponses.org/)-conforming backend (Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter), replace the LLM configuration in `config.yaml` (the `model` and `provider` keys near the top) with: +The user-simulator LLM defaults to OpenAI. To run the simulator on any [Open Responses](https://www.openresponses.org/)-conforming backend (Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter) while keeping the evaluator on OpenAI, replace the LLM configuration in `config.yaml`: ```yaml +# Simulator on Ollama provider: responses model: llama3.1 base_url: http://localhost:11434/v1 api_key: ollama + +# Evaluator stays on OpenAI for structured-output support +evaluator_provider: openai +evaluator_model: gpt-4o-mini ``` -Then start the local server. For Ollama: `ollama serve` and `ollama pull llama3.1` (requires Ollama v0.13.3 or newer). +Then start the local server. For Ollama: `ollama serve` and `ollama pull llama3.1` (requires Ollama v0.13.3 or newer). Set `OPENAI_API_KEY` in your environment for the evaluator. Run as usual: + +```bash +arksim simulate-evaluate config.yaml +``` -**Scope note:** these keys also configure the evaluator LLM, which passes structured-output (`text_format`) calls that Ollama does not currently accept. Run simulation against Ollama, then run evaluation separately with `provider: openai` (or set the env var override) until evaluator-side multi-provider support lands. See [User simulator on Open Responses](https://docs.arklex.ai/main/user-simulator-on-open-responses) for vLLM, NIM, and other backends. +See [User simulator on Open Responses](https://docs.arklex.ai/main/user-simulator-on-open-responses) for vLLM, NIM, and other backends. diff --git a/tests/unit/test_evaluation_input.py b/tests/unit/test_evaluation_input.py index a80f0741..9e5e1afa 100644 --- a/tests/unit/test_evaluation_input.py +++ b/tests/unit/test_evaluation_input.py @@ -28,6 +28,47 @@ def test_evaluation_input_defaults_base_url_and_api_key_to_none(self) -> None: assert settings.api_key is None +class TestEvaluationInputEvaluatorOverrides: + """Tests for the evaluator_* LLM override fields.""" + + def test_evaluator_overrides_default_to_none(self) -> None: + settings = EvaluationInput() + assert settings.evaluator_model is None + assert settings.evaluator_provider is None + assert settings.evaluator_base_url is None + assert settings.evaluator_api_key is None + + def test_evaluator_overrides_accept_values(self) -> None: + settings = EvaluationInput( + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + evaluator_model="gpt-4o-mini", + evaluator_provider="openai", + evaluator_base_url="https://api.openai.com/v1", + evaluator_api_key="sk-test", + ) + assert settings.evaluator_model == "gpt-4o-mini" + assert settings.evaluator_provider == "openai" + assert settings.evaluator_base_url == "https://api.openai.com/v1" + assert settings.evaluator_api_key == "sk-test" + + def test_evaluator_overrides_can_be_partial(self) -> None: + """A user can override just provider/base_url and keep shared + model/api_key, or any other partial pattern. + """ + settings = EvaluationInput( + provider="responses", + base_url="http://localhost:11434/v1", + evaluator_provider="openai", + ) + assert settings.provider == "responses" + assert settings.evaluator_provider == "openai" + assert settings.evaluator_base_url is None + assert settings.evaluator_api_key is None + + class TestEvaluationInputPathResolution: """Tests for config-relative path resolution in EvaluationInput.""" diff --git a/tests/unit/test_responses_yaml_plumbing.py b/tests/unit/test_responses_yaml_plumbing.py index bbbd6390..fa5c894b 100644 --- a/tests/unit/test_responses_yaml_plumbing.py +++ b/tests/unit/test_responses_yaml_plumbing.py @@ -86,3 +86,57 @@ def test_simulation_input_defaults_omit_base_url_and_api_key(self) -> None: # the SDK falls back to OPENAI_BASE_URL / OPENAI_API_KEY env vars. mock_sync.assert_called_once_with() mock_async.assert_called_once_with() + + def test_evaluator_overrides_reach_llm_constructor(self) -> None: + """When evaluator_* fields are set, they must override the shared + fields when the evaluator constructs its LLM. The simulator-side + `LLM(...)` call must NOT see the override values. + """ + settings = EvaluationInput( + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + evaluator_model="gpt-4o-mini", + evaluator_provider="openai", + evaluator_base_url="https://api.openai.com/v1", + evaluator_api_key="sk-test", + ) + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI"), + ): + # Simulate the evaluator's LLM construction path + LLM( + model=settings.evaluator_model or settings.model, + provider=settings.evaluator_provider or settings.provider, + base_url=settings.evaluator_base_url or settings.base_url, + api_key=settings.evaluator_api_key or settings.api_key, + ) + mock_sync.assert_called_once_with( + base_url="https://api.openai.com/v1", api_key="sk-test" + ) + + def test_evaluator_falls_back_to_shared_when_override_unset(self) -> None: + """When evaluator_* fields are unset, the evaluator's LLM uses the + shared model/provider/base_url/api_key. + """ + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + base_url="https://api.openai.com/v1", + api_key="sk-test", + ) + with ( + patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, + patch("arksim.llms.chat.providers.openai.AsyncOpenAI"), + ): + LLM( + model=settings.evaluator_model or settings.model, + provider=settings.evaluator_provider or settings.provider, + base_url=settings.evaluator_base_url or settings.base_url, + api_key=settings.evaluator_api_key or settings.api_key, + ) + mock_sync.assert_called_once_with( + base_url="https://api.openai.com/v1", api_key="sk-test" + ) From e3f98dfcd95f141005d7d0c15048b0498e5782e2 Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Tue, 19 May 2026 09:39:05 -0700 Subject: [PATCH 17/23] fix(evaluator): per-group LLM fallback to prevent cross-endpoint credential leak --- CHANGELOG.md | 3 +- arksim/evaluator/entities.py | 76 ++++++++- arksim/evaluator/evaluator.py | 17 +- .../main/user-simulator-on-open-responses.mdx | 22 ++- examples/customer-service/README.md | 5 +- tests/unit/test_evaluation_input.py | 34 ++++ tests/unit/test_responses_yaml_plumbing.py | 145 +++++++++++++----- 7 files changed, 252 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8049a59..6807417b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * **evaluator:** error-to-scenario mappings included in `evaluation.json` output * **evaluator:** shared evaluation constants (`SCORE_NOT_COMPUTED`, `BEHAVIOR_FAILURE_THRESHOLD`) * **a2a:** add tool call capture via A2A AgentExtension (`https://arksim.arklex.ai/a2a/tool-call-capture/v1`); agents declare the extension in the AgentCard and surface tool calls in `Task.artifacts[*].metadata`. See [Tool Call Capture docs](https://docs.arklex.ai/main/tool-call-capture) -* **llm:** user simulator can now run on any Open Responses-conforming backend via `provider: responses` with configurable `base_url` and `api_key`. Works with OpenAI, Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter, LM Studio, and other consortium backers. The OpenAI provider has run on `/v1/responses` since v0.3.0; this change unlocks multi-provider reach. Add `evaluator_model`, `evaluator_provider`, `evaluator_base_url`, and `evaluator_api_key` for split simulator/evaluator backends (e.g. Ollama simulator + OpenAI evaluator in a single `config.yaml`). +* **llm:** user simulator can now run on any Open Responses-conforming backend via `provider: responses` with configurable `base_url` and `api_key`. Works with OpenAI, Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter, LM Studio, and other consortium backers. The OpenAI provider has run on `/v1/responses` since v0.3.0; this change unlocks multi-provider reach. +* **evaluator:** add `evaluator_model`, `evaluator_provider`, `evaluator_base_url`, and `evaluator_api_key` to split simulator and evaluator backends in a single `config.yaml`. When the evaluator endpoint differs from the simulator endpoint, shared `base_url`/`api_key` are not forwarded to prevent credential leaks; specify evaluator-side credentials explicitly or rely on SDK env-var fallback. * **report:** scenario IDs displayed in HTML report error cards * **ui:** version display in web UI sidebar (next to "Arksim" title) * **api:** path traversal protection on filesystem and results API endpoints diff --git a/arksim/evaluator/entities.py b/arksim/evaluator/entities.py index 08fad82e..1dc41ec7 100644 --- a/arksim/evaluator/entities.py +++ b/arksim/evaluator/entities.py @@ -85,9 +85,12 @@ class EvaluationInput(BaseModel): default=None, description=( "Optional override for the evaluator's LLM provider. Useful when " - "the simulator runs on a self-hosted backend (e.g. Ollama) that " - "does not support structured output, while the evaluator needs " - "OpenAI-compatible `text_format` support." + "the simulator runs on a backend that does not support structured " + "output, while the evaluator needs OpenAI-compatible `text_format` " + "support. When set with a value that differs from `provider`, the " + "shared `base_url` and `api_key` are NOT forwarded; specify " + "`evaluator_base_url` / `evaluator_api_key` explicitly or rely on " + "SDK env-var fallback." ), ) evaluator_base_url: str | None = Field( @@ -198,6 +201,73 @@ def validate_evaluation_input(self, info: ValidationInfo) -> Self: return self + def _evaluator_endpoint_differs(self) -> bool: + """Return True when the evaluator's endpoint differs from the + simulator's. Whitespace-only override values are treated as unset to + match the whitespace-stripping behavior of the OpenAI provider. + """ + provider = ( + self.evaluator_provider.strip() + if self.evaluator_provider is not None + else None + ) + base_url = ( + self.evaluator_base_url.strip() + if self.evaluator_base_url is not None + else None + ) + return bool(provider) or bool(base_url) + + def evaluator_llm_kwargs(self) -> dict[str, str | None]: + """Resolve LLM kwargs for the evaluator, with safe fallback semantics. + + - `evaluator_model` individually overrides `model` when set; otherwise + the evaluator uses the shared `model`. + - When the evaluator endpoint differs from the simulator endpoint + (`evaluator_provider` set OR `evaluator_base_url` set), the + credentials and base_url DO NOT cross-inherit from the shared + keys. They must be set explicitly via `evaluator_base_url` / + `evaluator_api_key`, or fall through to the SDK's env-var + defaults (`OPENAI_BASE_URL` / `OPENAI_API_KEY`). + - This prevents accidentally sending a credential intended for one + endpoint to another (e.g. forwarding a live OpenAI key to a + self-hosted Ollama endpoint). + - Whitespace-only override values are treated as unset. + """ + if self._evaluator_endpoint_differs(): + return { + "model": self.evaluator_model or self.model, + "provider": self.evaluator_provider or self.provider, + "base_url": self.evaluator_base_url, + "api_key": self.evaluator_api_key, + } + + return { + "model": self.evaluator_model or self.model, + "provider": self.provider, + "base_url": self.base_url, + "api_key": self.api_key, + } + + @model_validator(mode="after") + def _log_evaluator_endpoint_split(self) -> Self: + """Emit an info log when the evaluator endpoint differs from the + simulator endpoint but no evaluator_api_key is set. This warns the + user that the shared api_key will NOT be forwarded; the evaluator + relies on SDK env-var defaults instead. + """ + if self._evaluator_endpoint_differs() and self.evaluator_api_key is None: + _entities_logger.info( + "Evaluator endpoint differs from simulator " + "(evaluator_provider=%s, evaluator_base_url=%s). " + "evaluator_api_key is unset; falling back to SDK env-var " + "defaults (e.g. OPENAI_API_KEY). The shared `api_key` will " + "NOT be forwarded to the evaluator endpoint.", + self.evaluator_provider, + self.evaluator_base_url, + ) + return self + class EvaluationParams(BaseModel): """Parameters for the evaluation engine.""" diff --git a/arksim/evaluator/evaluator.py b/arksim/evaluator/evaluator.py index cdf3c546..d0014b79 100644 --- a/arksim/evaluator/evaluator.py +++ b/arksim/evaluator/evaluator.py @@ -938,12 +938,13 @@ def run_evaluation( "scenario context will be unavailable" ) - llm = LLM( - model=settings.evaluator_model or settings.model, - provider=settings.evaluator_provider or settings.provider, - base_url=settings.evaluator_base_url or settings.base_url, - api_key=settings.evaluator_api_key or settings.api_key, - ) + # See EvaluationInput.evaluator_llm_kwargs() for the per-group fallback + # rule. When evaluator endpoint differs from simulator, base_url and + # api_key do not cross-inherit from shared keys; they must be set + # explicitly or fall back to SDK env vars. This prevents credential + # leaks across endpoints. + evaluator_kwargs = settings.evaluator_llm_kwargs() + llm = LLM(**evaluator_kwargs) all_quant, all_qual = _load_custom_metrics( settings.custom_metrics_file_paths, llm=llm ) @@ -988,8 +989,8 @@ def run_evaluation( metric_descriptions=metric_descriptions, metric_ranges=metric_ranges, qual_label_colors=qual_label_colors, - evaluation_model=settings.model, - evaluation_provider=settings.provider, + evaluation_model=evaluator_kwargs["model"] or settings.model, + evaluation_provider=evaluator_kwargs["provider"] or settings.provider, ) generate_html_report(report_params) diff --git a/docs/main/user-simulator-on-open-responses.mdx b/docs/main/user-simulator-on-open-responses.mdx index 85a14559..0549f6c1 100644 --- a/docs/main/user-simulator-on-open-responses.mdx +++ b/docs/main/user-simulator-on-open-responses.mdx @@ -15,6 +15,8 @@ These keys go at the top level of your `config.yaml`, alongside other simulation ArkSim's user simulator sends a stable system prompt and scenario context to the LLM on every turn of every scenario. OpenAI's automatic prompt cache hits this stable prefix and discounts the cached portion by 50 to 75% depending on the model. Depending on prompt-cache hit rate and the output share of total cost, typical multi-turn runs see roughly 20 to 40% total cost reduction. The benefit was already active for OpenAI users; this page documents how to extend the same shape to self-hosted or alternate backends. +ArkSim uses Responses in stateless mode: it replays the full conversation history each turn rather than threading via `previous_response_id`. Stateful threading is a planned follow-up that would further improve cache utilization. + ## Splitting simulator and evaluator LLMs The evaluator passes `text_format` (OpenAI structured output) on every metric call. Self-hosted backends like Ollama do not currently accept `text_format`; vLLM accepts it only with `--guided-decoding-backend` enabled. @@ -33,6 +35,24 @@ evaluator_provider: openai evaluator_model: gpt-4o-mini ``` +Because the evaluator endpoint differs from the simulator endpoint (`evaluator_provider` is set), arksim does not forward the shared `api_key: ollama` to the evaluator. The evaluator's OpenAI SDK falls back to `OPENAI_API_KEY` in your environment. Set `OPENAI_API_KEY` before running, or specify `evaluator_api_key: ` explicitly. + +For full explicit control without relying on env vars: + +```yaml +# Simulator on Ollama +provider: responses +model: llama3.1 +base_url: http://localhost:11434/v1 +api_key: ollama + +# Evaluator fully specified +evaluator_provider: openai +evaluator_model: gpt-4o-mini +evaluator_base_url: https://api.openai.com/v1 +evaluator_api_key: +``` + If you omit the `evaluator_*` keys, the evaluator uses the same backend as the simulator. Available override fields: `evaluator_model`, `evaluator_provider`, `evaluator_base_url`, `evaluator_api_key`. ## Configuration @@ -41,7 +61,7 @@ If you omit the `evaluator_*` keys, the evaluator uses the same backend as the s ```yaml provider: openai -model: gpt-4.1-mini +model: gpt-4o-mini ``` `api.openai.com` is the default. No `base_url` needed. diff --git a/examples/customer-service/README.md b/examples/customer-service/README.md index 8fa5ffc0..9cdedbb5 100644 --- a/examples/customer-service/README.md +++ b/examples/customer-service/README.md @@ -144,10 +144,13 @@ evaluator_provider: openai evaluator_model: gpt-4o-mini ``` -Then start the local server. For Ollama: `ollama serve` and `ollama pull llama3.1` (requires Ollama v0.13.3 or newer). Set `OPENAI_API_KEY` in your environment for the evaluator. Run as usual: +The evaluator endpoint differs from the simulator endpoint, so arksim does not forward `api_key: ollama` to OpenAI. The evaluator falls back to `OPENAI_API_KEY` in your environment. Set it before running: ```bash +export OPENAI_API_KEY=sk-... arksim simulate-evaluate config.yaml ``` +Then start the local server. For Ollama: `ollama serve` and `ollama pull llama3.1` (requires Ollama v0.13.3 or newer). + See [User simulator on Open Responses](https://docs.arklex.ai/main/user-simulator-on-open-responses) for vLLM, NIM, and other backends. diff --git a/tests/unit/test_evaluation_input.py b/tests/unit/test_evaluation_input.py index 9e5e1afa..981405e6 100644 --- a/tests/unit/test_evaluation_input.py +++ b/tests/unit/test_evaluation_input.py @@ -68,6 +68,40 @@ def test_evaluator_overrides_can_be_partial(self) -> None: assert settings.evaluator_base_url is None assert settings.evaluator_api_key is None + def test_evaluator_kwargs_treats_whitespace_evaluator_provider_as_unset( + self, + ) -> None: + """Whitespace-only evaluator_provider should not trigger endpoint- + split semantics. The helper strips whitespace before deciding + whether the endpoint differs; this matches the existing whitespace- + stripping behavior in the OpenAI provider. The shared api_key + therefore flows through to the evaluator. + """ + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + api_key="sk-shared", + evaluator_provider=" ", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["api_key"] == "sk-shared" + assert kwargs["provider"] == "openai" + + def test_evaluator_kwargs_treats_whitespace_evaluator_base_url_as_unset( + self, + ) -> None: + """Whitespace-only evaluator_base_url should not trigger endpoint- + split semantics; the shared api_key still flows through. + """ + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + api_key="sk-shared", + evaluator_base_url=" ", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["api_key"] == "sk-shared" + class TestEvaluationInputPathResolution: """Tests for config-relative path resolution in EvaluationInput.""" diff --git a/tests/unit/test_responses_yaml_plumbing.py b/tests/unit/test_responses_yaml_plumbing.py index fa5c894b..edbb644f 100644 --- a/tests/unit/test_responses_yaml_plumbing.py +++ b/tests/unit/test_responses_yaml_plumbing.py @@ -87,56 +87,129 @@ def test_simulation_input_defaults_omit_base_url_and_api_key(self) -> None: mock_sync.assert_called_once_with() mock_async.assert_called_once_with() - def test_evaluator_overrides_reach_llm_constructor(self) -> None: - """When evaluator_* fields are set, they must override the shared - fields when the evaluator constructs its LLM. The simulator-side - `LLM(...)` call must NOT see the override values. + def test_evaluator_kwargs_inherit_when_no_overrides(self) -> None: + """When no evaluator_* fields are set, evaluator inherits all + simulator-side LLM keys. + """ + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + base_url=None, + api_key="sk-shared", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs == { + "model": "gpt-4o-mini", + "provider": "openai", + "base_url": None, + "api_key": "sk-shared", + } + + def test_evaluator_kwargs_model_only_override_keeps_shared_endpoint( + self, + ) -> None: + """evaluator_model alone overrides only the model; other shared + keys still apply because the endpoint did not change. + """ + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + api_key="sk-shared", + evaluator_model="gpt-4o", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["model"] == "gpt-4o" + assert kwargs["provider"] == "openai" + assert kwargs["api_key"] == "sk-shared" + + def test_evaluator_kwargs_provider_split_does_not_inherit_credentials( + self, + ) -> None: + """When evaluator_provider differs from provider, the shared + api_key and base_url MUST NOT be forwarded. This is the bug fix + for the cross-endpoint credential leak. """ settings = EvaluationInput( model="llama3.1", provider="responses", base_url="http://localhost:11434/v1", api_key="ollama", - evaluator_model="gpt-4o-mini", evaluator_provider="openai", - evaluator_base_url="https://api.openai.com/v1", - evaluator_api_key="sk-test", + evaluator_model="gpt-4o-mini", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["model"] == "gpt-4o-mini" + assert kwargs["provider"] == "openai" + assert kwargs["base_url"] is None, ( + "Shared base_url must not leak into evaluator when provider differs" + ) + assert kwargs["api_key"] is None, ( + "Shared api_key must not leak into evaluator when provider differs" ) - with ( - patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, - patch("arksim.llms.chat.providers.openai.AsyncOpenAI"), - ): - # Simulate the evaluator's LLM construction path - LLM( - model=settings.evaluator_model or settings.model, - provider=settings.evaluator_provider or settings.provider, - base_url=settings.evaluator_base_url or settings.base_url, - api_key=settings.evaluator_api_key or settings.api_key, - ) - mock_sync.assert_called_once_with( - base_url="https://api.openai.com/v1", api_key="sk-test" - ) - def test_evaluator_falls_back_to_shared_when_override_unset(self) -> None: - """When evaluator_* fields are unset, the evaluator's LLM uses the - shared model/provider/base_url/api_key. + def test_evaluator_kwargs_base_url_split_does_not_inherit_credentials( + self, + ) -> None: + """When evaluator_base_url differs from base_url (endpoint split), + api_key must NOT inherit from shared. Catches the credential-leak + scenario where a live OpenAI key would otherwise be sent to a + self-hosted endpoint. """ settings = EvaluationInput( model="gpt-4o-mini", provider="openai", - base_url="https://api.openai.com/v1", - api_key="sk-test", + api_key="sk-LIVE-OPENAI-KEY", + evaluator_base_url="http://localhost:11434/v1", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["api_key"] is None, ( + "Live api_key must not be forwarded to a different base_url" + ) + assert kwargs["base_url"] == "http://localhost:11434/v1" + + def test_evaluator_kwargs_explicit_overrides_used_verbatim(self) -> None: + """When evaluator_* fields are fully specified, they pass through + verbatim and shared keys are ignored. + """ + settings = EvaluationInput( + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + evaluator_model="gpt-4o-mini", + evaluator_provider="openai", + evaluator_base_url="https://api.openai.com/v1", + evaluator_api_key="sk-eval-explicit", ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs == { + "model": "gpt-4o-mini", + "provider": "openai", + "base_url": "https://api.openai.com/v1", + "api_key": "sk-eval-explicit", + } + + def test_evaluator_split_reaches_openai_sdk_without_shared_credentials( + self, + ) -> None: + """End-to-end: simulator on Ollama, evaluator on OpenAI. The OpenAI + SDK constructor must not see Ollama's base_url or api_key. + """ + settings = EvaluationInput( + model="llama3.1", + provider="responses", + base_url="http://localhost:11434/v1", + api_key="ollama", + evaluator_provider="openai", + evaluator_model="gpt-4o-mini", + ) + with ( patch("arksim.llms.chat.providers.openai.OpenAI") as mock_sync, - patch("arksim.llms.chat.providers.openai.AsyncOpenAI"), + patch("arksim.llms.chat.providers.openai.AsyncOpenAI") as mock_async, ): - LLM( - model=settings.evaluator_model or settings.model, - provider=settings.evaluator_provider or settings.provider, - base_url=settings.evaluator_base_url or settings.base_url, - api_key=settings.evaluator_api_key or settings.api_key, - ) - mock_sync.assert_called_once_with( - base_url="https://api.openai.com/v1", api_key="sk-test" - ) + LLM(**settings.evaluator_llm_kwargs()) + # Critical: NO base_url, NO api_key kwargs. SDK must fall back + # to OPENAI_BASE_URL / OPENAI_API_KEY env vars. + mock_sync.assert_called_once_with() + mock_async.assert_called_once_with() From cbee36f38ea30134a1221f8458237e7ea7f5c747 Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Tue, 19 May 2026 10:41:39 -0700 Subject: [PATCH 18/23] fix(evaluator): tighten endpoint-differs check and surface discarded overrides - _evaluator_endpoint_differs now compares evaluator_provider/evaluator_base_url against shared peers, so a redundant override (same value) no longer drops the shared api_key and crashes the evaluator at run time. - Add a WARNING when evaluator_api_key is set but the endpoint does not differ, so users notice when the override is silently ignored. - Strip whitespace in the existing endpoint-split INFO log so empty / whitespace-only values surface as None instead of literal blanks. - Drop dead `or settings.model` / `or settings.provider` fallbacks in the HTML report kwargs (evaluator_llm_kwargs already guarantees both keys). - Reorder examples/customer-service/README.md so the Ollama prereqs (`ollama serve`, model pull, OPENAI_API_KEY) appear before `arksim simulate-evaluate`, preventing a top-to-bottom reader from hitting a connection error. Adds 3 tests covering the new endpoint-differs branches and the discarded-override warning. --- arksim/evaluator/entities.py | 79 +++++++++++++++++++++-------- arksim/evaluator/evaluator.py | 4 +- examples/customer-service/README.md | 13 +++-- tests/unit/test_evaluation_input.py | 58 +++++++++++++++++++++ 4 files changed, 129 insertions(+), 25 deletions(-) diff --git a/arksim/evaluator/entities.py b/arksim/evaluator/entities.py index 1dc41ec7..448be9d3 100644 --- a/arksim/evaluator/entities.py +++ b/arksim/evaluator/entities.py @@ -29,6 +29,22 @@ _entities_logger = logging.getLogger(__name__) + +def _norm_endpoint_value(value: str | None) -> str | None: + """Normalize an endpoint field for comparison. + + Returns the stripped value when non-empty after stripping; otherwise + ``None``. This treats ``None``, ``""``, and ``" "`` as equivalent + ("unset"), matching the whitespace-stripping behavior of the OpenAI + provider so users do not silently trigger endpoint-split semantics by + typing trailing whitespace. + """ + if value is None: + return None + stripped = value.strip() + return stripped if stripped else None + + _DEFAULT_METRICS_TO_RUN = [ "faithfulness", "helpfulness", @@ -202,21 +218,25 @@ def validate_evaluation_input(self, info: ValidationInfo) -> Self: return self def _evaluator_endpoint_differs(self) -> bool: - """Return True when the evaluator's endpoint differs from the - simulator's. Whitespace-only override values are treated as unset to - match the whitespace-stripping behavior of the OpenAI provider. + """Return True when the evaluator's endpoint *actually differs* from + the simulator's. Whitespace-only override values are treated as unset + to match the whitespace-stripping behavior of the OpenAI provider. + + A redundant override (e.g. ``evaluator_provider`` equal to the + shared ``provider``) is NOT a real endpoint split, so the shared + ``api_key`` / ``base_url`` continue to apply. This avoids the + footgun where typing the same provider twice silently drops the + shared credentials and crashes the evaluator at run time. """ - provider = ( - self.evaluator_provider.strip() - if self.evaluator_provider is not None - else None + eval_provider = _norm_endpoint_value(self.evaluator_provider) + eval_base_url = _norm_endpoint_value(self.evaluator_base_url) + if eval_provider is not None and eval_provider != _norm_endpoint_value( + self.provider + ): + return True + return eval_base_url is not None and eval_base_url != _norm_endpoint_value( + self.base_url ) - base_url = ( - self.evaluator_base_url.strip() - if self.evaluator_base_url is not None - else None - ) - return bool(provider) or bool(base_url) def evaluator_llm_kwargs(self) -> dict[str, str | None]: """Resolve LLM kwargs for the evaluator, with safe fallback semantics. @@ -251,20 +271,39 @@ def evaluator_llm_kwargs(self) -> dict[str, str | None]: @model_validator(mode="after") def _log_evaluator_endpoint_split(self) -> Self: - """Emit an info log when the evaluator endpoint differs from the - simulator endpoint but no evaluator_api_key is set. This warns the - user that the shared api_key will NOT be forwarded; the evaluator - relies on SDK env-var defaults instead. + """Emit diagnostics for the two endpoint-split footguns: + + 1. INFO: evaluator endpoint differs from simulator but no + ``evaluator_api_key`` is set. Warns the user that the shared + ``api_key`` will NOT be forwarded; the evaluator falls back to + SDK env-var defaults instead. + 2. WARNING: ``evaluator_api_key`` is set but the evaluator endpoint + does NOT differ from the simulator endpoint. In that case the + shared ``api_key`` is used and ``evaluator_api_key`` is silently + ignored. Warn so the user does not assume the override took effect. """ - if self._evaluator_endpoint_differs() and self.evaluator_api_key is None: + endpoint_differs = self._evaluator_endpoint_differs() + if endpoint_differs and self.evaluator_api_key is None: _entities_logger.info( "Evaluator endpoint differs from simulator " "(evaluator_provider=%s, evaluator_base_url=%s). " "evaluator_api_key is unset; falling back to SDK env-var " "defaults (e.g. OPENAI_API_KEY). The shared `api_key` will " "NOT be forwarded to the evaluator endpoint.", - self.evaluator_provider, - self.evaluator_base_url, + _norm_endpoint_value(self.evaluator_provider), + _norm_endpoint_value(self.evaluator_base_url), + ) + if ( + not endpoint_differs + and self.evaluator_api_key is not None + and self.evaluator_api_key.strip() + ): + _entities_logger.warning( + "evaluator_api_key is set but the evaluator endpoint matches " + "the simulator endpoint, so the shared api_key is used and " + "evaluator_api_key is ignored. To use a different key, also " + "set evaluator_provider or evaluator_base_url to point at a " + "different endpoint." ) return self diff --git a/arksim/evaluator/evaluator.py b/arksim/evaluator/evaluator.py index d0014b79..3ba7e479 100644 --- a/arksim/evaluator/evaluator.py +++ b/arksim/evaluator/evaluator.py @@ -989,8 +989,8 @@ def run_evaluation( metric_descriptions=metric_descriptions, metric_ranges=metric_ranges, qual_label_colors=qual_label_colors, - evaluation_model=evaluator_kwargs["model"] or settings.model, - evaluation_provider=evaluator_kwargs["provider"] or settings.provider, + evaluation_model=evaluator_kwargs["model"], + evaluation_provider=evaluator_kwargs["provider"], ) generate_html_report(report_params) diff --git a/examples/customer-service/README.md b/examples/customer-service/README.md index 9cdedbb5..81477f28 100644 --- a/examples/customer-service/README.md +++ b/examples/customer-service/README.md @@ -144,13 +144,20 @@ evaluator_provider: openai evaluator_model: gpt-4o-mini ``` -The evaluator endpoint differs from the simulator endpoint, so arksim does not forward `api_key: ollama` to OpenAI. The evaluator falls back to `OPENAI_API_KEY` in your environment. Set it before running: +The evaluator endpoint differs from the simulator endpoint, so arksim does not forward `api_key: ollama` to OpenAI. The evaluator falls back to `OPENAI_API_KEY` in your environment. + +Before running, start the local server and set the evaluator key. For Ollama: ```bash +ollama serve +ollama pull llama3.1 # requires Ollama v0.13.3 or newer export OPENAI_API_KEY=sk-... -arksim simulate-evaluate config.yaml ``` -Then start the local server. For Ollama: `ollama serve` and `ollama pull llama3.1` (requires Ollama v0.13.3 or newer). +Then run: + +```bash +arksim simulate-evaluate config.yaml +``` See [User simulator on Open Responses](https://docs.arklex.ai/main/user-simulator-on-open-responses) for vLLM, NIM, and other backends. diff --git a/tests/unit/test_evaluation_input.py b/tests/unit/test_evaluation_input.py index 981405e6..69d10809 100644 --- a/tests/unit/test_evaluation_input.py +++ b/tests/unit/test_evaluation_input.py @@ -3,9 +3,12 @@ from __future__ import annotations +import logging from pathlib import Path from typing import Any +import pytest + from arksim.evaluator.entities import EvaluationInput @@ -102,6 +105,61 @@ def test_evaluator_kwargs_treats_whitespace_evaluator_base_url_as_unset( kwargs = settings.evaluator_llm_kwargs() assert kwargs["api_key"] == "sk-shared" + def test_endpoint_does_not_differ_when_evaluator_provider_matches_shared( + self, + ) -> None: + """Setting evaluator_provider to the same value as provider must not + trigger endpoint-split (the shared api_key must NOT be dropped). + A redundant override is not a real endpoint change. + """ + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + api_key="sk-shared", + evaluator_provider="openai", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["api_key"] == "sk-shared", ( + "Redundant evaluator_provider must not strip shared credentials" + ) + assert kwargs["provider"] == "openai" + + def test_endpoint_does_not_differ_when_evaluator_base_url_matches_shared( + self, + ) -> None: + """Same as above for base_url: a redundant override that matches the + shared value must not trigger endpoint-split semantics. + """ + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + base_url="https://api.openai.com/v1", + api_key="sk-shared", + evaluator_base_url="https://api.openai.com/v1", + ) + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["api_key"] == "sk-shared" + assert kwargs["base_url"] == "https://api.openai.com/v1" + + def test_warns_when_evaluator_api_key_set_alone( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """Setting evaluator_api_key alone (no endpoint split) silently + discards the override. Warn the user so they do not assume the + override took effect. + """ + with caplog.at_level(logging.WARNING, logger="arksim.evaluator.entities"): + EvaluationInput( + model="gpt-4o-mini", + provider="openai", + api_key="sk-shared", + evaluator_api_key="sk-eval-IGNORED", + ) + assert any( + "evaluator_api_key is set" in rec.message and "ignored" in rec.message + for rec in caplog.records + ) + class TestEvaluationInputPathResolution: """Tests for config-relative path resolution in EvaluationInput.""" From b83c36250ee96daa63411f0cb8aa1dab57ada2cb Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Tue, 19 May 2026 11:05:41 -0700 Subject: [PATCH 19/23] fix(evaluator): broader whitespace normalization and field-tuple guard rail --- arksim/evaluator/entities.py | 45 +++++++++++++++---- .../main/user-simulator-on-open-responses.mdx | 5 ++- tests/unit/test_evaluation_input.py | 28 ++++++++++++ 3 files changed, 67 insertions(+), 11 deletions(-) diff --git a/arksim/evaluator/entities.py b/arksim/evaluator/entities.py index 448be9d3..9ce64649 100644 --- a/arksim/evaluator/entities.py +++ b/arksim/evaluator/entities.py @@ -30,21 +30,40 @@ _entities_logger = logging.getLogger(__name__) +# Invisible Unicode characters that copy-paste from web UIs (Notion, Slack, +# Google Docs) can silently introduce: zero-width space (U+200B), zero-width +# non-joiner (U+200C), zero-width joiner (U+200D), word joiner (U+2060), and +# BOM / zero-width no-break space (U+FEFF). Python's ``str.strip()`` already +# handles regular ASCII whitespace and ``U+00A0`` non-breaking space, but it +# does not touch these zero-width characters. +_INVISIBLE_CHARS = "​‌‍⁠" + + def _norm_endpoint_value(value: str | None) -> str | None: """Normalize an endpoint field for comparison. Returns the stripped value when non-empty after stripping; otherwise - ``None``. This treats ``None``, ``""``, and ``" "`` as equivalent - ("unset"), matching the whitespace-stripping behavior of the OpenAI - provider so users do not silently trigger endpoint-split semantics by - typing trailing whitespace. + ``None``. This treats ``None``, ``""``, ``" "``, and copy-paste + artifacts like a leading zero-width space as equivalent ("unset"), + matching the whitespace-stripping behavior of the OpenAI provider so + users do not silently trigger endpoint-split semantics by typing + trailing whitespace or pasting from a rich-text editor. """ if value is None: return None - stripped = value.strip() + stripped = value.strip().strip(_INVISIBLE_CHARS).strip() return stripped if stripped else None +# Canonical LLM kwarg names returned by ``EvaluationInput.evaluator_llm_kwargs``. +# Adding a new field here requires also adding a matching ``evaluator_`` +# attribute to ``EvaluationInput``; the same-endpoint branch of the helper +# iterates this tuple so it picks the new field up automatically. The +# endpoint-differs branch is intentionally explicit because ``base_url`` and +# ``api_key`` have special "do not cross-inherit" credential semantics. +_LLM_KWARG_FIELDS: tuple[str, ...] = ("model", "provider", "base_url", "api_key") + + _DEFAULT_METRICS_TO_RUN = [ "faithfulness", "helpfulness", @@ -255,6 +274,10 @@ def evaluator_llm_kwargs(self) -> dict[str, str | None]: - Whitespace-only override values are treated as unset. """ if self._evaluator_endpoint_differs(): + # Endpoint split: ``model`` and ``provider`` can fall back to the + # shared values (neither is credential-bearing), but ``base_url`` + # and ``api_key`` must NOT cross-inherit. Leave them as-is so the + # SDK env-var fallback (e.g. ``OPENAI_API_KEY``) applies. return { "model": self.evaluator_model or self.model, "provider": self.evaluator_provider or self.provider, @@ -262,11 +285,15 @@ def evaluator_llm_kwargs(self) -> dict[str, str | None]: "api_key": self.evaluator_api_key, } + # Same endpoint: every field inherits from the shared key when the + # matching ``evaluator_`` is unset (None, empty, or + # whitespace-only / invisible-only after normalization). Iterating + # ``_LLM_KWARG_FIELDS`` keeps this branch in sync with future kwarg + # additions automatically. return { - "model": self.evaluator_model or self.model, - "provider": self.provider, - "base_url": self.base_url, - "api_key": self.api_key, + field: _norm_endpoint_value(getattr(self, f"evaluator_{field}")) + or getattr(self, field) + for field in _LLM_KWARG_FIELDS } @model_validator(mode="after") diff --git a/docs/main/user-simulator-on-open-responses.mdx b/docs/main/user-simulator-on-open-responses.mdx index 0549f6c1..71d11c9e 100644 --- a/docs/main/user-simulator-on-open-responses.mdx +++ b/docs/main/user-simulator-on-open-responses.mdx @@ -15,8 +15,6 @@ These keys go at the top level of your `config.yaml`, alongside other simulation ArkSim's user simulator sends a stable system prompt and scenario context to the LLM on every turn of every scenario. OpenAI's automatic prompt cache hits this stable prefix and discounts the cached portion by 50 to 75% depending on the model. Depending on prompt-cache hit rate and the output share of total cost, typical multi-turn runs see roughly 20 to 40% total cost reduction. The benefit was already active for OpenAI users; this page documents how to extend the same shape to self-hosted or alternate backends. -ArkSim uses Responses in stateless mode: it replays the full conversation history each turn rather than threading via `previous_response_id`. Stateful threading is a planned follow-up that would further improve cache utilization. - ## Splitting simulator and evaluator LLMs The evaluator passes `text_format` (OpenAI structured output) on every metric call. Self-hosted backends like Ollama do not currently accept `text_format`; vLLM accepts it only with `--guided-decoding-backend` enabled. @@ -37,6 +35,8 @@ evaluator_model: gpt-4o-mini Because the evaluator endpoint differs from the simulator endpoint (`evaluator_provider` is set), arksim does not forward the shared `api_key: ollama` to the evaluator. The evaluator's OpenAI SDK falls back to `OPENAI_API_KEY` in your environment. Set `OPENAI_API_KEY` before running, or specify `evaluator_api_key: ` explicitly. +Make sure Ollama is running (`ollama serve`) and the model is pulled (`ollama pull llama3.1`). Requires Ollama v0.13.3 or newer for `/v1/responses` support; see the Ollama section below for details. + For full explicit control without relying on env vars: ```yaml @@ -116,6 +116,7 @@ Self-hosted backends do not have OpenAI-equivalent automatic prefix caching out - The OpenAI prompt cache requires prompts of at least 1024 tokens to activate. Toy scenarios with tiny user profiles and no knowledge base will not hit it. - Cache TTL is roughly 5 to 10 minutes. ArkSim runs fast enough that this is rarely a constraint, but a paused or debugged run can lose its cached prefix. - ArkSim does not stream Responses API output today; the user simulator uses non-streaming `responses.parse()`. +- ArkSim uses Responses in stateless mode: it replays the full conversation history each turn rather than threading via `previous_response_id`. Stateful threading is a planned follow-up that would further improve cache utilization. ## Limitations diff --git a/tests/unit/test_evaluation_input.py b/tests/unit/test_evaluation_input.py index 69d10809..d0340c00 100644 --- a/tests/unit/test_evaluation_input.py +++ b/tests/unit/test_evaluation_input.py @@ -105,6 +105,34 @@ def test_evaluator_kwargs_treats_whitespace_evaluator_base_url_as_unset( kwargs = settings.evaluator_llm_kwargs() assert kwargs["api_key"] == "sk-shared" + def test_endpoint_norm_strips_zero_width_characters(self) -> None: + """Zero-width characters from copy-paste (e.g. Notion, Slack) must + not bypass the endpoint-differs check or otherwise reach the SDK. + """ + # Zero-width space prepended to evaluator_provider + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + api_key="sk-shared", + evaluator_provider="​openai", # ZWSP + "openai" + ) + # After normalization, evaluator_provider == "openai" == provider, + # so endpoint does NOT differ; shared api_key flows through. + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["api_key"] == "sk-shared" + assert kwargs["provider"] == "openai" + + def test_endpoint_norm_strips_bom_and_word_joiner(self) -> None: + """BOM (U+FEFF) and word joiner (U+2060) also handled.""" + settings = EvaluationInput( + model="gpt-4o-mini", + provider="openai", + evaluator_provider="⁠", # all-invisible string + ) + # Treated as unset after normalization; falls through to shared. + kwargs = settings.evaluator_llm_kwargs() + assert kwargs["provider"] == "openai" + def test_endpoint_does_not_differ_when_evaluator_provider_matches_shared( self, ) -> None: From d808ff544fea908a4422a6b0dd0e7a3cf7e29f4b Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Tue, 19 May 2026 11:11:46 -0700 Subject: [PATCH 20/23] refactor(evaluator): drop out-of-scope unicode normalization The _INVISIBLE_CHARS handling was added in 466d7af as part of the zero-nits sweep, defending against zero-width characters from rich-text copy-paste into config.yaml. Probability of organic occurrence is near-zero per the original adversary review; the feature is generic string handling unrelated to the Open Responses scope of this PR. Keep _norm_endpoint_value with plain str.strip() only. --- arksim/evaluator/entities.py | 20 +++++--------------- tests/unit/test_evaluation_input.py | 28 ---------------------------- 2 files changed, 5 insertions(+), 43 deletions(-) diff --git a/arksim/evaluator/entities.py b/arksim/evaluator/entities.py index 9ce64649..cf5e0550 100644 --- a/arksim/evaluator/entities.py +++ b/arksim/evaluator/entities.py @@ -30,28 +30,18 @@ _entities_logger = logging.getLogger(__name__) -# Invisible Unicode characters that copy-paste from web UIs (Notion, Slack, -# Google Docs) can silently introduce: zero-width space (U+200B), zero-width -# non-joiner (U+200C), zero-width joiner (U+200D), word joiner (U+2060), and -# BOM / zero-width no-break space (U+FEFF). Python's ``str.strip()`` already -# handles regular ASCII whitespace and ``U+00A0`` non-breaking space, but it -# does not touch these zero-width characters. -_INVISIBLE_CHARS = "​‌‍⁠" - - def _norm_endpoint_value(value: str | None) -> str | None: """Normalize an endpoint field for comparison. Returns the stripped value when non-empty after stripping; otherwise - ``None``. This treats ``None``, ``""``, ``" "``, and copy-paste - artifacts like a leading zero-width space as equivalent ("unset"), - matching the whitespace-stripping behavior of the OpenAI provider so - users do not silently trigger endpoint-split semantics by typing - trailing whitespace or pasting from a rich-text editor. + ``None``. This treats ``None``, ``""``, and whitespace-only as + equivalent ("unset"), matching the whitespace-stripping behavior of + the OpenAI provider so users do not silently trigger endpoint-split + semantics by typing trailing whitespace. """ if value is None: return None - stripped = value.strip().strip(_INVISIBLE_CHARS).strip() + stripped = value.strip() return stripped if stripped else None diff --git a/tests/unit/test_evaluation_input.py b/tests/unit/test_evaluation_input.py index d0340c00..69d10809 100644 --- a/tests/unit/test_evaluation_input.py +++ b/tests/unit/test_evaluation_input.py @@ -105,34 +105,6 @@ def test_evaluator_kwargs_treats_whitespace_evaluator_base_url_as_unset( kwargs = settings.evaluator_llm_kwargs() assert kwargs["api_key"] == "sk-shared" - def test_endpoint_norm_strips_zero_width_characters(self) -> None: - """Zero-width characters from copy-paste (e.g. Notion, Slack) must - not bypass the endpoint-differs check or otherwise reach the SDK. - """ - # Zero-width space prepended to evaluator_provider - settings = EvaluationInput( - model="gpt-4o-mini", - provider="openai", - api_key="sk-shared", - evaluator_provider="​openai", # ZWSP + "openai" - ) - # After normalization, evaluator_provider == "openai" == provider, - # so endpoint does NOT differ; shared api_key flows through. - kwargs = settings.evaluator_llm_kwargs() - assert kwargs["api_key"] == "sk-shared" - assert kwargs["provider"] == "openai" - - def test_endpoint_norm_strips_bom_and_word_joiner(self) -> None: - """BOM (U+FEFF) and word joiner (U+2060) also handled.""" - settings = EvaluationInput( - model="gpt-4o-mini", - provider="openai", - evaluator_provider="⁠", # all-invisible string - ) - # Treated as unset after normalization; falls through to shared. - kwargs = settings.evaluator_llm_kwargs() - assert kwargs["provider"] == "openai" - def test_endpoint_does_not_differ_when_evaluator_provider_matches_shared( self, ) -> None: From a3042001c09ee62efd18a4ec0c86c151eb1605fa Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Tue, 19 May 2026 15:21:24 -0700 Subject: [PATCH 21/23] feat(llm): surface cache-hit stats and verify the cache benefit --- CHANGELOG.md | 1 + arksim/evaluator/evaluator.py | 3 + arksim/llms/chat/providers/openai.py | 52 +++++++++++++ arksim/llms/chat/utils.py | 27 +++++++ arksim/simulation_engine/simulator.py | 4 + .../main/user-simulator-on-open-responses.mdx | 8 +- tests/integration/test_responses_provider.py | 61 +++++++++++++++ tests/unit/test_openai_llm_provider.py | 78 +++++++++++++++++++ 8 files changed, 233 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6807417b..262b8877 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * **evaluator:** shared evaluation constants (`SCORE_NOT_COMPUTED`, `BEHAVIOR_FAILURE_THRESHOLD`) * **a2a:** add tool call capture via A2A AgentExtension (`https://arksim.arklex.ai/a2a/tool-call-capture/v1`); agents declare the extension in the AgentCard and surface tool calls in `Task.artifacts[*].metadata`. See [Tool Call Capture docs](https://docs.arklex.ai/main/tool-call-capture) * **llm:** user simulator can now run on any Open Responses-conforming backend via `provider: responses` with configurable `base_url` and `api_key`. Works with OpenAI, Ollama, vLLM, NVIDIA NIM, Vercel AI Gateway, OpenRouter, LM Studio, and other consortium backers. The OpenAI provider has run on `/v1/responses` since v0.3.0; this change unlocks multi-provider reach. +* **llm:** per-instance cache-hit telemetry on the OpenAI provider via `OpenAILLM.cache_stats()`. `run_simulation` and `run_evaluation` log a one-line summary at end of run (call count, input tokens, cached tokens, cache hit rate, output tokens) so users can verify the documented prompt-caching cost reduction without leaving arksim. Tolerates missing `usage` fields from non-OpenAI backends (Ollama, vLLM). * **evaluator:** add `evaluator_model`, `evaluator_provider`, `evaluator_base_url`, and `evaluator_api_key` to split simulator and evaluator backends in a single `config.yaml`. When the evaluator endpoint differs from the simulator endpoint, shared `base_url`/`api_key` are not forwarded to prevent credential leaks; specify evaluator-side credentials explicitly or rely on SDK env-var fallback. * **report:** scenario IDs displayed in HTML report error cards * **ui:** version display in web UI sidebar (next to "Arksim" title) diff --git a/arksim/evaluator/evaluator.py b/arksim/evaluator/evaluator.py index 3ba7e479..86d094a3 100644 --- a/arksim/evaluator/evaluator.py +++ b/arksim/evaluator/evaluator.py @@ -14,6 +14,7 @@ from tqdm import tqdm from arksim.llms.chat import LLM +from arksim.llms.chat.utils import log_cache_stats from arksim.scenario import Scenarios from arksim.scenario.entities import AssertionType, ExpectedToolCall @@ -964,6 +965,8 @@ def run_evaluation( evaluator_output = evaluator.evaluate(simulation, on_progress=on_progress) evaluator.save_results() evaluator.display_evaluation_summary() + # Surface cache-hit telemetry from the evaluator LLM. + log_cache_stats(llm, phase="evaluation") if settings.generate_html_report: from arksim.utils.html_report.generate_html_report import ( diff --git a/arksim/llms/chat/providers/openai.py b/arksim/llms/chat/providers/openai.py index 9e1c7bd2..39ac73f3 100644 --- a/arksim/llms/chat/providers/openai.py +++ b/arksim/llms/chat/providers/openai.py @@ -44,6 +44,56 @@ def __init__( "See https://docs.arklex.ai/main/user-simulator-on-open-responses" ) from exc + # Per-instance cumulative usage counters. Read via `cache_stats()`. + # Surfaces OpenAI's automatic prompt-cache hits so users can verify + # the cost-reduction story without leaving arksim. + self._cumulative_input_tokens: int = 0 + self._cumulative_cached_input_tokens: int = 0 + self._cumulative_output_tokens: int = 0 + self._call_count: int = 0 + + def _record_usage(self, response: object) -> None: + """Accumulate token-usage counters from a Responses API response. + + Tolerates missing fields: older SDK versions or non-OpenAI backends + (Ollama, vLLM) may omit `cached_tokens` or even the full `usage` + object. We never raise from telemetry. + """ + usage = getattr(response, "usage", None) + if usage is None: + return + self._call_count += 1 + self._cumulative_input_tokens += getattr(usage, "input_tokens", 0) or 0 + self._cumulative_output_tokens += getattr(usage, "output_tokens", 0) or 0 + details = getattr(usage, "input_tokens_details", None) + if details is not None: + self._cumulative_cached_input_tokens += ( + getattr(details, "cached_tokens", 0) or 0 + ) + + def cache_stats(self) -> dict[str, int | float]: + """Return cumulative cache-hit statistics across all calls. + + Keys: + - ``call_count``: total responses.parse calls observed + - ``input_tokens``: total input tokens billed + - ``cached_input_tokens``: subset of input_tokens that hit cache + - ``output_tokens``: total output tokens + - ``cache_hit_rate``: cached_input_tokens / input_tokens (0.0 if no calls) + """ + rate = ( + self._cumulative_cached_input_tokens / self._cumulative_input_tokens + if self._cumulative_input_tokens > 0 + else 0.0 + ) + return { + "call_count": self._call_count, + "input_tokens": self._cumulative_input_tokens, + "cached_input_tokens": self._cumulative_cached_input_tokens, + "output_tokens": self._cumulative_output_tokens, + "cache_hit_rate": rate, + } + def _prepare_params( self, messages: str | list[LLMMessage], @@ -87,6 +137,7 @@ def call( ) -> T | str: params = self._prepare_params(messages, schema=schema) response = self.client.responses.parse(**params) + self._record_usage(response) # For structured output, return the parsed output if schema: return response.output_parsed @@ -112,6 +163,7 @@ async def call_async( ) -> T | str: params = self._prepare_params(messages, schema=schema) response = await self.async_client.responses.parse(**params) + self._record_usage(response) # For structured output, return the parsed output if schema: return response.output_parsed diff --git a/arksim/llms/chat/utils.py b/arksim/llms/chat/utils.py index 0ac13f20..a71bea4c 100644 --- a/arksim/llms/chat/utils.py +++ b/arksim/llms/chat/utils.py @@ -79,3 +79,30 @@ def sync_wrapper(*args: object, **kwargs: object) -> object: return sync_wrapper return decorator + + +def log_cache_stats(llm: object, phase: str) -> None: + """Log a one-line summary of LLM cache-hit stats if available. + + Tolerates LLM instances that do not implement ``cache_stats`` (e.g. + Anthropic, Google providers) and never raises from telemetry. + """ + stats_fn = getattr(llm, "cache_stats", None) + if stats_fn is None: + return + try: + stats = stats_fn() + except Exception: + return + if stats.get("call_count", 0) == 0: + return + logger.info( + "%s LLM cache stats: %d calls, %d input tokens, " + "%d cached (%.1f%% hit rate), %d output tokens", + phase, + stats["call_count"], + stats["input_tokens"], + stats["cached_input_tokens"], + stats["cache_hit_rate"] * 100, + stats["output_tokens"], + ) diff --git a/arksim/simulation_engine/simulator.py b/arksim/simulation_engine/simulator.py index 952d677c..096fc2f8 100644 --- a/arksim/simulation_engine/simulator.py +++ b/arksim/simulation_engine/simulator.py @@ -14,6 +14,7 @@ from arksim.config import AgentConfig, AgentType from arksim.llms.chat import LLM +from arksim.llms.chat.utils import log_cache_stats from arksim.scenario import ( KnowledgeItem, Scenarios, @@ -534,6 +535,9 @@ async def run_simulation( ) await simulator.save() + # Surface cache-hit telemetry from the simulator LLM. Helps users + # verify the documented prompt-caching benefit without leaving arksim. + log_cache_stats(llm, phase="simulation") return simulation_output finally: if trace_receiver is not None: diff --git a/docs/main/user-simulator-on-open-responses.mdx b/docs/main/user-simulator-on-open-responses.mdx index 71d11c9e..7b7a6ea8 100644 --- a/docs/main/user-simulator-on-open-responses.mdx +++ b/docs/main/user-simulator-on-open-responses.mdx @@ -107,7 +107,13 @@ Both names resolve to the same Python class under the hood. The distinction is d ## Verifying caching is working -For OpenAI-hosted models, the response payload exposes `usage.input_tokens_details.cached_tokens` on every call. ArkSim does not surface this metric into reports yet. For now, check the [OpenAI usage dashboard](https://platform.openai.com/usage) and look for `cached_input_tokens` on your ArkSim runs. You should see the cached fraction climb across turns of a single scenario. +For OpenAI-hosted models, the response payload exposes `usage.input_tokens_details.cached_tokens` on every call. ArkSim logs cache-hit statistics at the end of each simulation and evaluation run at INFO level. Look for lines like: + +``` +simulation LLM cache stats: 12 calls, 18420 input tokens, 9210 cached (50.0% hit rate), 1840 output tokens +``` + +For deeper detail, the [OpenAI usage dashboard](https://platform.openai.com/usage) exposes `cached_input_tokens` per request. Self-hosted backends do not have OpenAI-equivalent automatic prefix caching out of the box. Ollama maintains a KV cache within a single process but does not persist it across HTTP calls in a way that benefits cross-turn ArkSim runs. vLLM enables prefix caching by default on the V1 engine (the default since v0.8.1, April 2025). Older deployments still on V0 must opt in with `--enable-prefix-caching`. diff --git a/tests/integration/test_responses_provider.py b/tests/integration/test_responses_provider.py index 4123c6fa..00ba3342 100644 --- a/tests/integration/test_responses_provider.py +++ b/tests/integration/test_responses_provider.py @@ -15,12 +15,14 @@ from __future__ import annotations import os +import time import pytest from arksim.llms.chat.llm import LLM _SMOKE_MODEL = os.environ.get("ARKSIM_SMOKE_MODEL", "gpt-4.1-mini") +_CACHE_TEST_MODEL = os.environ.get("ARKSIM_CACHE_TEST_MODEL", "gpt-4o-mini") @pytest.mark.integration @@ -56,3 +58,62 @@ def test_responses_provider_smoke_with_explicit_base_url() -> None: out = llm.call("Say only the word 'hello' and nothing else.") assert isinstance(out, str) assert "hello" in out.lower() + + +@pytest.mark.integration +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), + reason="OPENAI_API_KEY not set; opt-in cache test skipped", +) +def test_responses_provider_cache_hit_on_repeat_prefix() -> None: + """Empirically verify OpenAI's prompt cache fires for repeat prefixes. + + Per OpenAI's prompt-caching docs, prompts >= 1024 tokens with stable + prefixes are cached automatically. This test sends several calls with + the same large prefix and asserts at least one observes cached_tokens + > 0. Multiple calls are needed because OpenAI load-balances across + servers and a fresh prefix may not hit a warm cache on the first + retry. The cumulative ``cached_input_tokens`` counter on the LLM + captures any hit across the batch. + + Why this exists: PR #170 documents a 20-40% cost reduction from + automatic prompt caching. Without this test, the claim is unverified. + + Model pin: this test defaults to gpt-4o-mini regardless of + ``ARKSIM_SMOKE_MODEL`` because some newer models intermittently + report zero cache hits even with valid stable prefixes (observed + on gpt-4.1-mini and reported on gpt-5.x-nano in OpenAI community + threads). Override via ``ARKSIM_CACHE_TEST_MODEL`` if your account + has a different model with reliable caching. + """ + llm = LLM(model=_CACHE_TEST_MODEL, provider="responses") + + # Build a stable prefix comfortably above the 1024-token cache floor. + # Empirically the paragraph below tokenizes to ~35 tokens, so 60 + # repetitions yields ~2100 tokens. + paragraph = ( + "You are a friendly customer service simulator. The user is " + "trying to resolve a billing issue with their subscription. " + "Be patient, ask clarifying questions, and stay in character. " + ) + stable_prefix = paragraph * 60 # ~2100 tokens, comfortably above 1024 + + # Send several calls with a small inter-call delay. The first primes + # the cache; subsequent calls should hit it. OpenAI load-balances + # across cache shards, so we look for at least one hit across the + # batch rather than asserting per-call. + num_calls = 4 + for i in range(num_calls): + llm.call(stable_prefix + f"Reply with only the number {i}.") + if i < num_calls - 1: + time.sleep(2) + + stats = llm.cache_stats() + assert stats["cached_input_tokens"] > 0, ( + f"Expected at least one cache hit across {num_calls} calls with " + f"a ~2100-token stable prefix on model {_CACHE_TEST_MODEL!r}, " + f"got cached_input_tokens=0. Full stats: {stats}. " + f"Possible causes: model does not support prompt caching, " + f"OpenAI cache shard miss across all retries, " + f"or the cached_tokens field is not populated by this backend." + ) diff --git a/tests/unit/test_openai_llm_provider.py b/tests/unit/test_openai_llm_provider.py index 38143cde..e1f9c6d9 100644 --- a/tests/unit/test_openai_llm_provider.py +++ b/tests/unit/test_openai_llm_provider.py @@ -137,3 +137,81 @@ def test_missing_credentials_raises_actionable_error(self) -> None: ) with pytest.raises(ValueError, match="OpenAI SDK requires an api_key"): OpenAILLM(model="gpt-4o-mini") + + +class TestOpenAILLMCacheStats: + def test_cache_stats_starts_at_zero(self) -> None: + with ( + patch("arksim.llms.chat.providers.openai.OpenAI"), + patch("arksim.llms.chat.providers.openai.AsyncOpenAI"), + ): + llm = OpenAILLM(model="gpt-4o-mini") + stats = llm.cache_stats() + assert stats == { + "call_count": 0, + "input_tokens": 0, + "cached_input_tokens": 0, + "output_tokens": 0, + "cache_hit_rate": 0.0, + } + + def test_record_usage_accumulates(self) -> None: + from types import SimpleNamespace + + with ( + patch("arksim.llms.chat.providers.openai.OpenAI"), + patch("arksim.llms.chat.providers.openai.AsyncOpenAI"), + ): + llm = OpenAILLM(model="gpt-4o-mini") + + # First call: no cache hits + response1 = SimpleNamespace( + usage=SimpleNamespace( + input_tokens=1024, + output_tokens=50, + input_tokens_details=SimpleNamespace(cached_tokens=0), + ) + ) + llm._record_usage(response1) + + # Second call: prefix hits cache + response2 = SimpleNamespace( + usage=SimpleNamespace( + input_tokens=1100, + output_tokens=60, + input_tokens_details=SimpleNamespace(cached_tokens=900), + ) + ) + llm._record_usage(response2) + + stats = llm.cache_stats() + assert stats["call_count"] == 2 + assert stats["input_tokens"] == 2124 + assert stats["cached_input_tokens"] == 900 + assert stats["output_tokens"] == 110 + assert stats["cache_hit_rate"] == 900 / 2124 + + def test_record_usage_tolerates_missing_usage_field(self) -> None: + """Self-hosted backends (Ollama, vLLM) may omit usage entirely. + Telemetry must never raise. + """ + from types import SimpleNamespace + + with ( + patch("arksim.llms.chat.providers.openai.OpenAI"), + patch("arksim.llms.chat.providers.openai.AsyncOpenAI"), + ): + llm = OpenAILLM(model="llama3.1") + + # Response without usage attribute at all + response = SimpleNamespace() + llm._record_usage(response) + assert llm.cache_stats()["call_count"] == 0 + + # Response with usage but no input_tokens_details (e.g. older SDK) + response = SimpleNamespace( + usage=SimpleNamespace(input_tokens=500, output_tokens=20) + ) + llm._record_usage(response) + assert llm.cache_stats()["call_count"] == 1 + assert llm.cache_stats()["cached_input_tokens"] == 0 From 942f2148ad40e149d07155ac7c315e178d4c9b3f Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Tue, 19 May 2026 16:16:31 -0700 Subject: [PATCH 22/23] test(integration): add multi-model cache verification matrix --- tests/integration/test_responses_provider.py | 92 ++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tests/integration/test_responses_provider.py b/tests/integration/test_responses_provider.py index 00ba3342..77bd64a5 100644 --- a/tests/integration/test_responses_provider.py +++ b/tests/integration/test_responses_provider.py @@ -24,6 +24,30 @@ _SMOKE_MODEL = os.environ.get("ARKSIM_SMOKE_MODEL", "gpt-4.1-mini") _CACHE_TEST_MODEL = os.environ.get("ARKSIM_CACHE_TEST_MODEL", "gpt-4o-mini") +# Per-model cache verification matrix. Order matters only for readability: +# the arksim DEFAULT_MODEL (`arksim/constants.py`) leads, family siblings +# follow, then known-working baseline, then known-broken regression sentinel. +# +# Names that no longer resolve (``gpt-5.1-mini``, ``gpt-5.1-nano``) skip +# at construction time; we keep them in the list so a future OpenAI release +# of those SKUs is automatically picked up without a test edit. +_CACHE_MATRIX_MODELS = ( + "gpt-5.1", # arksim DEFAULT_MODEL + "gpt-5.1-mini", + "gpt-5.1-nano", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-4o-mini", # historical baseline (cache routing variable) + "gpt-4.1-mini", # slow-warm regression sentinel +) + +# Models that empirically return ``cached_tokens=0`` across all four calls +# even with a stable >=1024-token prefix. The matrix test treats hits on +# these as a regression notice (remove them from the set) rather than a +# failure of arksim. Empty today, retained as the documented escape hatch. +_CACHE_MATRIX_KNOWN_BROKEN: frozenset[str] = frozenset() + @pytest.mark.integration @pytest.mark.skipif( @@ -117,3 +141,71 @@ def test_responses_provider_cache_hit_on_repeat_prefix() -> None: f"OpenAI cache shard miss across all retries, " f"or the cached_tokens field is not populated by this backend." ) + + +@pytest.mark.integration +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), + reason="OPENAI_API_KEY not set; opt-in cache matrix skipped", +) +@pytest.mark.parametrize("model", _CACHE_MATRIX_MODELS) +def test_responses_provider_cache_matrix(model: str) -> None: + """Per-model verification that OpenAI's prompt cache fires. + + Status per model is dynamic (depends on OpenAI's server-side cache + routing for each model and shard warmth). The matrix exists to + (1) prove arksim's DEFAULT_MODEL actually benefits from caching, + (2) flag regressions when OpenAI changes model behavior, and + (3) give PR reviewers an at-a-glance answer for which models + deliver the documented benefit. + + Shape mirrors ``test_responses_provider_cache_hit_on_repeat_prefix``: + four calls with a ~2100-token stable prefix, asserting cumulative + ``cached_input_tokens > 0``. A two-call shape is unreliable because + OpenAI load-balances across cache shards and a fresh prefix often + misses the first repeat. + + Cost: roughly $0.01-0.05 per model per run. + """ + try: + llm = LLM(model=model, provider="responses") + except Exception as exc: + pytest.skip(f"Could not construct LLM for {model}: {exc}") + + paragraph = ( + "You are a friendly customer service simulator. The user is " + "trying to resolve a billing issue with their subscription. " + "Be patient, ask clarifying questions, and stay in character. " + ) + stable_prefix = paragraph * 60 # ~2100 tokens, comfortably above 1024 + + num_calls = 4 + for i in range(num_calls): + try: + llm.call(stable_prefix + f"Reply with only the number {i}.") + except Exception as exc: + pytest.skip(f"Model {model} unavailable or call failed: {exc}") + if i < num_calls - 1: + time.sleep(2) + + stats = llm.cache_stats() + cached = stats["cached_input_tokens"] + + if model in _CACHE_MATRIX_KNOWN_BROKEN: + if cached > 0: + pytest.fail( + f"Model {model} was known-broken (cached_tokens always 0 " + f"across {num_calls} calls) but now shows {cached} cached " + f"tokens. Remove {model!r} from _CACHE_MATRIX_KNOWN_BROKEN." + ) + pytest.skip( + f"Model {model} known to return cached_tokens=0 (server-side " + f"cache-routing anomaly). Observed cached={cached}." + ) + + assert cached > 0, ( + f"Expected at least one cache hit across {num_calls} calls to " + f"{model} with a ~2100-token stable prefix. Got cached={cached}. " + f"Full stats: {stats}. If this is a known model-side regression, " + f"add {model!r} to _CACHE_MATRIX_KNOWN_BROKEN." + ) From d18b16337dc1cf8d2957487aee0a0ef194c15ab6 Mon Sep 17 00:00:00 2001 From: Arbit Chen Date: Tue, 19 May 2026 16:37:09 -0700 Subject: [PATCH 23/23] test(integration): verify cache hits grow across simulator turns --- tests/integration/test_responses_provider.py | 112 +++++++++++++++++-- 1 file changed, 105 insertions(+), 7 deletions(-) diff --git a/tests/integration/test_responses_provider.py b/tests/integration/test_responses_provider.py index 77bd64a5..192ef767 100644 --- a/tests/integration/test_responses_provider.py +++ b/tests/integration/test_responses_provider.py @@ -22,7 +22,7 @@ from arksim.llms.chat.llm import LLM _SMOKE_MODEL = os.environ.get("ARKSIM_SMOKE_MODEL", "gpt-4.1-mini") -_CACHE_TEST_MODEL = os.environ.get("ARKSIM_CACHE_TEST_MODEL", "gpt-4o-mini") +_CACHE_TEST_MODEL = os.environ.get("ARKSIM_CACHE_TEST_MODEL", "gpt-5.1") # Per-model cache verification matrix. Order matters only for readability: # the arksim DEFAULT_MODEL (`arksim/constants.py`) leads, family siblings @@ -103,12 +103,11 @@ def test_responses_provider_cache_hit_on_repeat_prefix() -> None: Why this exists: PR #170 documents a 20-40% cost reduction from automatic prompt caching. Without this test, the claim is unverified. - Model pin: this test defaults to gpt-4o-mini regardless of - ``ARKSIM_SMOKE_MODEL`` because some newer models intermittently - report zero cache hits even with valid stable prefixes (observed - on gpt-4.1-mini and reported on gpt-5.x-nano in OpenAI community - threads). Override via ``ARKSIM_CACHE_TEST_MODEL`` if your account - has a different model with reliable caching. + Model pin: this test defaults to arksim's DEFAULT_MODEL (gpt-5.1) + regardless of ``ARKSIM_SMOKE_MODEL`` so the verified benefit + tracks the model arksim users actually run. Override via + ``ARKSIM_CACHE_TEST_MODEL`` if your account routes a different + model for caching tests. """ llm = LLM(model=_CACHE_TEST_MODEL, provider="responses") @@ -209,3 +208,102 @@ def test_responses_provider_cache_matrix(model: str) -> None: f"Full stats: {stats}. If this is a known model-side regression, " f"add {model!r} to _CACHE_MATRIX_KNOWN_BROKEN." ) + + +@pytest.mark.integration +@pytest.mark.skipif( + not os.environ.get("OPENAI_API_KEY"), + reason="OPENAI_API_KEY not set; opt-in growth test skipped", +) +def test_cache_hits_grow_across_simulator_turns() -> None: + """Cache fires across multi-turn simulator calls (arksim's real shape). + + arksim's user simulator sends [system + scenario + growing history + + trigger] on every turn. The stable prefix is reused across turns, so + OpenAI's prompt cache fires from turn 2 onward and the cumulative hit + rate climbs sharply across a multi-turn scenario. + + Observed cold-cache shape on gpt-5.1 (turn-1 starts cold, turns 2-5 + hit 85-95%): per-turn rates [0%, 86%, 92%, 89%, 95%], cumulative 74%. + + Distinct from the matrix test, which sends the SAME prefix twice + (idealized retry shape). This test exercises the real arksim call + pattern where each turn's prompt differs but shares a stable prefix. + """ + llm = LLM(model=_CACHE_TEST_MODEL, provider="responses") + + # OpenAI's prompt cache only fires for prefixes >= 1024 tokens. Size the + # stable portion (system + scenario) to ~1250 tokens so the cache is + # eligible from turn 1. Real arksim user-simulator prompts in + # production tend to land in the 1500-3000 token range once the + # scenario goal and persona are populated. + system_prompt = ( + "You are a simulated user testing a customer-service AI agent. " + "Stay in character, respond naturally, and explore the agent's " + "capabilities. Do not break character or reveal that you are a " + "simulator. Always end your response with a follow-up question " + "or a clarification request when appropriate. " + ) * 10 # ~530 tokens + + scenario_context = ( + "Scenario: You are a long-time customer of XYZ Bank. Your goal " + "is to dispute a charge on your credit card statement from last " + "month. You believe the charge of $89.99 from 'Acme Subscriptions' " + "is unauthorized; you never signed up for this service. You are " + "polite but firm. You have your account number ready: 4532-XXXX. " + ) * 10 # ~720 tokens + + history: list[str] = [] + hit_rates: list[float] = [] + cached_per_turn: list[int] = [] + + for turn in range(5): + previous_cached = llm.cache_stats()["cached_input_tokens"] + previous_input = llm.cache_stats()["input_tokens"] + + message = ( + system_prompt + + "\n\n" + + scenario_context + + "\n\nConversation so far:\n" + + "\n".join(history) + + f"\n\nIt is turn {turn + 1}. Respond as the user " + "with one short sentence." + ) + response = llm.call(message) + history.append(f"user (turn {turn + 1}): {response}") + history.append( + f"agent (turn {turn + 1}): I understand. Let me look into that for you." + ) + + new_cached = llm.cache_stats()["cached_input_tokens"] - previous_cached + new_input = llm.cache_stats()["input_tokens"] - previous_input + rate = new_cached / new_input if new_input > 0 else 0.0 + hit_rates.append(rate) + cached_per_turn.append(new_cached) + + final_stats = llm.cache_stats() + + # Headline assertion: cumulative cache hit rate must reflect the + # growing prefix benefit. After 5 turns of a scenario, at least 30% + # of input tokens should be cached. This is a conservative floor. + overall_rate = final_stats["cache_hit_rate"] + assert overall_rate >= 0.30, ( + f"Expected >= 30% cumulative cache hit rate after 5 turns, " + f"got {overall_rate:.1%}. Per-turn: {hit_rates}. " + f"Cached tokens per turn: {cached_per_turn}. " + f"This is the realistic shape arksim users hit; if this drops " + f"below 30% the documented cost-reduction claim is wrong." + ) + + # Cache must keep firing as the conversation grows. OpenAI's prompt + # cache rounds to 128-token blocks and load-balances across shards, + # so we cannot assert strict monotonic growth (cold shards may flake + # a single turn back to 0). We assert that the cache fires on the + # majority of turns and that the cached-token count never collapses. + turns_with_hits = sum(1 for c in cached_per_turn if c > 0) + assert turns_with_hits >= 3, ( + f"Cache must fire on at least 3 of 5 turns for the growth claim " + f"to hold. Got {turns_with_hits} turns with hits. " + f"Per-turn cached tokens: {cached_per_turn}." + )