From 0e1cf406209e330677b55ce4a4a85f46b754048e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 04:39:45 +0000 Subject: [PATCH 1/6] move the LLM instance directly to Agent to make it cleaner to share with tests This gets rid of the awkward AGENT_MODEL constant by making the LLM an inherent property of the Agent, which is more intuitive. Also switched the test judge model to base 5.2 instead of the chat version to make it clearer that you can (and should) use a different model for evals than for core chat. --- src/agent.test.ts | 12 +++++------- src/agent.ts | 17 ++++++++++++++--- src/main.ts | 18 +----------------- 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/src/agent.test.ts b/src/agent.test.ts index 366c04e..28edb72 100644 --- a/src/agent.test.ts +++ b/src/agent.test.ts @@ -1,7 +1,7 @@ import { dedent, inference, initializeLogger, voice } from '@livekit/agents'; import dotenv from 'dotenv'; import { afterEach, beforeEach, describe, it } from 'vitest'; -import { AGENT_MODEL, Agent } from './agent'; +import { Agent } from './agent'; dotenv.config({ path: '.env.local' }); @@ -11,20 +11,18 @@ initializeLogger({ pretty: true, level: 'warn' }); describe('agent evaluation', () => { let session: voice.AgentSession; - let agentLlm: inference.LLM; let judgeLlm: inference.LLM; beforeEach(async () => { - agentLlm = new inference.LLM({ model: AGENT_MODEL }); - // The judge LLM can be a cheaper model since it only evaluates agent responses - judgeLlm = new inference.LLM({ model: 'openai/gpt-4.1-mini' }); - session = new voice.AgentSession({ llm: agentLlm }); + // We can use a different LLM to evaluate the agent's responses than the one used in the agent itself + // This allows you to use reasoning capabilities or larger models than would be practical for realtime chat + judgeLlm = new inference.LLM({ model: 'openai/gpt-5.2' }); + session = new voice.AgentSession(); await session.start({ agent: new Agent() }); }); afterEach(async () => { await session?.close(); - await agentLlm?.aclose(); await judgeLlm?.aclose(); }); diff --git a/src/agent.ts b/src/agent.ts index 0b9ee7d..39aded6 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -1,6 +1,4 @@ -import { dedent, voice } from '@livekit/agents'; - -export const AGENT_MODEL = 'openai/gpt-5.2-chat-latest'; +import { dedent, inference, voice } from '@livekit/agents'; // Define a custom voice AI assistant by extending the base Agent class export class Agent extends voice.Agent { @@ -13,6 +11,19 @@ export class Agent extends voice.Agent { You are curious, friendly, and have a sense of humor. `, + // A Large Language Model (LLM) is your agent's brain, processing user input and generating a response + // See all available models at https://docs.livekit.io/agents/models/llm/ + llm: new inference.LLM({ model: 'openai/gpt-5.2-chat-latest' }), + + // To use a realtime model instead of a voice pipeline, replace the LLM + // with a RealtimeModel and remove the STT/TTS from the AgentSession + // (Note: This is for the OpenAI Realtime API. For other providers, see https://docs.livekit.io/agents/models/realtime/) + // 1. Install '@livekit/agents-plugin-openai' + // 2. Set OPENAI_API_KEY in .env.local + // 3. Add `import * as openai from '@livekit/agents-plugin-openai'` to the top of this file + // 4. Replace the llm option with: + // llm: new openai.realtime.RealtimeModel({ voice: 'marin' }), + // To add tools, specify `tools` in the constructor. // Here's an example that adds a simple weather tool. // You also have to add `import { llm } from '@livekit/agents' and `import { z } from 'zod'` to the top of this file diff --git a/src/main.ts b/src/main.ts index 11ffdbc..d46bcbc 100644 --- a/src/main.ts +++ b/src/main.ts @@ -4,7 +4,7 @@ import * as silero from '@livekit/agents-plugin-silero'; import { audioEnhancement } from '@livekit/plugins-ai-coustics'; import dotenv from 'dotenv'; import { fileURLToPath } from 'node:url'; -import { AGENT_MODEL, Agent } from './agent'; +import { Agent } from './agent'; // Load environment variables from a local file. // Make sure to set LIVEKIT_URL, LIVEKIT_API_KEY, and LIVEKIT_API_SECRET @@ -29,12 +29,6 @@ export default defineAgent({ language: 'multi', }), - // A Large Language Model (LLM) is your agent's brain, processing user input and generating a response - // See all providers at https://docs.livekit.io/agents/models/llm/ - llm: new inference.LLM({ - model: AGENT_MODEL, - }), - // Text-to-speech (TTS) is your agent's voice, turning the LLM's text into speech that the user can hear // See all available models as well as voice selections at https://docs.livekit.io/agents/models/tts/ tts: new inference.TTS({ @@ -52,16 +46,6 @@ export default defineAgent({ }, }); - // To use a realtime model instead of a voice pipeline, use the following session setup instead. - // (Note: This is for the OpenAI Realtime API. For other providers, see https://docs.livekit.io/agents/models/realtime/)) - // 1. Install '@livekit/agents-plugin-openai' - // 2. Set OPENAI_API_KEY in .env.local - // 3. Add import `import * as openai from '@livekit/agents-plugin-openai'` to the top of this file - // 4. Use the following session setup instead of the version above - // const session = new voice.AgentSession({ - // llm: new openai.realtime.RealtimeModel({ voice: 'marin' }), - // }); - // Start the session, which initializes the voice pipeline and warms up the models await session.start({ agent: new Agent(), From 8f4715b11772fbf45526b3956ef1327938c1190f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Apr 2026 04:58:20 +0000 Subject: [PATCH 2/6] pass empty options to AgentSession in tests The constructor requires an options arg and destructures it; calling new voice.AgentSession() with no arg throws at runtime. --- src/agent.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent.test.ts b/src/agent.test.ts index 28edb72..0deda05 100644 --- a/src/agent.test.ts +++ b/src/agent.test.ts @@ -17,7 +17,7 @@ describe('agent evaluation', () => { // We can use a different LLM to evaluate the agent's responses than the one used in the agent itself // This allows you to use reasoning capabilities or larger models than would be practical for realtime chat judgeLlm = new inference.LLM({ model: 'openai/gpt-5.2' }); - session = new voice.AgentSession(); + session = new voice.AgentSession({}); await session.start({ agent: new Agent() }); }); From 485e48f85f08db437c5adfe37457baddd2b9e625 Mon Sep 17 00:00:00 2001 From: Ben Cherry Date: Mon, 4 May 2026 14:09:25 -0700 Subject: [PATCH 3/6] 41 --- src/agent.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/agent.test.ts b/src/agent.test.ts index 0deda05..47b148b 100644 --- a/src/agent.test.ts +++ b/src/agent.test.ts @@ -15,8 +15,7 @@ describe('agent evaluation', () => { beforeEach(async () => { // We can use a different LLM to evaluate the agent's responses than the one used in the agent itself - // This allows you to use reasoning capabilities or larger models than would be practical for realtime chat - judgeLlm = new inference.LLM({ model: 'openai/gpt-5.2' }); + judgeLlm = new inference.LLM({ model: 'openai/gpt-4.1-mini' }); session = new voice.AgentSession({}); await session.start({ agent: new Agent() }); }); From ed1569eca2608701b3c4c740da8e83f70925863a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 4 May 2026 22:17:01 +0000 Subject: [PATCH 4/6] pin openai to <6.36.0 to work around upstream apiKey check openai 6.36.0 changed its credential check to reject empty-string apiKey, but @livekit/agents@1.3.x hardcodes apiKey: '' when constructing its OpenAI client inside inference.LLM. Pin to <6.36.0 until the agents SDK is patched. --- package.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/package.json b/package.json index 388ecef..8642008 100644 --- a/package.json +++ b/package.json @@ -40,5 +40,10 @@ "@livekit/plugins-ai-coustics": "^0.2.10", "dotenv": "^17.4.1", "zod": "^3.25.76" + }, + "pnpm": { + "overrides": { + "openai": "<6.36.0" + } } } From c5f141269822b5bbf28cf75cec4781bba62a18bd Mon Sep 17 00:00:00 2001 From: Ben Cherry Date: Thu, 7 May 2026 12:22:00 -0700 Subject: [PATCH 5/6] cleanup --- package.json | 5 ----- src/agent.test.ts | 1 - 2 files changed, 6 deletions(-) diff --git a/package.json b/package.json index 8642008..388ecef 100644 --- a/package.json +++ b/package.json @@ -40,10 +40,5 @@ "@livekit/plugins-ai-coustics": "^0.2.10", "dotenv": "^17.4.1", "zod": "^3.25.76" - }, - "pnpm": { - "overrides": { - "openai": "<6.36.0" - } } } diff --git a/src/agent.test.ts b/src/agent.test.ts index 47b148b..c2841cc 100644 --- a/src/agent.test.ts +++ b/src/agent.test.ts @@ -14,7 +14,6 @@ describe('agent evaluation', () => { let judgeLlm: inference.LLM; beforeEach(async () => { - // We can use a different LLM to evaluate the agent's responses than the one used in the agent itself judgeLlm = new inference.LLM({ model: 'openai/gpt-4.1-mini' }); session = new voice.AgentSession({}); await session.start({ agent: new Agent() }); From bd6b26145dcfb85b0113ed64e5eade198ea04968 Mon Sep 17 00:00:00 2001 From: Ben Cherry Date: Thu, 7 May 2026 12:46:23 -0700 Subject: [PATCH 6/6] v --- package.json | 8 ++++---- src/agent.test.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 388ecef..eda6c56 100644 --- a/package.json +++ b/package.json @@ -34,10 +34,10 @@ "vitest": "^4.1.4" }, "dependencies": { - "@livekit/agents": "^1.3.0", - "@livekit/agents-plugin-livekit": "^1.3.0", - "@livekit/agents-plugin-silero": "^1.3.0", - "@livekit/plugins-ai-coustics": "^0.2.10", + "@livekit/agents": "^1.4.0", + "@livekit/agents-plugin-livekit": "^1.4.0", + "@livekit/agents-plugin-silero": "^1.4.0", + "@livekit/plugins-ai-coustics": "^0.2.12", "dotenv": "^17.4.1", "zod": "^3.25.76" } diff --git a/src/agent.test.ts b/src/agent.test.ts index c2841cc..46de9fa 100644 --- a/src/agent.test.ts +++ b/src/agent.test.ts @@ -15,7 +15,7 @@ describe('agent evaluation', () => { beforeEach(async () => { judgeLlm = new inference.LLM({ model: 'openai/gpt-4.1-mini' }); - session = new voice.AgentSession({}); + session = new voice.AgentSession(); await session.start({ agent: new Agent() }); });