feat(livekit): add agents integration

Add LiveKit Agents instrumentation, e2e VCR coverage, and nox/CI wiring for the livekit-agents matrix.

Also preserve OpenAI async streaming speech responses when tracing raw response calls.

Author: AbhiPrasad

.github/workflows/checks.yaml

@@ -113,6 +113,25 @@ jobs:
           dir="$HOME/.cache/braintrust/temporal-test-server"
           mkdir -p "$dir"
           echo "BRAINTRUST_TEMPORAL_TEST_SERVER_DIR=$dir" >> "$GITHUB_ENV"
+      - name: Cache LiveKit server binaries
+        if: runner.os == 'Linux'
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          # The LiveKit Agents nox session downloads a pinned standalone
+          # livekit-server binary here when livekit-server is not already on
+          # PATH. Caching avoids repeated GitHub release downloads across CI shards.
+          path: ~/.cache/braintrust/livekit-server
+          key: livekit-server-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('py/noxfile.py') }}
+          restore-keys: |
+            livekit-server-${{ runner.os }}-${{ runner.arch }}-
+      - name: Configure LiveKit server cache dir
+        if: runner.os == 'Linux'
+        shell: bash
+        run: |
+          set -euo pipefail
+          dir="$HOME/.cache/braintrust/livekit-server"
+          mkdir -p "$dir"
+          echo "BRAINTRUST_LIVEKIT_SERVER_DIR=$dir" >> "$GITHUB_ENV"
       - name: Run nox tests (shard ${{ matrix.shard }}/6)
         shell: bash
         run: |

py/noxfile.py

@@ -12,11 +12,16 @@
 
 import functools
 import glob
+import hashlib
 import os
 import pathlib
+import platform
 import re
+import shutil
 import sys
+import tarfile
 import tempfile
+import urllib.request
 
 from packaging.version import Version
 
@@ -46,6 +51,54 @@
 _PROJECT_DIR = str(pathlib.Path(__file__).parent)
 
 
+def _ensure_livekit_server(session: nox.Session) -> str:
+    """Ensure a standalone livekit-server binary is available for LiveKit e2e tests."""
+    existing = shutil.which("livekit-server")
+    if existing:
+        return os.path.dirname(existing)
+
+    system = platform.system().lower()
+    machine = platform.machine().lower()
+    arch = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"}.get(machine)
+    if arch is None:
+        session.skip(f"No pinned livekit-server binary for architecture {machine!r}")
+
+    if system != "linux":
+        session.skip(
+            "No pinned standalone livekit-server release asset is available for this platform; "
+            "install livekit-server on PATH to run LiveKit e2e tests locally"
+        )
+
+    cache_root = pathlib.Path(os.environ.get("BRAINTRUST_LIVEKIT_SERVER_DIR", ".nox/livekit-server"))
+    install_dir = cache_root / LIVEKIT_SERVER_VERSION / f"{system}_{arch}"
+    binary = install_dir / "livekit-server"
+    if binary.exists():
+        return str(install_dir.resolve())
+
+    install_dir.mkdir(parents=True, exist_ok=True)
+    asset = f"livekit_{LIVEKIT_SERVER_VERSION}_{system}_{arch}.tar.gz"
+    url = f"https://github.com/livekit/livekit/releases/download/v{LIVEKIT_SERVER_VERSION}/{asset}"
+    archive = install_dir / asset
+    expected_sha256 = LIVEKIT_SERVER_SHA256[f"{system}_{arch}"]
+    session.log(f"Downloading {url}")
+    urllib.request.urlretrieve(url, archive)  # noqa: S310 - pinned public release asset for test infra.
+    actual_sha256 = hashlib.sha256(archive.read_bytes()).hexdigest()
+    if actual_sha256 != expected_sha256:
+        archive.unlink(missing_ok=True)
+        session.error(
+            f"SHA256 mismatch for {asset}: expected {expected_sha256}, got {actual_sha256}. "
+            "Refusing to extract downloaded livekit-server archive."
+        )
+    with tarfile.open(archive, "r:gz") as tar:
+        if sys.version_info >= (3, 12):
+            tar.extract("livekit-server", path=install_dir, filter="data")
+        else:
+            tar.extract("livekit-server", path=install_dir)  # noqa: S202
+    binary.chmod(0o755)
+    archive.unlink()
+    return str(install_dir.resolve())
+
+
 def _install_group_locked(session: nox.Session, *group_names: str) -> None:
     """Install deps from one or more dependency groups using the lockfile.
 
@@ -128,6 +181,11 @@ def _pinned_python_version():
 
 SILENT_INSTALLS = True
 LATEST = "latest"
+LIVEKIT_SERVER_VERSION = "1.11.0"
+LIVEKIT_SERVER_SHA256 = {
+    "linux_amd64": "3e76ed51ecdfefc3005e4257095dccd1ccc8f8b77517d9f2353de7906650b68b",
+    "linux_arm64": "6741466bc12e75544338292ab2c1c02c02f3c626568230b5548fffc53e5a87ff",
+}
 ERROR_CODES = tuple(range(1, 256))
 INTERNAL_TEST_FLAGS = {"--wheel", "--disable-vcr"}
 GENERATED_LINT_EXCLUDES = {
@@ -266,6 +324,27 @@ def test_agno(session, version):
     _run_tests(session, f"{INTEGRATION_DIR}/agno/test_workflow.py", version=version)
 
 
+LIVEKIT_AGENTS_VERSIONS = _get_matrix_versions("livekit-agents")
+
+
+@nox.session()
+@nox.parametrize("version", LIVEKIT_AGENTS_VERSIONS, ids=LIVEKIT_AGENTS_VERSIONS)
+def test_livekit_agents(session, version):
+    if sys.version_info >= (3, 14):
+        session.skip("LiveKit Agents Silero VAD depends on onnxruntime, which does not ship Python 3.14 wheels")
+    _install_test_deps(session)
+    _install_matrix_dep(session, "livekit-agents", version)
+    _install_group_locked(session, "test-livekit-agents")
+    livekit_server_dir = _ensure_livekit_server(session)
+    env = {
+        "LIVEKIT_URL": os.environ.get("LIVEKIT_URL", "ws://localhost:7880"),
+        "LIVEKIT_API_KEY": os.environ.get("LIVEKIT_API_KEY", "devkey"),
+        "LIVEKIT_API_SECRET": os.environ.get("LIVEKIT_API_SECRET", "secret"),
+        "PATH": f"{livekit_server_dir}{os.pathsep}{os.environ.get('PATH', '')}",
+    }
+    _run_tests(session, f"{INTEGRATION_DIR}/livekit_agents/test_livekit_agents.py", version=version, env=env)
+
+
 STRANDS_VERSIONS = _get_matrix_versions("strands-agents")
 
 

py/pyproject.toml

@@ -157,6 +157,15 @@ test-langchain = [
     "langgraph==1.1.6",
 ]
 
+test-livekit-agents = [
+    {include-group = "test"},
+    "livekit-plugins-openai",
+    "livekit-plugins-silero",
+    # livekit-agents 1.3.x imports opentelemetry.sdk._logs.LogData, removed in newer SDKs.
+    # This is a LiveKit runtime compatibility constraint, not a Braintrust OTel integration dependency.
+    "opentelemetry-sdk<1.39",
+]
+
 test-crewai = [
     {include-group = "test"},
     # CrewAI's no-network smoke test forces the LiteLLM fallback path via
@@ -212,6 +221,8 @@ lint = [
     "google-adk",
     "google-genai",
     "litellm>=1.83.10",
+    "livekit-agents",
+    "livekit-plugins-openai",
     "mistralai",
     "openai",
     "openai-agents",
@@ -263,12 +274,14 @@ conflicts = [
         {group = "test-agentscope"},
         {group = "test-strands"},
         {group = "test-langchain"},
+        {group = "test-livekit-agents"},
         {group = "lint"},
     ],
-    # opentelemetry-sdk version conflicts (google-adk vs logfire).
+    # opentelemetry-sdk version conflicts (google-adk/livekit vs logfire).
     [
         {group = "lint"},
         {group = "test-pydantic-ai-logfire"},
+        {group = "test-livekit-agents"},
     ],
 ]
 
@@ -309,6 +322,10 @@ latest = "openai-agents==0.17.1"
 latest = "litellm==1.83.14"
 "1.74.0" = "litellm==1.74.0"
 
+[tool.braintrust.matrix.livekit-agents]
+latest = "livekit-agents==1.3.6"
+"1.3.1" = "livekit-agents==1.3.1"
+
 [tool.braintrust.matrix.claude-agent-sdk]
 latest = "claude-agent-sdk==0.1.80"
 "0.1.10" = "claude-agent-sdk==0.1.10"
@@ -427,6 +444,7 @@ dspy = ["dspy"]
 google_genai = ["google-genai"]
 langchain = ["langchain-core"]
 litellm = ["litellm"]
+livekit_agents = ["livekit-agents"]
 llamaindex = ["llama-index-core"]
 mistral = ["mistralai"]
 openai = ["openai"]
@@ -449,6 +467,7 @@ dspy = "dspy"
 google-adk = "google.adk"
 google-genai = "google.genai"
 litellm = "litellm"
+livekit-agents = "livekit.agents"
 mistralai = "mistralai"
 openai = "openai"
 openai-agents = "agents"

py/scripts/bump-livekit-server.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Update pinned livekit-server version and archive SHA256 hashes in noxfile.py."""
+
+import argparse
+import hashlib
+import pathlib
+import re
+import urllib.request
+
+
+PROJECT_DIR = pathlib.Path(__file__).resolve().parents[1]
+NOXFILE = PROJECT_DIR / "noxfile.py"
+PLATFORMS = ("linux_amd64", "linux_arm64")
+
+
+def _asset_url(version: str, platform: str) -> str:
+    system, arch = platform.split("_", 1)
+    asset = f"livekit_{version}_{system}_{arch}.tar.gz"
+    return f"https://github.com/livekit/livekit/releases/download/v{version}/{asset}"
+
+
+def _sha256_url(url: str) -> str:
+    with urllib.request.urlopen(url, timeout=120) as response:  # noqa: S310 - pinned GitHub release URL.
+        digest = hashlib.sha256()
+        while chunk := response.read(1024 * 1024):
+            digest.update(chunk)
+        return digest.hexdigest()
+
+
+def _replace_constants(contents: str, version: str, hashes: dict[str, str]) -> str:
+    contents = re.sub(
+        r'LIVEKIT_SERVER_VERSION = "[^"]+"',
+        f'LIVEKIT_SERVER_VERSION = "{version}"',
+        contents,
+        count=1,
+    )
+    sha_block = (
+        "LIVEKIT_SERVER_SHA256 = {\n"
+        + "".join(f'    "{platform}": "{hashes[platform]}",\n' for platform in PLATFORMS)
+        + "}"
+    )
+    contents = re.sub(
+        r'LIVEKIT_SERVER_SHA256 = \{\n(?:    "[^"]+": "[0-9a-f]+",\n)+\}',
+        sha_block,
+        contents,
+        count=1,
+    )
+    return contents
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("version", help="livekit-server version, e.g. 1.11.1")
+    parser.add_argument("--dry-run", action="store_true", help="print the new constants without editing noxfile.py")
+    args = parser.parse_args()
+
+    hashes = {}
+    for platform in PLATFORMS:
+        url = _asset_url(args.version, platform)
+        print(f"Downloading {url}")
+        hashes[platform] = _sha256_url(url)
+        print(f"{platform}: {hashes[platform]}")
+
+    contents = NOXFILE.read_text()
+    updated = _replace_constants(contents, args.version, hashes)
+
+    if args.dry_run:
+        print()
+        print(f'LIVEKIT_SERVER_VERSION = "{args.version}"')
+        print("LIVEKIT_SERVER_SHA256 = {")
+        for platform in PLATFORMS:
+            print(f'    "{platform}": "{hashes[platform]}",')
+        print("}")
+        return
+
+    if updated == contents:
+        raise SystemExit("noxfile.py did not change; constants may not have matched expected format")
+
+    NOXFILE.write_text(updated)
+    print(f"Updated {NOXFILE.relative_to(PROJECT_DIR)}")
+
+
+if __name__ == "__main__":
+    main()

py/src/braintrust/auto.py

@@ -20,6 +20,7 @@
     GoogleGenAIIntegration,
     LangChainIntegration,
     LiteLLMIntegration,
+    LiveKitAgentsIntegration,
     LlamaIndexIntegration,
     MistralIntegration,
     OpenAIAgentsIntegration,
@@ -70,6 +71,7 @@ def auto_instrument(
     crewai: bool = True,
     strands: bool = True,
     temporal: bool = True,
+    livekit_agents: bool = True,
 ) -> dict[str, bool]:
     """
     Auto-instrument supported AI/ML libraries for Braintrust tracing.
@@ -101,6 +103,7 @@ def auto_instrument(
         crewai: Enable CrewAI instrumentation (default: True)
         strands: Enable Strands Agents instrumentation (default: True)
         temporal: Enable Temporal instrumentation (default: True)
+        livekit_agents: Enable LiveKit Agents instrumentation (default: True)
 
     Returns:
         Dict mapping integration name to whether it was successfully instrumented.
@@ -188,6 +191,8 @@ def auto_instrument(
         results["strands"] = _instrument_integration(StrandsIntegration)
     if temporal:
         results["temporal"] = _instrument_integration(TemporalIntegration)
+    if livekit_agents:
+        results["livekit_agents"] = _instrument_integration(LiveKitAgentsIntegration)
 
     return results
 

py/src/braintrust/integrations/__init__.py

@@ -10,6 +10,7 @@
 from .google_genai import GoogleGenAIIntegration
 from .langchain import LangChainIntegration
 from .litellm import LiteLLMIntegration
+from .livekit_agents import LiveKitAgentsIntegration
 from .llamaindex import LlamaIndexIntegration
 from .mistral import MistralIntegration
 from .openai import OpenAIIntegration
@@ -32,6 +33,7 @@
     "DSPyIntegration",
     "GoogleGenAIIntegration",
     "LiteLLMIntegration",
+    "LiveKitAgentsIntegration",
     "LangChainIntegration",
     "LlamaIndexIntegration",
     "MistralIntegration",

py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents.py

@@ -0,0 +1,47 @@
+"""Test auto_instrument for LiveKit Agents."""
+
+import inspect
+
+from braintrust.auto import auto_instrument
+from wrapt import FunctionWrapper
+
+
+def _is_braintrust_wrapped(target, attr: str) -> bool:
+    return isinstance(inspect.getattr_static(target, attr, None), FunctionWrapper)
+
+
+# Import the provider classes before auto-instrumentation to verify setup handles
+# normal user import order in a fresh process.
+from livekit.agents import AgentSession  # noqa: E402
+from livekit.agents.inference.llm import LLMStream  # noqa: E402
+from livekit.agents.stt import STT  # noqa: E402
+from livekit.agents.tts import TTS  # noqa: E402
+from livekit.agents.voice import generation  # noqa: E402
+from livekit.agents.voice.io import AudioOutput  # noqa: E402
+
+
+assert not _is_braintrust_wrapped(AgentSession, "run")
+assert not _is_braintrust_wrapped(AgentSession, "_on_audio_output_changed")
+assert not _is_braintrust_wrapped(AgentSession, "_update_user_state")
+assert not isinstance(generation._execute_tools_task, FunctionWrapper)
+assert not _is_braintrust_wrapped(LLMStream, "_run")
+assert not _is_braintrust_wrapped(STT, "recognize")
+assert not _is_braintrust_wrapped(TTS, "synthesize")
+assert not _is_braintrust_wrapped(AudioOutput, "capture_frame")
+
+results = auto_instrument()
+assert results.get("livekit_agents") is True
+assert _is_braintrust_wrapped(AgentSession, "run")
+assert _is_braintrust_wrapped(AgentSession, "_on_audio_output_changed")
+assert _is_braintrust_wrapped(AgentSession, "_update_user_state")
+assert isinstance(generation._execute_tools_task, FunctionWrapper)
+assert _is_braintrust_wrapped(LLMStream, "_run")
+assert _is_braintrust_wrapped(STT, "recognize")
+assert _is_braintrust_wrapped(TTS, "synthesize")
+assert _is_braintrust_wrapped(AudioOutput, "capture_frame")
+
+# Idempotent.
+results2 = auto_instrument()
+assert results2.get("livekit_agents") is True
+
+print("SUCCESS")