Run tests against different dependencies.

.github/workflows/ci.yml

@@ -29,13 +29,6 @@ jobs:
         BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
 
-    - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v4
-      if: matrix.ruby-version == '3.4' && matrix.os == 'ubuntu-latest'
-      with:
-        files: ./coverage/.resultset.json
-        fail_ci_if_error: false
-
   # Summary job that requires all matrix tests to pass
   # This is what branch protection will check
   ci-success:

.gitignore

@@ -48,3 +48,6 @@
 
 # Release artifacts
 changelog.md
+
+# Appraisal: commit gemfiles but ignore lock files
+gemfiles/*.gemfile.lock

Appraisals

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Test with OpenAI gem 0.33.x (previous stable version)
+appraise "openai-0.33" do
+  gem "openai", "~> 0.33.0"
+end
+
+# Test with current stable OpenAI gem version
+appraise "openai-0.34" do
+  gem "openai", "~> 0.34.0"
+end
+
+# Test with latest OpenAI gem version (allows newer patch/minor versions)
+appraise "openai-latest" do
+  gem "openai", ">= 0.34"
+end
+
+# Test without OpenAI gem (verify SDK works without optional dependency)
+appraise "openai-uninstalled" do
+  remove_gem "openai"
+end

Gemfile.lock

@@ -12,11 +12,14 @@ GEM
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
     ansi (1.5.0)
+    appraisal (2.5.0)
+      bundler
+      rake
+      thor (>= 0.14.0)
     ast (2.4.3)
     base64 (0.3.0)
     bigdecimal (3.3.1)
     builder (3.3.0)
-    connection_pool (2.5.4)
     crack (1.0.1)
       bigdecimal
       rexml
@@ -42,8 +45,6 @@ GEM
       builder
       minitest (>= 5.0)
       ruby-progressbar
-    openai (0.34.1)
-      connection_pool
     openssl (3.3.1)
     opentelemetry-api (1.7.0)
     opentelemetry-common (0.23.0)
@@ -112,6 +113,7 @@ GEM
     standard-performance (1.8.0)
       lint_roller (~> 1.1)
       rubocop-performance (~> 1.25.0)
+    thor (1.4.0)
     unicode-display_width (3.2.0)
       unicode-emoji (~> 4.1)
     unicode-emoji (4.1.0)
@@ -130,10 +132,10 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  appraisal (~> 2.5)
   braintrust!
   minitest (~> 5.0)
   minitest-reporters (~> 1.6)
-  openai (~> 0.34)
   rake (~> 13.0)
   simplecov (~> 0.22)
   standard (~> 1.0)

Rakefile

@@ -9,12 +9,6 @@ Rake::TestTask.new(:test) do |t|
   t.warning = false
 end
 
-desc "Run tests with verbose timing output"
-task :"test:verbose" do
-  ENV["MT_VERBOSE"] = "1"
-  Rake::Task[:test].invoke
-end
-
 desc "Run Standard linter"
 task :lint do
   sh "bundle exec standardrb"
@@ -70,13 +64,62 @@ task coverage: :test do
   end
 end
 
-desc "Verify CI (lint + test)"
-task ci: [:lint, :test]
+desc "Verify CI (lint + test all appraisal scenarios)"
+task ci: [:lint, :"test:appraisal:install", :"test:appraisal"]
 
 task default: :ci
 
-# VCR tasks for managing HTTP cassettes
+# Test-related tasks
 namespace :test do
+  desc "Run tests with verbose timing output"
+  task :verbose do
+    ENV["MT_VERBOSE"] = "1"
+    Rake::Task[:test].invoke
+  end
+
+  desc "Install optional test dependencies (e.g., openai gem)"
+  task :install do
+    puts "Installing optional test dependencies..."
+    sh "gem install openai -v '~> 0.34'"
+    puts "✓ Optional dependencies installed"
+    puts ""
+    puts "Now run 'rake test' to run tests with OpenAI integration"
+  end
+
+  # Appraisal tasks for testing with/without optional dependencies
+  # Run directly: bundle exec appraisal [scenario] rake test
+  # List scenarios: bundle exec appraisal list
+  desc "Run tests against different dependencies"
+  task :appraisal do
+    sh "bundle exec appraisal rake test"
+  end
+
+  namespace :appraisal do
+    desc "Show help for appraisal scenarios and usage"
+    task :help do
+      puts "\n=== Appraisal Test Scenarios ==="
+      puts "\nAvailable scenarios:"
+      sh "bundle exec appraisal list"
+      puts "\n=== Usage ==="
+      puts "Run specific scenario:"
+      puts "  bundle exec appraisal <scenario> rake test"
+      puts ""
+      puts "Example:"
+      puts "  bundle exec appraisal openai-0.34 rake test"
+      puts ""
+      puts "Run all scenarios:"
+      puts "  bundle exec appraisal rake test"
+      puts "  or: rake test:appraisal"
+      puts ""
+    end
+
+    desc "Install all appraisal gemfiles"
+    task :install do
+      sh "bundle exec appraisal install"
+    end
+  end
+
+  # VCR tasks for managing HTTP cassettes
   namespace :vcr do
     desc "Re-record all VCR cassettes"
     task :record_all do

braintrust.gemspec

@@ -23,7 +23,6 @@ Gem::Specification.new do |spec|
     lib/**/*.rb
     README.md
     LICENSE
-    CHANGELOG.md
   ])
   spec.bindir = "exe"
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
@@ -44,7 +43,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "standard", "~> 1.0"
   spec.add_development_dependency "simplecov", "~> 0.22"
-  spec.add_development_dependency "openai", "~> 0.34"
   spec.add_development_dependency "vcr", "~> 6.0"
   spec.add_development_dependency "webmock", "~> 3.0"
+  spec.add_development_dependency "appraisal", "~> 2.5"
 end

gemfiles/openai_0.33.gemfile

@@ -0,0 +1,14 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "rake", "~> 13.0"
+gem "minitest", "~> 5.0"
+gem "minitest-reporters", "~> 1.6"
+gem "standard", "~> 1.0"
+gem "simplecov", "~> 0.22"
+gem "vcr", "~> 6.0"
+gem "webmock", "~> 3.0"
+gem "openai", "~> 0.33.0"
+
+gemspec path: "../"

gemfiles/openai_0.34.gemfile

@@ -0,0 +1,14 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "rake", "~> 13.0"
+gem "minitest", "~> 5.0"
+gem "minitest-reporters", "~> 1.6"
+gem "standard", "~> 1.0"
+gem "simplecov", "~> 0.22"
+gem "vcr", "~> 6.0"
+gem "webmock", "~> 3.0"
+gem "openai", "~> 0.34.0"
+
+gemspec path: "../"

gemfiles/openai_latest.gemfile

@@ -0,0 +1,14 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "rake", "~> 13.0"
+gem "minitest", "~> 5.0"
+gem "minitest-reporters", "~> 1.6"
+gem "standard", "~> 1.0"
+gem "simplecov", "~> 0.22"
+gem "vcr", "~> 6.0"
+gem "webmock", "~> 3.0"
+gem "openai", ">= 0.34"
+
+gemspec path: "../"

gemfiles/openai_uninstalled.gemfile

@@ -0,0 +1,13 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "rake", "~> 13.0"
+gem "minitest", "~> 5.0"
+gem "minitest-reporters", "~> 1.6"
+gem "standard", "~> 1.0"
+gem "simplecov", "~> 0.22"
+gem "vcr", "~> 6.0"
+gem "webmock", "~> 3.0"
+
+gemspec path: "../"

lib/braintrust/trace.rb

@@ -3,9 +3,16 @@
 require "opentelemetry/sdk"
 require "opentelemetry/exporter/otlp"
 require_relative "trace/span_processor"
-require_relative "trace/openai"
 require_relative "logger"
 
+# OpenAI integration is optional - automatically loaded if openai gem is available
+begin
+  require "openai"
+  require_relative "trace/contrib/openai"
+rescue LoadError
+  # OpenAI gem not installed - integration will not be available
+end
+
 module Braintrust
   module Trace
     def self.enable(tracer_provider, state: nil, exporter: nil)

mise.toml

@@ -10,22 +10,23 @@ rust = "1.83"
 watchexec = "latest"
 
 [tasks.lint]
-run = "bundle exec standardrb"
+run = "bundle exec rake lint"
 
 [tasks."lint:fix"]
-run = "bundle exec standardrb --fix"
+run = "bundle exec rake lint:fix"
 
 [tasks.watch-test]
 description = "Runs tests when files change"
 run = "watchexec --exts rb --watch lib --watch test --restart --clear -- rake test"
 
-[tasks.verify-fmt]
-run = "bundle exec standardrb --format progress"
+[tasks.precommit]
+description = "Run linter before commit (full CI runs on push)"
+run = "bundle exec rake lint"
 
 [hooks]
 postinstall = """
 echo "Installing gem dependencies..."
 bundle install
 echo "Setting up git pre-commit hook..."
-mise generate git-pre-commit --write --task=verify-fmt
+mise generate git-pre-commit --write --task=precommit
 """

test/braintrust/api/datasets_test.rb

@@ -9,7 +9,7 @@ def setup
   end
 
   def get_test_api
-    state = Braintrust.init(set_global: false, blocking_login: true)
+    state = get_non_global_state
     Braintrust::API.new(state: state)
   end
 

test/braintrust/api/functions_test.rb

@@ -9,7 +9,7 @@ def setup
   end
 
   def get_test_api
-    state = Braintrust.init(set_global: false, blocking_login: true)
+    state = get_non_global_state
     Braintrust::API.new(state: state)
   end
 

test/braintrust/api_test.rb

@@ -9,7 +9,7 @@ def setup
 
   def test_api_new_with_explicit_state
     VCR.use_cassette("api/new_explicit_state") do
-      state = Braintrust.init(set_global: false, blocking_login: true)
+      state = get_non_global_state
 
       api = Braintrust::API.new(state: state)
       assert_equal state, api.state
@@ -41,7 +41,7 @@ def test_api_new_raises_without_state
 
   def test_api_datasets_returns_datasets_instance
     VCR.use_cassette("api/datasets_instance") do
-      state = Braintrust.init(set_global: false, blocking_login: true)
+      state = get_non_global_state
       api = Braintrust::API.new(state: state)
 
       datasets = api.datasets
@@ -51,7 +51,7 @@ def test_api_datasets_returns_datasets_instance
 
   def test_api_datasets_is_memoized
     VCR.use_cassette("api/datasets_memoized") do
-      state = Braintrust.init(set_global: false, blocking_login: true)
+      state = get_non_global_state
       api = Braintrust::API.new(state: state)
 
       datasets1 = api.datasets

test/braintrust/eval/functions_test.rb

@@ -11,7 +11,7 @@ def setup
   end
 
   def get_test_state_and_api
-    state = Braintrust.init(set_global: false, blocking_login: true)
+    state = get_non_global_state
     api = Braintrust::API.new(state: state)
     [state, api]
   end