diff --git a/examples/internal/login_in_thread.rb b/examples/internal/login_in_thread.rb new file mode 100644 index 00000000..8b0e2332 --- /dev/null +++ b/examples/internal/login_in_thread.rb @@ -0,0 +1,39 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "bundler/setup" +require "braintrust" +require "opentelemetry/sdk" + +# Example: Non-blocking login with login_in_thread +# +# This example demonstrates how to: +# 1. Initialize Braintrust without blocking on login (uses login_in_thread internally) +# 2. Do other work while login happens in background +# 3. Create a span after work is done +# 4. Print a permalink to view the trace in Braintrust +# +# Usage: +# bundle exec ruby examples/internal/login_in_thread.rb + +# Initialize Braintrust - this returns immediately and logs in via background thread +Braintrust.init(blocking_login: false) + +puts "Doing work while login completes in background..." +sleep 2 + +# Get a tracer +tracer = OpenTelemetry.tracer_provider.tracer("login-in-thread-example") + +# Create a span +root_span = nil +tracer.in_span("examples/internal/login_in_thread.rb") do |span| + root_span = span + sleep 0.1 +end + +# Print permalink to view this trace in Braintrust +puts "\nView trace: #{Braintrust::Trace.permalink(root_span)}" + +# Shutdown to flush spans to Braintrust +OpenTelemetry.tracer_provider.shutdown diff --git a/lib/braintrust/eval.rb b/lib/braintrust/eval.rb index 4f0ca1d6..fdc9782b 100644 --- a/lib/braintrust/eval.rb +++ b/lib/braintrust/eval.rb @@ -295,7 +295,7 @@ def normalize_scorers(scorers_input) # @param scorers [Array] The scorers # @param errors [Array] Error collection array # @param tracer [Tracer] OpenTelemetry tracer - # @param parent_attr [String] Parent attribute (experiment_id:project/exp_id) + # @param parent_attr [String] Parent attribute (experiment_id:exp_id) def run_case(test_case, task, scorers, errors, tracer, parent_attr) # Create eval span (parent) tracer.in_span("eval") do |eval_span| diff --git a/lib/braintrust/trace.rb b/lib/braintrust/trace.rb index f88eb646..f689da85 100644 --- a/lib/braintrust/trace.rb +++ b/lib/braintrust/trace.rb @@ -94,14 +94,8 @@ def self.permalink(span) # Build the permalink URL based on parent type if parent_type == "experiment_id" - # For experiments: {app_url}/app/{org}/p/{project}/experiments/{experiment_id}?r={trace_id}&s={span_id} - project_name, experiment_id = parent_id.split("/", 2) - unless project_name && experiment_id - Log.error("Invalid experiment parent format: #{parent_id}") - return "" - end - - "#{app_url}/app/#{org_name}/p/#{project_name}/experiments/#{experiment_id}?r=#{trace_id}&s=#{span_id}" + # For experiments: {app_url}/app/{org}/object?object_type=experiment&object_id={experiment_id}&r={trace_id}&s={span_id} + "#{app_url}/app/#{org_name}/object?object_type=experiment&object_id=#{parent_id}&r=#{trace_id}&s=#{span_id}" else # For projects: {app_url}/app/{org}/p/{project}/logs?r={trace_id}&s={span_id} # parent_type is typically "project_name" diff --git a/mise.toml b/mise.toml index cd39c811..1e4dadbe 100644 --- a/mise.toml +++ b/mise.toml @@ -15,6 +15,9 @@ run = "bundle exec rake lint" [tasks."lint:fix"] run = "bundle exec rake lint:fix" +[tasks."verify"] +run = "bundle exec rake lint test" + [tasks.watch-test] description = "Runs tests when files change" run = "watchexec --exts rb --watch lib --watch test --restart --clear -- rake test" diff --git a/test/braintrust/api/internal/auth_test.rb b/test/braintrust/api/internal/auth_test.rb new file mode 100644 index 00000000..70b1bb17 --- /dev/null +++ b/test/braintrust/api/internal/auth_test.rb @@ -0,0 +1,238 @@ +# frozen_string_literal: true + +require "test_helper" +require "braintrust/api/internal/auth" + +class Braintrust::API::Internal::AuthTest < Minitest::Test + def test_mask_api_key_with_nil + assert_equal "nil", Braintrust::API::Internal::Auth.mask_api_key(nil) + end + + def test_mask_api_key_short + assert_equal "short", Braintrust::API::Internal::Auth.mask_api_key("short") + end + + def test_mask_api_key_long + assert_equal "12345678...abcd", Braintrust::API::Internal::Auth.mask_api_key("1234567890abcdefghijklmnopqrstuvwxyzabcd") + end + + def test_login_with_test_api_key + result = Braintrust::API::Internal::Auth.login( + api_key: "test-api-key", + app_url: "https://www.braintrust.dev" + ) + + assert_equal "test-org-id", result.org_id + assert_equal "test-org", result.org_name + assert_equal "https://api.ruby-sdk-fixture.com", result.api_url + assert_equal "https://proxy.ruby-sdk-fixture.com", result.proxy_url + end + + def test_login_with_test_api_key_and_org_name + result = Braintrust::API::Internal::Auth.login( + api_key: "test-api-key", + app_url: "https://www.braintrust.dev", + org_name: "custom-org" + ) + + assert_equal "custom-org", result.org_name + end + + def test_login_bad_request + stub = stub_request(:post, "https://www.braintrust.dev/api/apikey/login") + .to_return(status: 400, body: "Invalid request format") + + error = assert_raises(Braintrust::Error) do + Braintrust::API::Internal::Auth.login( + api_key: "real-key", + app_url: "https://www.braintrust.dev" + ) + end + + assert_match(/bad request/i, error.message) + assert_match(/400/, error.message) + ensure + remove_request_stub(stub) + end + + def test_login_client_error + stub = stub_request(:post, "https://www.braintrust.dev/api/apikey/login") + .to_return(status: 404, body: "", headers: {}) + + error = assert_raises(Braintrust::Error) do + Braintrust::API::Internal::Auth.login( + api_key: "real-key", + app_url: "https://www.braintrust.dev" + ) + end + + assert_match(/client error/i, error.message) + assert_match(/404/, error.message) + ensure + remove_request_stub(stub) + end + + def test_login_server_error + stub = stub_request(:post, "https://www.braintrust.dev/api/apikey/login") + .to_return(status: 500, body: "", headers: {}) + + error = assert_raises(Braintrust::Error) do + Braintrust::API::Internal::Auth.login( + api_key: "real-key", + app_url: "https://www.braintrust.dev" + ) + end + + assert_match(/server error/i, error.message) + assert_match(/500/, error.message) + ensure + remove_request_stub(stub) + end + + def test_login_unexpected_response + stub = stub_request(:post, "https://www.braintrust.dev/api/apikey/login") + .to_return(status: 301, body: "", headers: {}) + + error = assert_raises(Braintrust::Error) do + Braintrust::API::Internal::Auth.login( + api_key: "real-key", + app_url: "https://www.braintrust.dev" + ) + end + + assert_match(/unexpected response/i, error.message) + assert_match(/301/, error.message) + ensure + remove_request_stub(stub) + end + + def test_login_no_organizations + stub = stub_request(:post, "https://www.braintrust.dev/api/apikey/login") + .to_return( + status: 200, + body: JSON.generate({org_info: []}), + headers: {"Content-Type" => "application/json"} + ) + + error = assert_raises(Braintrust::Error) do + Braintrust::API::Internal::Auth.login( + api_key: "real-key", + app_url: "https://www.braintrust.dev" + ) + end + + assert_match(/no organizations found/i, error.message) + ensure + remove_request_stub(stub) + end + + def test_login_org_name_not_found + stub = stub_request(:post, "https://www.braintrust.dev/api/apikey/login") + .to_return( + status: 200, + body: JSON.generate({ + org_info: [ + { + id: "org1", + name: "first-org", + api_url: "https://api.braintrust.dev", + proxy_url: "https://api.braintrust.dev" + }, + { + id: "org2", + name: "second-org", + api_url: "https://api.braintrust.dev", + proxy_url: "https://api.braintrust.dev" + } + ] + }), + headers: {"Content-Type" => "application/json"} + ) + + error = assert_raises(Braintrust::Error) do + Braintrust::API::Internal::Auth.login( + api_key: "real-key", + app_url: "https://www.braintrust.dev", + org_name: "nonexistent-org" + ) + end + + assert_match(/organization 'nonexistent-org' not found/i, error.message) + assert_match(/first-org, second-org/, error.message) + ensure + remove_request_stub(stub) + end + + def test_login_selects_first_org_when_no_org_name + stub = stub_request(:post, "https://www.braintrust.dev/api/apikey/login") + .to_return( + status: 200, + body: JSON.generate({ + org_info: [ + { + id: "first-id", + name: "first-org", + api_url: "https://api1.braintrust.dev", + proxy_url: "https://proxy1.braintrust.dev" + }, + { + id: "second-id", + name: "second-org", + api_url: "https://api2.braintrust.dev", + proxy_url: "https://proxy2.braintrust.dev" + } + ] + }), + headers: {"Content-Type" => "application/json"} + ) + + result = Braintrust::API::Internal::Auth.login( + api_key: "real-key", + app_url: "https://www.braintrust.dev" + ) + + assert_equal "first-id", result.org_id + assert_equal "first-org", result.org_name + assert_equal "https://api1.braintrust.dev", result.api_url + assert_equal "https://proxy1.braintrust.dev", result.proxy_url + ensure + remove_request_stub(stub) + end + + def test_login_selects_matching_org_when_org_name_provided + stub = stub_request(:post, "https://www.braintrust.dev/api/apikey/login") + .to_return( + status: 200, + body: JSON.generate({ + org_info: [ + { + id: "first-id", + name: "first-org", + api_url: "https://api1.braintrust.dev", + proxy_url: "https://proxy1.braintrust.dev" + }, + { + id: "second-id", + name: "second-org", + api_url: "https://api2.braintrust.dev", + proxy_url: "https://proxy2.braintrust.dev" + } + ] + }), + headers: {"Content-Type" => "application/json"} + ) + + result = Braintrust::API::Internal::Auth.login( + api_key: "real-key", + app_url: "https://www.braintrust.dev", + org_name: "second-org" + ) + + assert_equal "second-id", result.org_id + assert_equal "second-org", result.org_name + assert_equal "https://api2.braintrust.dev", result.api_url + assert_equal "https://proxy2.braintrust.dev", result.proxy_url + ensure + remove_request_stub(stub) + end +end diff --git a/test/braintrust/eval/scorer_test.rb b/test/braintrust/eval/scorer_test.rb index 602e6904..65074837 100644 --- a/test/braintrust/eval/scorer_test.rb +++ b/test/braintrust/eval/scorer_test.rb @@ -162,4 +162,19 @@ def call(input, expected, output) # Should auto-detect name from object assert_equal "auto_name", scorer.name end + + def test_scorer_with_method_object + # Test Method object name detection (is_a?(Method) branch) + obj = Object.new + def obj.my_scorer(input, expected, output) + (output == expected) ? 1.0 : 0.0 + end + + method_obj = obj.method(:my_scorer) + scorer = Braintrust::Eval::Scorer.new(method_obj) + + assert_equal "my_scorer", scorer.name + assert_equal 1.0, scorer.call("i", "match", "match") + assert_equal 0.0, scorer.call("i", "match", "no_match") + end end diff --git a/test/braintrust/eval_test.rb b/test/braintrust/eval_test.rb index f49c2234..8bacd071 100644 --- a/test/braintrust/eval_test.rb +++ b/test/braintrust/eval_test.rb @@ -353,6 +353,14 @@ def test_eval_run_with_tracing # Verify score span assert score_span.attributes["braintrust.scores"] assert_includes score_span.attributes["braintrust.scores"], "exact" + + # Verify experiment result has permalink in correct format + assert result.permalink.include?("object_type=experiment"), "Result permalink should be experiment URL" + assert result.permalink.include?("object_id="), "Result permalink should have experiment ID" + + # Verify eval span has correct parent for experiment + parent_attr = eval_span.attributes["braintrust.parent"] + assert parent_attr.start_with?("experiment_id:"), "Eval span should have experiment_id parent" end end diff --git a/test/braintrust/trace_test.rb b/test/braintrust/trace_test.rb index ce7ec17d..91451a8d 100644 --- a/test/braintrust/trace_test.rb +++ b/test/braintrust/trace_test.rb @@ -114,7 +114,7 @@ def test_permalink_with_experiment_parent # Experiment parents come from evals, not from default_project otel_span = nil rig.tracer.in_span("test-operation") do |span| - span.set_attribute("braintrust.parent", "experiment_id:test-project/exp-123") + span.set_attribute("braintrust.parent", "experiment_id:exp-123") otel_span = span end @@ -127,46 +127,32 @@ def test_permalink_with_experiment_parent span_id = span_data.hex_span_id # Verify URL format for experiment parent - expected = "https://app.example.com/app/test-org/p/test-project/experiments/exp-123?r=#{trace_id}&s=#{span_id}" + expected = "https://app.example.com/app/test-org/object?object_type=experiment&object_id=exp-123&r=#{trace_id}&s=#{span_id}" assert_equal expected, link end - def test_permalink_with_missing_attributes - # Set up OpenTelemetry WITHOUT Braintrust processor (to test missing attributes) - require "opentelemetry/sdk" - - exporter = OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter.new - tracer_provider = OpenTelemetry::SDK::Trace::TracerProvider.new - - # Add only a simple processor (no Braintrust processor) - span_processor = OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(exporter) - tracer_provider.add_span_processor(span_processor) - - tracer = tracer_provider.tracer("test") - - # Create a span WITHOUT Braintrust attributes - otel_span = nil - tracer.in_span("test-operation") do |span| - otel_span = span - end + def test_permalink_with_nil_span + link = Braintrust::Trace.permalink(nil) + assert_equal "", link + end - # Suppress error logs for this test (we're intentionally testing missing attributes) + def test_permalink_with_invalid_parent_format original_level = Braintrust::Log.logger.level Braintrust::Log.logger.level = Logger::FATAL begin - # Should return empty string for missing attributes instead of raising - link = Braintrust::Trace.permalink(otel_span) + rig = setup_otel_test_rig + + span_invalid_parent = nil + rig.tracer.in_span("test-operation") do |span| + span.set_attribute("braintrust.parent", "invalid-no-colon") + span_invalid_parent = span + end + + link = Braintrust::Trace.permalink(span_invalid_parent) assert_equal "", link ensure - # Restore original log level Braintrust::Log.logger.level = original_level end end - - def test_permalink_with_nil_span - # Should return empty string for nil span instead of raising - link = Braintrust::Trace.permalink(nil) - assert_equal "", link - end end