From 054e3f4b3dfe3b5be03dc3f266c9b6c89c02f80b Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Tue, 23 Sep 2025 14:00:29 -0700 Subject: [PATCH 01/14] chore: use X-Project-Id for Span Exporter --- pom.xml | 2 +- .../judgeval/data/TraceEvaluationRun.java | 12 +++---- .../exporters/JudgmentSpanExporter.java | 32 ++----------------- 3 files changed, 7 insertions(+), 39 deletions(-) diff --git a/pom.xml b/pom.xml index 59c5a10..f658e54 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.judgmentlabs judgeval-java - 0.2.0 + 0.2.1 jar Judgeval Java Java SDK for Judgeval diff --git a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java index 691223c..311b31a 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java @@ -65,12 +65,9 @@ public String getOrganizationId() { @Override public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null || getClass() != obj.getClass()) - return false; - if (!super.equals(obj)) - return false; + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + if (!super.equals(obj)) return false; TraceEvaluationRun other = (TraceEvaluationRun) obj; return Objects.equals(organizationId, other.organizationId); } @@ -96,8 +93,7 @@ public static final class Builder { private String organizationId; private List> traceAndSpanIds; - private Builder() { - } + private Builder() {} private Builder(String projectName, String evalName) { this.projectName = projectName; diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java index 837fa84..b57f07c 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java @@ -1,18 +1,9 @@ package com.judgmentlabs.judgeval.tracer.exporters; import java.util.Collection; -import java.util.List; -import java.util.stream.Collectors; -import com.judgmentlabs.judgeval.tracer.OpenTelemetryKeys; -import com.judgmentlabs.judgeval.utils.Logger; - -import io.opentelemetry.api.common.AttributeKey; -import io.opentelemetry.api.common.Attributes; import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; import io.opentelemetry.sdk.common.CompletableResultCode; -import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.data.DelegatingSpanData; import io.opentelemetry.sdk.trace.data.SpanData; import io.opentelemetry.sdk.trace.export.SpanExporter; @@ -24,7 +15,6 @@ */ public class JudgmentSpanExporter implements SpanExporter { private final SpanExporter delegate; - private final String projectId; /** * Creates a new JudgmentSpanExporter with the specified configuration. @@ -45,8 +35,8 @@ public JudgmentSpanExporter( .setEndpoint(endpoint) .addHeader("Authorization", "Bearer " + apiKey) .addHeader("X-Organization-Id", organizationId) + .addHeader("X-Project-Id", projectId) .build(); - this.projectId = projectId; } /** @@ -60,25 +50,7 @@ public static Builder builder() { @Override public CompletableResultCode export(Collection spans) { - Logger.info( - "JudgmentSpanExporter exporting spans: count=" - + (spans != null ? spans.size() : 0)); - AttributeKey key = - AttributeKey.stringKey(OpenTelemetryKeys.ResourceKeys.JUDGMENT_PROJECT_ID); - Attributes extra = Attributes.of(key, projectId); - Resource added = Resource.create(extra); - List withResource = - spans.stream() - .map( - s -> - new DelegatingSpanData(s) { - @Override - public Resource getResource() { - return s.getResource().merge(added); - } - }) - .collect(Collectors.toList()); - return delegate.export(withResource); + return delegate.export(spans); } @Override From 5d448c33dedb1d4857a12503d8ce8363aa50542c Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Tue, 23 Sep 2025 16:34:16 -0700 Subject: [PATCH 02/14] chore: getScorerConfig instead of toTransport --- .../java/com/judgmentlabs/judgeval/scorers/APIScorer.java | 2 +- .../java/com/judgmentlabs/judgeval/scorers/BaseScorer.java | 2 +- .../scorers/api_scorers/prompt_scorer/PromptScorer.java | 2 +- .../scorers/api_scorers/prompt_scorer/TracePromptScorer.java | 2 +- src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java index 186025e..32e24b6 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java @@ -42,7 +42,7 @@ public void setRequiredParams(List requiredParams) { } @Override - public ScorerConfig toTransport() { + public ScorerConfig getScorerConfig() { ScorerConfig cfg = new ScorerConfig(); cfg.setScoreType(getScoreType()); cfg.setThreshold(getThreshold()); diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java index fc26dd2..daac522 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java @@ -19,7 +19,7 @@ public BaseScorer() { } } - public abstract ScorerConfig toTransport(); + public abstract ScorerConfig getScorerConfig(); public void addModel(String model) { setModel(model); diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java index 1b74b12..f555641 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java @@ -108,7 +108,7 @@ public static PromptScorer create( } @Override - public ScorerConfig toTransport() { + public ScorerConfig getScorerConfig() { ScorerConfig cfg = new ScorerConfig(); cfg.setScoreType(getScoreType()); cfg.setThreshold(getThreshold()); diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java index 4439fc8..ac2c5f9 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java @@ -110,7 +110,7 @@ public static TracePromptScorer create( } @Override - public ScorerConfig toTransport() { + public ScorerConfig getScorerConfig() { ScorerConfig cfg = new ScorerConfig(); cfg.setScoreType(getScoreType()); cfg.setThreshold(getThreshold()); diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java index 7b305fd..4c0e8eb 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java @@ -404,7 +404,7 @@ private ExampleEvaluationRun createEvaluationRun( configuration.projectName(), runId, List.of(example), - List.of(scorer.toTransport()), + List.of(scorer.getScorerConfig()), modelName, configuration.organizationId()); evaluationRun.setTraceId(traceId); @@ -421,7 +421,7 @@ private TraceEvaluationRun createTraceEvaluationRun( return TraceEvaluationRun.builder() .projectName(configuration.projectName()) .evalName(evalName) - .scorer(scorer.toTransport()) + .scorer(scorer.getScorerConfig()) .model(modelName) .organizationId(configuration.organizationId()) .traceAndSpanId(traceId, spanId) From 23d58e7d05bac1869a2011b715557b0d911f2785 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Sat, 27 Sep 2025 12:24:43 -0700 Subject: [PATCH 03/14] feat: wrappers and initializaion --- .../java/com/judgmentlabs/judgeval/Env.java | 1 + .../judgeval/tracer/BaseTracer.java | 410 +++++++++++++++ .../judgmentlabs/judgeval/tracer/Tracer.java | 485 ++---------------- .../judgmentlabs/judgeval/utils/Logger.java | 35 +- 4 files changed, 473 insertions(+), 458 deletions(-) create mode 100644 src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java diff --git a/src/main/java/com/judgmentlabs/judgeval/Env.java b/src/main/java/com/judgmentlabs/judgeval/Env.java index 16994af..e2a07e8 100644 --- a/src/main/java/com/judgmentlabs/judgeval/Env.java +++ b/src/main/java/com/judgmentlabs/judgeval/Env.java @@ -10,6 +10,7 @@ private Env() {} public static final String JUDGMENT_DEFAULT_GPT_MODEL = getEnvVar("JUDGMENT_DEFAULT_GPT_MODEL", "gpt-4.1"); public static final String JUDGMENT_NO_COLOR = getEnvVar("JUDGMENT_NO_COLOR"); + public static final String JUDGMENT_LOG_LEVEL = getEnvVar("JUDGMENT_LOG_LEVEL", "warn"); private static String getEnvVar(String varName) { return getEnvVar(varName, null); diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java new file mode 100644 index 0000000..373b666 --- /dev/null +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java @@ -0,0 +1,410 @@ +package com.judgmentlabs.judgeval.tracer; + +import java.lang.reflect.Type; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.Callable; +import java.util.function.Function; +import java.util.function.Supplier; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import com.judgmentlabs.judgeval.Env; +import com.judgmentlabs.judgeval.data.Example; +import com.judgmentlabs.judgeval.data.ExampleEvaluationRun; +import com.judgmentlabs.judgeval.data.TraceEvaluationRun; +import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; +import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameRequest; +import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameResponse; +import com.judgmentlabs.judgeval.scorers.BaseScorer; +import com.judgmentlabs.judgeval.tracer.exporters.JudgmentSpanExporter; +import com.judgmentlabs.judgeval.tracer.exporters.NoOpSpanExporter; +import com.judgmentlabs.judgeval.utils.Logger; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.trace.export.SpanExporter; + +/** Base tracer providing core functionality for distributed tracing and evaluation. */ +public abstract class BaseTracer { + protected final TracerConfiguration configuration; + protected final JudgmentSyncClient apiClient; + protected final ISerializer serializer; + protected final ObjectMapper jacksonMapper; + protected final String projectId; + + protected BaseTracer( + TracerConfiguration configuration, ISerializer serializer, boolean initialize) { + this.configuration = Objects.requireNonNull(configuration, "Configuration cannot be null"); + this.apiClient = + new JudgmentSyncClient( + configuration.apiUrl(), + configuration.apiKey(), + configuration.organizationId()); + this.serializer = Objects.requireNonNull(serializer, "Serializer cannot be null"); + this.jacksonMapper = new ObjectMapper(); + this.projectId = resolveProjectId(configuration.projectName()); + if (this.projectId == null) { + Logger.error( + "Failed to resolve project " + + configuration.projectName() + + ", please create it first at https://app.judgmentlabs.ai/org/" + + configuration.organizationId() + + "/projects. Skipping Judgment export."); + } + + if (initialize) { + initialize(); + } + } + + /** Initializes the OpenTelemetry SDK with span processors and exporters. */ + public abstract void initialize(); + + /** + * Gets the SpanExporter for OpenTelemetry integration. Returns NoOpSpanExporter if project ID + * cannot be resolved. + */ + public SpanExporter getSpanExporter() { + if (projectId == null) { + Logger.error( + "Project not resolved; cannot create exporter, returning NoOpSpanExporter"); + return new NoOpSpanExporter(); + } + return createJudgmentSpanExporter(projectId); + } + + /** Sets the span kind attribute on the current span. Common kinds: "span", "llm", "tool". */ + public void setSpanKind(String kind) { + Optional.ofNullable(Span.current()) + .filter(span -> kind != null) + .ifPresent( + span -> + span.setAttribute( + OpenTelemetryKeys.AttributeKeys.JUDGMENT_SPAN_KIND, kind)); + } + + /** Sets a custom attribute on the current span. Value is serialized to JSON. */ + public void setAttribute(String key, Object value) { + Optional.ofNullable(Span.current()) + .ifPresent(span -> span.setAttribute(key, serializer.serialize(value))); + } + + /** Sets a custom attribute on the current span with specific type for JSON serialization. */ + public void setAttribute(String key, Object value, Type type) { + Optional.ofNullable(Span.current()) + .ifPresent(span -> span.setAttribute(key, serializer.serialize(value, type))); + } + + /** Asynchronously evaluates a scorer with an example using current trace context. */ + public void asyncEvaluate(BaseScorer scorer, Example example, String model) { + try { + if (!configuration.enableEvaluation()) { + return; + } + + Span currentSpan = Span.current(); + if (currentSpan == null || !currentSpan.getSpanContext().isSampled()) { + return; + } + + SpanContext spanContext = currentSpan.getSpanContext(); + String traceId = spanContext.getTraceId(); + String spanId = spanContext.getSpanId(); + + Logger.info( + "asyncEvaluate: project=" + + configuration.projectName() + + ", traceId=" + + traceId + + ", spanId=" + + spanId + + ", scorer=" + + scorer.getName()); + + ExampleEvaluationRun evaluationRun = + createEvaluationRun(scorer, example, model, traceId, spanId); + enqueueEvaluation(evaluationRun); + } catch (Exception e) { + Logger.error("Failed to evaluate scorer: " + e.getMessage()); + } + } + + public void asyncEvaluate(BaseScorer scorer, Example example) { + asyncEvaluate(scorer, example, null); + } + + /** Asynchronously evaluates a scorer with trace context. Sets evaluation as span attribute. */ + public void asyncTraceEvaluate(BaseScorer scorer, String model) { + try { + if (!configuration.enableEvaluation()) { + return; + } + + Span currentSpan = Span.current(); + if (currentSpan == null || !currentSpan.getSpanContext().isSampled()) { + return; + } + + SpanContext spanContext = currentSpan.getSpanContext(); + String traceId = spanContext.getTraceId(); + String spanId = spanContext.getSpanId(); + + Logger.info( + "asyncTraceEvaluate: project=" + + configuration.projectName() + + ", traceId=" + + traceId + + ", spanId=" + + spanId + + ", scorer=" + + scorer.getName()); + + TraceEvaluationRun evaluationRun = + createTraceEvaluationRun(scorer, model, traceId, spanId); + try { + String traceEvalJson = jacksonMapper.writeValueAsString(evaluationRun); + currentSpan.setAttribute( + OpenTelemetryKeys.AttributeKeys.PENDING_TRACE_EVAL, traceEvalJson); + } catch (Exception e) { + Logger.error("Failed to serialize trace evaluation: " + e.getMessage()); + } + } catch (Exception e) { + Logger.error("Failed to evaluate trace scorer: " + e.getMessage()); + } + } + + public void asyncTraceEvaluate(BaseScorer scorer) { + asyncTraceEvaluate(scorer, null); + } + + public void setAttributes(Map attributes) { + if (attributes == null) { + return; + } + Optional.ofNullable(Span.current()) + .ifPresent( + span -> + attributes.forEach( + (key, value) -> + span.setAttribute( + key, serializer.serialize(value)))); + } + + public void setLLMSpan() { + setSpanKind("llm"); + } + + public void setToolSpan() { + setSpanKind("tool"); + } + + public void setGeneralSpan() { + setSpanKind("span"); + } + + public void setInput(Object input) { + setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_INPUT, input); + } + + public void setOutput(Object output) { + setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_OUTPUT, output); + } + + public void setInput(Object input, Type type) { + setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_INPUT, input, type); + } + + public void setOutput(Object output, Type type) { + setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_OUTPUT, output, type); + } + + /** Wraps a Callable with an OpenTelemetry span for observability. */ + public Callable observe(Callable callable, String spanName) { + Objects.requireNonNull(callable, "Callable cannot be null"); + String finalSpanName = spanName != null ? spanName : "span"; + + return () -> { + io.opentelemetry.api.trace.Tracer otelTracer = + GlobalOpenTelemetry.get().getTracer("judgeval"); + Span span = otelTracer.spanBuilder(finalSpanName).startSpan(); + + try (Scope scope = span.makeCurrent()) { + T result = callable.call(); + + if (result != null) { + setOutput(result); + } + + span.setStatus(StatusCode.OK); + return result; + } catch (Throwable throwable) { + span.recordException(throwable); + span.setStatus(StatusCode.ERROR, "Exception occurred: " + throwable.getMessage()); + throw throwable; + } finally { + span.end(); + } + }; + } + + /** Wraps a Callable with an OpenTelemetry span using default span name "span". */ + public Callable observe(Callable callable) { + return observe(callable, "span"); + } + + /** Wraps a Function with an OpenTelemetry span with automatic input/output capture. */ + public Function observe(Function function, String spanName) { + Objects.requireNonNull(function, "Function cannot be null"); + String finalSpanName = spanName != null ? spanName : "span"; + + return input -> { + io.opentelemetry.api.trace.Tracer otelTracer = + GlobalOpenTelemetry.get().getTracer("com.judgmentlabs.judgeval"); + Span span = otelTracer.spanBuilder(finalSpanName).startSpan(); + + try (Scope scope = span.makeCurrent()) { + setInput(input); + + R result = function.apply(input); + + if (result != null) { + setOutput(result); + } + + span.setStatus(StatusCode.OK); + return result; + } catch (Throwable throwable) { + span.recordException(throwable); + span.setStatus(StatusCode.ERROR, "Exception occurred: " + throwable.getMessage()); + throw throwable; + } finally { + span.end(); + } + }; + } + + /** Wraps a Function with an OpenTelemetry span using default span name "span". */ + public Function observe(Function function) { + return observe(function, "span"); + } + + /** Wraps a Supplier with an OpenTelemetry span for observability. */ + public Supplier observe(Supplier supplier, String spanName) { + Objects.requireNonNull(supplier, "Supplier cannot be null"); + String finalSpanName = spanName != null ? spanName : "span"; + + return () -> { + io.opentelemetry.api.trace.Tracer otelTracer = + GlobalOpenTelemetry.get().getTracer("com.judgmentlabs.judgeval"); + Span span = otelTracer.spanBuilder(finalSpanName).startSpan(); + + try (Scope scope = span.makeCurrent()) { + T result = supplier.get(); + + if (result != null) { + setOutput(result); + } + + span.setStatus(StatusCode.OK); + return result; + } catch (Throwable throwable) { + span.recordException(throwable); + span.setStatus(StatusCode.ERROR, "Exception occurred: " + throwable.getMessage()); + throw throwable; + } finally { + span.end(); + } + }; + } + + /** Wraps a Supplier with an OpenTelemetry span using default span name "span". */ + public Supplier observe(Supplier supplier) { + return observe(supplier, "span"); + } + + private String resolveProjectId(String name) { + try { + ResolveProjectNameRequest request = new ResolveProjectNameRequest(); + request.setProjectName(name); + ResolveProjectNameResponse response = apiClient.projectsResolve(request); + return Optional.ofNullable(response.getProjectId()).map(Object::toString).orElse(null); + } catch (Exception e) { + return null; + } + } + + private JudgmentSpanExporter createJudgmentSpanExporter(String projectId) { + String endpoint = + configuration.apiUrl().endsWith("/") + ? configuration.apiUrl() + "otel/v1/traces" + : configuration.apiUrl() + "/otel/v1/traces"; + return JudgmentSpanExporter.builder() + .endpoint(endpoint) + .apiKey(configuration.apiKey()) + .organizationId(configuration.organizationId()) + .projectId(projectId) + .build(); + } + + private ExampleEvaluationRun createEvaluationRun( + BaseScorer scorer, Example example, String model, String traceId, String spanId) { + String runId = "async_evaluate_" + (spanId != null ? spanId : System.currentTimeMillis()); + String modelName = model != null ? model : Env.JUDGMENT_DEFAULT_GPT_MODEL; + + ExampleEvaluationRun evaluationRun = + new ExampleEvaluationRun( + configuration.projectName(), + runId, + List.of(example), + List.of(scorer.getScorerConfig()), + modelName, + configuration.organizationId()); + evaluationRun.setTraceId(traceId); + evaluationRun.setTraceSpanId(spanId); + return evaluationRun; + } + + private TraceEvaluationRun createTraceEvaluationRun( + BaseScorer scorer, String model, String traceId, String spanId) { + String evalName = + "async_trace_evaluate_" + (spanId != null ? spanId : System.currentTimeMillis()); + String modelName = model != null ? model : Env.JUDGMENT_DEFAULT_GPT_MODEL; + + return TraceEvaluationRun.builder() + .projectName(configuration.projectName()) + .evalName(evalName) + .scorer(scorer.getScorerConfig()) + .model(modelName) + .organizationId(configuration.organizationId()) + .traceAndSpanId(traceId, spanId) + .build(); + } + + private void enqueueEvaluation(ExampleEvaluationRun evaluationRun) { + try { + apiClient.addToRunEvalQueue(evaluationRun); + } catch (Exception e) { + Logger.error("Failed to enqueue evaluation run: " + e.getMessage()); + } + } + + protected static class GsonSerializer implements ISerializer { + private final Gson gson = new Gson(); + + @Override + public String serialize(Object obj) { + return gson.toJson(obj); + } + + @Override + public String serialize(Object obj, Type type) { + return gson.toJson(obj, type); + } + } +} diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java index 4c0e8eb..21bff77 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java @@ -1,122 +1,25 @@ package com.judgmentlabs.judgeval.tracer; -import java.lang.reflect.Type; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; -import com.judgmentlabs.judgeval.Env; -import com.judgmentlabs.judgeval.data.Example; -import com.judgmentlabs.judgeval.data.ExampleEvaluationRun; -import com.judgmentlabs.judgeval.data.TraceEvaluationRun; -import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; -import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameRequest; -import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameResponse; -import com.judgmentlabs.judgeval.scorers.BaseScorer; -import com.judgmentlabs.judgeval.tracer.exporters.JudgmentSpanExporter; -import com.judgmentlabs.judgeval.tracer.exporters.NoOpSpanExporter; -import com.judgmentlabs.judgeval.utils.Logger; - -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; import io.opentelemetry.sdk.trace.export.SpanExporter; /** - * Main tracer class for integrating with Judgment Labs for distributed tracing and evaluation. - * - *

The Tracer class provides functionality to: - * - *

    - *
  • Export OpenTelemetry spans to Judgment Labs - *
  • Automatically evaluate spans using configured scorers - *
  • Set custom attributes on spans for better observability - *
  • Integrate with existing OpenTelemetry instrumentation - *
- * - *

Basic Usage

- * - * 
{@code
- * // Simple usage with defaults
- * Tracer tracer = Tracer.createDefault("my-project");
- *
- * // Get the span exporter for OpenTelemetry integration
- * SpanExporter exporter = tracer.getSpanExporter();
- *
- * // Set custom attributes on the current span
- * tracer.setAttribute("custom.key", "custom.value");
- *
- * // Evaluate a scorer with an example
- * Example example = new Example();
- * example.setAdditionalProperty("input", "user input");
- * example.setAdditionalProperty("output", "model output");
- *
- * BaseScorer scorer = new AnswerRelevancyScorer();
- * tracer.asyncEvaluate(scorer, example, "gpt-4");
- *
- * // Evaluate a scorer with trace context
- * tracer.asyncTraceEvaluate(scorer, "gpt-4");
- * }
- * - *

Advanced Configuration

- * - * 
{@code
- * TracerConfiguration config = TracerConfiguration.builder()
- *         .projectName("my-project")
- *         .apiKey("custom-api-key")
- *         .organizationId("custom-org-id")
- *         .enableEvaluation(false)
- *         .build();
- *
- * Tracer tracer = Tracer.createWithConfiguration(config);
- * }
- * - *

OpenTelemetry Integration

- * - * 
{@code
- * Tracer tracer = Tracer.createDefault("my-project");
- * SpanExporter exporter = tracer.getSpanExporter();
- *
- * SdkTracerProvider provider = SdkTracerProvider.builder()
- *         .addSpanProcessor(BatchSpanProcessor.builder(exporter).build())
- *         .build();
- *
- * OpenTelemetrySdk.builder()
- *         .setTracerProvider(provider)
- *         .buildAndRegisterGlobal();
- * }
+ * Main tracer for Judgment Labs distributed tracing and evaluation. * * @see TracerConfiguration * @see SpanExporter - * @see BaseScorer - * @see Example + * @see com.judgmentlabs.judgeval.scorers.BaseScorer + * @see com.judgmentlabs.judgeval.data.Example */ -public final class Tracer { - private final TracerConfiguration configuration; - private final JudgmentSyncClient apiClient; - private final ISerializer serializer; - private final ObjectMapper jacksonMapper; - private final String projectId; +public final class Tracer extends BaseTracer { private Tracer( - TracerConfiguration configuration, - JudgmentSyncClient apiClient, - ISerializer serializer) { - this.configuration = Objects.requireNonNull(configuration, "Configuration cannot be null"); - this.apiClient = Objects.requireNonNull(apiClient, "API client cannot be null"); - this.serializer = Objects.requireNonNull(serializer, "Serializer cannot be null"); - this.jacksonMapper = new ObjectMapper(); - this.projectId = resolveProjectId(configuration.projectName()); - if (this.projectId == null) { - Logger.error( - "Failed to resolve project " - + configuration.projectName() - + ", please create it first at https://app.judgmentlabs.ai/org/" - + configuration.organizationId() - + "/projects. Skipping Judgment export."); - } + TracerConfiguration configuration, ISerializer serializer, boolean shouldInitialize) { + super(configuration, serializer, shouldInitialize); } public static TracerBuilder builder() { @@ -131,348 +34,40 @@ public static Tracer createWithConfiguration(TracerConfiguration configuration) return builder().configuration(configuration).build(); } - /** - * Gets the SpanExporter for integration with OpenTelemetry. - * - *

If the project ID cannot be resolved (e.g., the project doesn't exist), this method - * returns a NoOpSpanExporter. - * - *

Example usage: - * - * 

{@code
-     * Tracer tracer = Tracer.createDefault("my-project");
-     * SpanExporter exporter = tracer.getSpanExporter();
-     *
-     * SdkTracerProvider provider = SdkTracerProvider.builder()
-     *         .addSpanProcessor(BatchSpanProcessor.builder(exporter).build())
-     *         .build();
-     * }
- * - * @return a SpanExporter that sends spans to Judgment Labs, or a NoOpSpanExporter if project - * resolution fails - */ - public SpanExporter getSpanExporter() { - if (projectId == null) { - Logger.error( - "Project not resolved; cannot create exporter, returning NoOpSpanExporter"); - return new NoOpSpanExporter(); - } - return createJudgmentSpanExporter(projectId); - } - - /** - * Sets the span kind attribute on the current span. - * - *

Common span kinds include: - * - *

    - *
  • "span" - General application spans - *
  • "llm" - Language model interactions - *
  • "tool" - External tool or API calls - *
- * - * @param kind the span kind to set (if null, no attribute is set) - */ - public void setSpanKind(String kind) { - Optional.ofNullable(Span.current()) - .filter(span -> kind != null) - .ifPresent( - span -> - span.setAttribute( - OpenTelemetryKeys.AttributeKeys.JUDGMENT_SPAN_KIND, kind)); - } - - /** - * Sets a custom attribute on the current span. - * - *

This method serializes the value to JSON and sets it as a string attribute on the current - * span. This is useful for adding custom metadata to spans for better observability. - * - *

If there is no current span, this method does nothing. - * - * @param key the attribute key (must not be null) - * @param value the attribute value (will be serialized to JSON) - */ - public void setAttribute(String key, Object value) { - Optional.ofNullable(Span.current()) - .ifPresent(span -> span.setAttribute(key, serializer.serialize(value))); - } - - /** - * Sets a custom attribute on the current span with a specific type. - * - *

This method is similar to {@link #setAttribute(String, Object)} but allows you to specify - * the type for JSON serialization. This is useful when you need to preserve type information or - * when dealing with generic types. - * - *

If there is no current span, this method does nothing. - * - * @param key the attribute key (must not be null) - * @param value the attribute value (will be serialized to JSON) - * @param type the type to use for JSON serialization (must not be null) - */ - public void setAttribute(String key, Object value, Type type) { - Optional.ofNullable(Span.current()) - .ifPresent(span -> span.setAttribute(key, serializer.serialize(value, type))); - } - - /** - * Asynchronously evaluates a scorer with an example. - * - *

The evaluation includes: - * - *

    - *
  • The current trace and span context for correlation - *
  • The scorer configuration and threshold - *
  • The example input and output data - *
  • The model used for generation - *
- * - *

If evaluation is disabled in the configuration, this method does nothing. The method - * respects OpenTelemetry's internal sampler - evaluation only runs if the current span is - * recording. - * - * @param scorer the scorer to use for evaluation (must not be null) - * @param example the example to evaluate (must not be null) - * @param model the model used for generation (can be null, will use default) - */ - public void asyncEvaluate(BaseScorer scorer, Example example, String model) { - try { - if (!configuration.enableEvaluation()) { - return; - } - - Span currentSpan = Span.current(); - if (currentSpan == null || !currentSpan.getSpanContext().isSampled()) { - return; - } - - SpanContext spanContext = currentSpan.getSpanContext(); - String traceId = spanContext.getTraceId(); - String spanId = spanContext.getSpanId(); - - Logger.info( - "asyncEvaluate: project=" - + configuration.projectName() - + ", traceId=" - + traceId - + ", spanId=" - + spanId - + ", scorer=" - + scorer.getName()); - - ExampleEvaluationRun evaluationRun = - createEvaluationRun(scorer, example, model, traceId, spanId); - enqueueEvaluation(evaluationRun); - } catch (Exception e) { - Logger.error("Failed to evaluate scorer: " + e.getMessage()); - } - } - - public void asyncEvaluate(BaseScorer scorer, Example example) { - asyncEvaluate(scorer, example, null); - } - - /** - * Asynchronously evaluates a scorer with trace and span context. - * - *

This method creates a trace evaluation run and sets it as an attribute on the current - * span. The evaluation will be processed when the span is exported. - * - *

If evaluation is disabled in the configuration, this method does nothing. The method - * respects OpenTelemetry's internal sampler - evaluation only runs if the current span is - * recording. - * - * @param scorer the scorer to use for evaluation (must not be null) - * @param model the model used for generation (can be null, will use default) - */ - public void asyncTraceEvaluate(BaseScorer scorer, String model) { - try { - if (!configuration.enableEvaluation()) { - return; - } - - Span currentSpan = Span.current(); - if (currentSpan == null || !currentSpan.getSpanContext().isSampled()) { - return; - } - - SpanContext spanContext = currentSpan.getSpanContext(); - String traceId = spanContext.getTraceId(); - String spanId = spanContext.getSpanId(); - - Logger.info( - "asyncTraceEvaluate: project=" - + configuration.projectName() - + ", traceId=" - + traceId - + ", spanId=" - + spanId - + ", scorer=" - + scorer.getName()); - - TraceEvaluationRun evaluationRun = - createTraceEvaluationRun(scorer, model, traceId, spanId); - try { - String traceEvalJson = jacksonMapper.writeValueAsString(evaluationRun); - currentSpan.setAttribute( - OpenTelemetryKeys.AttributeKeys.PENDING_TRACE_EVAL, traceEvalJson); - } catch (Exception e) { - Logger.error("Failed to serialize trace evaluation: " + e.getMessage()); - } - } catch (Exception e) { - Logger.error("Failed to evaluate trace scorer: " + e.getMessage()); - } - } - - public void asyncTraceEvaluate(BaseScorer scorer) { - asyncTraceEvaluate(scorer, null); - } - - public void setAttributes(Map attributes) { - if (attributes == null) { - return; - } - Optional.ofNullable(Span.current()) - .ifPresent( - span -> - attributes.forEach( - (key, value) -> - span.setAttribute( - key, serializer.serialize(value)))); - } - - public void setLLMSpan() { - setSpanKind("llm"); - } - - public void setToolSpan() { - setSpanKind("tool"); - } - - public void setGeneralSpan() { - setSpanKind("span"); - } - - public void setInput(Object input) { - setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_INPUT, input); - } - - public void setOutput(Object output) { - setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_OUTPUT, output); - } - - public void setInput(Object input, Type type) { - setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_INPUT, input, type); - } - - public void setOutput(Object output, Type type) { - setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_OUTPUT, output, type); - } - - private String resolveProjectId(String name) { - try { - ResolveProjectNameRequest request = new ResolveProjectNameRequest(); - request.setProjectName(name); - ResolveProjectNameResponse response = apiClient.projectsResolve(request); - return Optional.ofNullable(response.getProjectId()).map(Object::toString).orElse(null); - } catch (Exception e) { - return null; - } - } + /** Initializes the OpenTelemetry SDK with batch span processor and registers it globally. */ + @Override + public void initialize() { + SpanExporter spanExporter = getSpanExporter(); - private JudgmentSpanExporter createJudgmentSpanExporter(String projectId) { - String endpoint = - configuration.apiUrl().endsWith("/") - ? configuration.apiUrl() + "otel/v1/traces" - : configuration.apiUrl() + "/otel/v1/traces"; - return JudgmentSpanExporter.builder() - .endpoint(endpoint) - .apiKey(configuration.apiKey()) - .organizationId(configuration.organizationId()) - .projectId(projectId) - .build(); - } + SdkTracerProvider tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build()) + .build(); - private ExampleEvaluationRun createEvaluationRun( - BaseScorer scorer, Example example, String model, String traceId, String spanId) { - String runId = "async_evaluate_" + (spanId != null ? spanId : System.currentTimeMillis()); - String modelName = model != null ? model : Env.JUDGMENT_DEFAULT_GPT_MODEL; + OpenTelemetry openTelemetry = + OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build(); - ExampleEvaluationRun evaluationRun = - new ExampleEvaluationRun( - configuration.projectName(), - runId, - List.of(example), - List.of(scorer.getScorerConfig()), - modelName, - configuration.organizationId()); - evaluationRun.setTraceId(traceId); - evaluationRun.setTraceSpanId(spanId); - return evaluationRun; - } - - private TraceEvaluationRun createTraceEvaluationRun( - BaseScorer scorer, String model, String traceId, String spanId) { - String evalName = - "async_trace_evaluate_" + (spanId != null ? spanId : System.currentTimeMillis()); - String modelName = model != null ? model : Env.JUDGMENT_DEFAULT_GPT_MODEL; - - return TraceEvaluationRun.builder() - .projectName(configuration.projectName()) - .evalName(evalName) - .scorer(scorer.getScorerConfig()) - .model(modelName) - .organizationId(configuration.organizationId()) - .traceAndSpanId(traceId, spanId) - .build(); - } - - private void enqueueEvaluation(ExampleEvaluationRun evaluationRun) { - try { - apiClient.addToRunEvalQueue(evaluationRun); - } catch (Exception e) { - Logger.error("Failed to enqueue evaluation run: " + e.getMessage()); - } + GlobalOpenTelemetry.set(openTelemetry); } - /** - * Builder for creating Tracer instances with custom dependencies. - * - *

Example usage: - * - * 

{@code
-     * TracerConfiguration config = TracerConfiguration.builder()
-     *         .projectName("my-project")
-     *         .build();
-     *
-     * JudgmentSyncClient customClient = new JudgmentSyncClient(url, key, orgId);
-     * ISerializer customSerializer = obj -> obj.toString();
-     *
-     * Tracer tracer = Tracer.builder()
-     *         .configuration(config)
-     *         .apiClient(customClient)
-     *         .serializer(customSerializer)
-     *         .build();
-     * }
- */ + /** Builder for creating Tracer instances. */ public static final class TracerBuilder { private TracerConfiguration configuration; - private JudgmentSyncClient apiClient; private ISerializer serializer = new GsonSerializer(); + private boolean initialize = false; public TracerBuilder configuration(TracerConfiguration configuration) { this.configuration = configuration; return this; } - public TracerBuilder apiClient(JudgmentSyncClient apiClient) { - this.apiClient = apiClient; + public TracerBuilder serializer(ISerializer serializer) { + this.serializer = serializer; return this; } - public TracerBuilder serializer(ISerializer serializer) { - this.serializer = serializer; + public TracerBuilder initialize(boolean initialize) { + this.initialize = initialize; return this; } @@ -481,29 +76,7 @@ public Tracer build() { throw new IllegalArgumentException("Configuration is required"); } - JudgmentSyncClient client = - apiClient != null - ? apiClient - : new JudgmentSyncClient( - configuration.apiUrl(), - configuration.apiKey(), - configuration.organizationId()); - - return new Tracer(configuration, client, serializer); - } - } - - private static class GsonSerializer implements ISerializer { - private final Gson gson = new Gson(); - - @Override - public String serialize(Object obj) { - return gson.toJson(obj); - } - - @Override - public String serialize(Object obj, Type type) { - return gson.toJson(obj, type); + return new Tracer(configuration, serializer, initialize); } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java b/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java index 00417cf..b711dd1 100644 --- a/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java +++ b/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java @@ -5,6 +5,8 @@ import java.time.format.DateTimeFormatter; import java.util.concurrent.atomic.AtomicBoolean; +import com.judgmentlabs.judgeval.Env; + public class Logger { private static final String RESET = "\033[0m"; private static final String RED = "\033[31m"; @@ -39,14 +41,43 @@ public String getColor() { } private static final AtomicBoolean initialized = new AtomicBoolean(false); - private static Level currentLevel = Level.INFO; + private static Level currentLevel = Level.WARNING; private static boolean useColor = true; private static PrintStream output = System.out; private static void initialize() { if (initialized.compareAndSet(false, true)) { - String noColor = System.getenv("JUDGMENT_NO_COLOR"); + String noColor = Env.JUDGMENT_NO_COLOR; useColor = noColor == null && System.console() != null; + + String logLevel = Env.JUDGMENT_LOG_LEVEL; + if (logLevel != null) { + loadLogLevel(logLevel.toLowerCase()); + } + } + } + + private static void loadLogLevel(String logLevel) { + switch (logLevel) { + case "debug": + currentLevel = Level.DEBUG; + break; + case "info": + currentLevel = Level.INFO; + break; + case "warning": + case "warn": + currentLevel = Level.WARNING; + break; + case "error": + currentLevel = Level.ERROR; + break; + case "critical": + currentLevel = Level.CRITICAL; + break; + default: + currentLevel = Level.WARNING; + break; } } From aba8521855e0bc999eb0dfa2419a38ad7d9d2263 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Sun, 28 Sep 2025 15:36:02 -0700 Subject: [PATCH 04/14] tracer utils --- .../judgeval/tracer/BaseTracer.java | 118 ++++-------------- 1 file changed, 22 insertions(+), 96 deletions(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java index 373b666..0c9b599 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java @@ -5,9 +5,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.concurrent.Callable; -import java.util.function.Function; -import java.util.function.Supplier; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; @@ -25,13 +22,16 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.SpanContext; -import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Scope; import io.opentelemetry.sdk.trace.export.SpanExporter; /** Base tracer providing core functionality for distributed tracing and evaluation. */ public abstract class BaseTracer { + public static final String TRACER_NAME = "judgeval"; + protected final TracerConfiguration configuration; protected final JudgmentSyncClient apiClient; protected final ISerializer serializer; @@ -224,108 +224,34 @@ public void setOutput(Object output, Type type) { setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_OUTPUT, output, type); } - /** Wraps a Callable with an OpenTelemetry span for observability. */ - public Callable observe(Callable callable, String spanName) { - Objects.requireNonNull(callable, "Callable cannot be null"); - String finalSpanName = spanName != null ? spanName : "span"; - - return () -> { - io.opentelemetry.api.trace.Tracer otelTracer = - GlobalOpenTelemetry.get().getTracer("judgeval"); - Span span = otelTracer.spanBuilder(finalSpanName).startSpan(); - - try (Scope scope = span.makeCurrent()) { - T result = callable.call(); - - if (result != null) { - setOutput(result); - } - - span.setStatus(StatusCode.OK); - return result; - } catch (Throwable throwable) { - span.recordException(throwable); - span.setStatus(StatusCode.ERROR, "Exception occurred: " + throwable.getMessage()); - throw throwable; - } finally { - span.end(); - } - }; + public Tracer getTracer() { + return GlobalOpenTelemetry.get().getTracer(TRACER_NAME); } - /** Wraps a Callable with an OpenTelemetry span using default span name "span". */ - public Callable observe(Callable callable) { - return observe(callable, "span"); + public Span getCurrentSpan() { + return Span.current(); } - /** Wraps a Function with an OpenTelemetry span with automatic input/output capture. */ - public Function observe(Function function, String spanName) { - Objects.requireNonNull(function, "Function cannot be null"); - String finalSpanName = spanName != null ? spanName : "span"; - - return input -> { - io.opentelemetry.api.trace.Tracer otelTracer = - GlobalOpenTelemetry.get().getTracer("com.judgmentlabs.judgeval"); - Span span = otelTracer.spanBuilder(finalSpanName).startSpan(); - - try (Scope scope = span.makeCurrent()) { - setInput(input); - - R result = function.apply(input); - - if (result != null) { - setOutput(result); - } - - span.setStatus(StatusCode.OK); - return result; - } catch (Throwable throwable) { - span.recordException(throwable); - span.setStatus(StatusCode.ERROR, "Exception occurred: " + throwable.getMessage()); - throw throwable; - } finally { - span.end(); - } - }; + public SpanBuilder spanBuilder(String spanName) { + return getTracer().spanBuilder(spanName); } - /** Wraps a Function with an OpenTelemetry span using default span name "span". */ - public Function observe(Function function) { - return observe(function, "span"); + public Span startSpan(String spanName) { + return spanBuilder(spanName).startSpan(); } - /** Wraps a Supplier with an OpenTelemetry span for observability. */ - public Supplier observe(Supplier supplier, String spanName) { - Objects.requireNonNull(supplier, "Supplier cannot be null"); - String finalSpanName = spanName != null ? spanName : "span"; - - return () -> { - io.opentelemetry.api.trace.Tracer otelTracer = - GlobalOpenTelemetry.get().getTracer("com.judgmentlabs.judgeval"); - Span span = otelTracer.spanBuilder(finalSpanName).startSpan(); - - try (Scope scope = span.makeCurrent()) { - T result = supplier.get(); - - if (result != null) { - setOutput(result); - } - - span.setStatus(StatusCode.OK); - return result; - } catch (Throwable throwable) { - span.recordException(throwable); - span.setStatus(StatusCode.ERROR, "Exception occurred: " + throwable.getMessage()); - throw throwable; - } finally { - span.end(); - } - }; + public Scope makeCurrent(Span span) { + return span.makeCurrent(); + } + + public SpanContext getCurrentSpanContext() { + Span current = getCurrentSpan(); + return current != null ? current.getSpanContext() : null; } - /** Wraps a Supplier with an OpenTelemetry span using default span name "span". */ - public Supplier observe(Supplier supplier) { - return observe(supplier, "span"); + public boolean isCurrentSpanSampled() { + SpanContext context = getCurrentSpanContext(); + return context != null && context.isSampled(); } private String resolveProjectId(String name) { From 98c4489f72fb833b4dd3550a2327b904666b0dbc Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Sun, 28 Sep 2025 15:37:33 -0700 Subject: [PATCH 05/14] format --- src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java index 0c9b599..b3da538 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java @@ -28,7 +28,6 @@ import io.opentelemetry.context.Scope; import io.opentelemetry.sdk.trace.export.SpanExporter; -/** Base tracer providing core functionality for distributed tracing and evaluation. */ public abstract class BaseTracer { public static final String TRACER_NAME = "judgeval"; @@ -63,7 +62,6 @@ protected BaseTracer( } } - /** Initializes the OpenTelemetry SDK with span processors and exporters. */ public abstract void initialize(); /** From 65f755233d6da4cf2230e2133a9f30ab12d45cf4 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Sun, 28 Sep 2025 18:03:31 -0700 Subject: [PATCH 06/14] telemetry version and set resource attrs --- pom.xml | 13 +++++++++++++ .../com/judgmentlabs/judgeval/Version.java | 18 ++++++++++++++++++ .../judgmentlabs/judgeval/tracer/Tracer.java | 15 +++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 src/main/java/com/judgmentlabs/judgeval/Version.java diff --git a/pom.xml b/pom.xml index f658e54..e0e2f4a 100644 --- a/pom.xml +++ b/pom.xml @@ -109,6 +109,19 @@ + + org.apache.maven.plugins + maven-jar-plugin + 3.4.1 + + + + ${project.version} + + + + + org.apache.maven.plugins maven-source-plugin diff --git a/src/main/java/com/judgmentlabs/judgeval/Version.java b/src/main/java/com/judgmentlabs/judgeval/Version.java new file mode 100644 index 0000000..f8d252e --- /dev/null +++ b/src/main/java/com/judgmentlabs/judgeval/Version.java @@ -0,0 +1,18 @@ +package com.judgmentlabs.judgeval; + +public final class Version { + + private Version() {} + + /** + * Gets the current SDK version. + * + * @return the version string + */ + public static String getVersion() { + Package pkg = Version.class.getPackage(); + return (pkg != null && pkg.getImplementationVersion() != null) + ? pkg.getImplementationVersion() + : "unknown"; + } +} diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java index 21bff77..b3e4dc7 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java @@ -1,8 +1,12 @@ package com.judgmentlabs.judgeval.tracer; +import com.judgmentlabs.judgeval.Version; + import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.resources.Resource; import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; import io.opentelemetry.sdk.trace.export.SpanExporter; @@ -39,8 +43,19 @@ public static Tracer createWithConfiguration(TracerConfiguration configuration) public void initialize() { SpanExporter spanExporter = getSpanExporter(); + var resource = + Resource.getDefault() + .merge( + Resource.create( + Attributes.builder() + .put("service.name", "SqlAgent") + .put("telemetry.sdk.name", TRACER_NAME) + .put("telemetry.sdk.version", Version.getVersion()) + .build())); + SdkTracerProvider tracerProvider = SdkTracerProvider.builder() + .setResource(resource) .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build()) .build(); From ceb3b5abda22de0bcb072a96a93ae90ece0e1001 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Tue, 30 Sep 2025 12:27:32 -0700 Subject: [PATCH 07/14] chore: tracer api --- pom.xml | 30 ++++++++++ .../judgeval/tracer/BaseTracer.java | 57 ++++++++++--------- 2 files changed, 61 insertions(+), 26 deletions(-) diff --git a/pom.xml b/pom.xml index e0e2f4a..56c452e 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,36 @@ 5.10.1 test + + org.mockito + mockito-core + 5.8.0 + test + + + org.mockito + mockito-junit-jupiter + 5.8.0 + test + + + io.opentelemetry + opentelemetry-sdk-testing + 1.39.0 + test + + + com.github.tomakehurst + wiremock-jre8 + 2.35.0 + test + + + org.awaitility + awaitility + 4.2.0 + test + io.opentelemetry opentelemetry-exporter-otlp diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java index b3da538..74b190b 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java @@ -22,7 +22,6 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Scope; @@ -222,37 +221,43 @@ public void setOutput(Object output, Type type) { setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_OUTPUT, output, type); } - public Tracer getTracer() { - return GlobalOpenTelemetry.get().getTracer(TRACER_NAME); - } - - public Span getCurrentSpan() { - return Span.current(); - } - - public SpanBuilder spanBuilder(String spanName) { - return getTracer().spanBuilder(spanName); - } - - public Span startSpan(String spanName) { - return spanBuilder(spanName).startSpan(); - } - - public Scope makeCurrent(Span span) { - return span.makeCurrent(); + /** + * Creates a new span with the given name and executes the provided runnable within its scope. + * The span is automatically ended when the runnable completes. + */ + public void span(String spanName, Runnable runnable) { + Span span = span(spanName); + try (Scope scope = span.makeCurrent()) { + runnable.run(); + } finally { + span.end(); + } } - public SpanContext getCurrentSpanContext() { - Span current = getCurrentSpan(); - return current != null ? current.getSpanContext() : null; + /** + * Creates a new span with the given name and executes the provided callable within its scope. + * The span is automatically ended when the callable completes. Returns the result of the + * callable. + */ + public T span(String spanName, java.util.concurrent.Callable callable) throws Exception { + Span span = span(spanName); + try (Scope scope = span.makeCurrent()) { + return callable.call(); + } finally { + span.end(); + } } - public boolean isCurrentSpanSampled() { - SpanContext context = getCurrentSpanContext(); - return context != null && context.isSampled(); + /** + * Creates a new span with the given name. The caller is responsible for ending the span + * manually. + */ + public static Span span(String spanName) { + Tracer tracer = GlobalOpenTelemetry.get().getTracer(TRACER_NAME); + return tracer.spanBuilder(spanName).startSpan(); } - private String resolveProjectId(String name) { + protected String resolveProjectId(String name) { try { ResolveProjectNameRequest request = new ResolveProjectNameRequest(); request.setProjectName(name); From ec347eb22ab9323e009efa4b95f98a50ef720377 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Tue, 30 Sep 2025 12:29:43 -0700 Subject: [PATCH 08/14] add back getTracer --- .../com/judgmentlabs/judgeval/tracer/BaseTracer.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java index 74b190b..36b8773 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java @@ -226,7 +226,7 @@ public void setOutput(Object output, Type type) { * The span is automatically ended when the runnable completes. */ public void span(String spanName, Runnable runnable) { - Span span = span(spanName); + Span span = getTracer().spanBuilder(spanName).startSpan(); try (Scope scope = span.makeCurrent()) { runnable.run(); } finally { @@ -240,7 +240,7 @@ public void span(String spanName, Runnable runnable) { * callable. */ public T span(String spanName, java.util.concurrent.Callable callable) throws Exception { - Span span = span(spanName); + Span span = getTracer().spanBuilder(spanName).startSpan(); try (Scope scope = span.makeCurrent()) { return callable.call(); } finally { @@ -248,13 +248,17 @@ public T span(String spanName, java.util.concurrent.Callable callable) th } } + /** Gets the OpenTelemetry tracer instance. */ + public Tracer getTracer() { + return GlobalOpenTelemetry.get().getTracer(TRACER_NAME); + } + /** * Creates a new span with the given name. The caller is responsible for ending the span * manually. */ public static Span span(String spanName) { - Tracer tracer = GlobalOpenTelemetry.get().getTracer(TRACER_NAME); - return tracer.spanBuilder(spanName).startSpan(); + return GlobalOpenTelemetry.get().getTracer(TRACER_NAME).spanBuilder(spanName).startSpan(); } protected String resolveProjectId(String name) { From 1ad2bb633939186a31821b3379edda8edbc6614d Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Tue, 30 Sep 2025 12:52:29 -0700 Subject: [PATCH 09/14] logger --- .../judgeval/tracer/exporters/JudgmentSpanExporter.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java index b57f07c..8de675a 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java @@ -2,6 +2,8 @@ import java.util.Collection; +import com.judgmentlabs.judgeval.utils.Logger; + import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.trace.data.SpanData; @@ -50,6 +52,7 @@ public static Builder builder() { @Override public CompletableResultCode export(Collection spans) { + Logger.info("Exported " + spans.size() + " spans"); return delegate.export(spans); } From 0bb2a680292cea2065b52730cd4334cb1d12db3b Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Tue, 30 Sep 2025 13:52:13 -0700 Subject: [PATCH 10/14] remove example params --- .../judgmentlabs/judgeval/data/Example.java | 63 ------------------- .../api_scorers/AnswerCorrectnessScorer.java | 7 +-- .../api_scorers/AnswerRelevancyScorer.java | 5 +- .../api_scorers/FaithfulnessScorer.java | 7 +-- .../InstructionAdherenceScorer.java | 5 +- 5 files changed, 4 insertions(+), 83 deletions(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/data/Example.java b/src/main/java/com/judgmentlabs/judgeval/data/Example.java index e26451c..ebfc049 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/Example.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/Example.java @@ -1,32 +1,10 @@ package com.judgmentlabs.judgeval.data; import java.time.Instant; -import java.util.Map; import java.util.UUID; public class Example extends com.judgmentlabs.judgeval.internal.api.models.Example { - public enum ExampleParams { - INPUT("input"), - ACTUAL_OUTPUT("actual_output"), - EXPECTED_OUTPUT("expected_output"), - CONTEXT("context"), - RETRIEVAL_CONTEXT("retrieval_context"), - TOOLS_CALLED("tools_called"), - EXPECTED_TOOLS("expected_tools"), - ADDITIONAL_METADATA("additional_metadata"); - - private final String value; - - ExampleParams(String value) { - this.value = value; - } - - public String getValue() { - return value; - } - } - public Example() { super(); setExampleId(UUID.randomUUID().toString()); @@ -45,47 +23,6 @@ private Builder() { this.example = new Example(); } - public Builder input(String input) { - example.setAdditionalProperty(ExampleParams.INPUT.getValue(), input); - return this; - } - - public Builder actualOutput(String actualOutput) { - example.setAdditionalProperty(ExampleParams.ACTUAL_OUTPUT.getValue(), actualOutput); - return this; - } - - public Builder expectedOutput(String expectedOutput) { - example.setAdditionalProperty(ExampleParams.EXPECTED_OUTPUT.getValue(), expectedOutput); - return this; - } - - public Builder context(String context) { - example.setAdditionalProperty(ExampleParams.CONTEXT.getValue(), context); - return this; - } - - public Builder retrievalContext(String retrievalContext) { - example.setAdditionalProperty( - ExampleParams.RETRIEVAL_CONTEXT.getValue(), retrievalContext); - return this; - } - - public Builder toolsCalled(Object toolsCalled) { - example.setAdditionalProperty(ExampleParams.TOOLS_CALLED.getValue(), toolsCalled); - return this; - } - - public Builder expectedTools(Object expectedTools) { - example.setAdditionalProperty(ExampleParams.EXPECTED_TOOLS.getValue(), expectedTools); - return this; - } - - public Builder additionalMetadata(Map metadata) { - example.setAdditionalProperty(ExampleParams.ADDITIONAL_METADATA.getValue(), metadata); - return this; - } - public Builder property(String key, Object value) { example.setAdditionalProperty(key, value); return this; diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java index ae65185..483d4e9 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java @@ -3,17 +3,12 @@ import java.util.Arrays; import com.judgmentlabs.judgeval.data.APIScorerType; -import com.judgmentlabs.judgeval.data.Example.ExampleParams; import com.judgmentlabs.judgeval.scorers.APIScorer; public class AnswerCorrectnessScorer extends APIScorer { public AnswerCorrectnessScorer() { super(APIScorerType.ANSWER_CORRECTNESS); - setRequiredParams( - Arrays.asList( - ExampleParams.INPUT.getValue(), - ExampleParams.ACTUAL_OUTPUT.getValue(), - ExampleParams.EXPECTED_OUTPUT.getValue())); + setRequiredParams(Arrays.asList("input", "actual_output", "expected_output")); } public static APIScorer.Builder builder() { diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java index acac8cf..ece195b 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java @@ -3,15 +3,12 @@ import java.util.Arrays; import com.judgmentlabs.judgeval.data.APIScorerType; -import com.judgmentlabs.judgeval.data.Example.ExampleParams; import com.judgmentlabs.judgeval.scorers.APIScorer; public class AnswerRelevancyScorer extends APIScorer { public AnswerRelevancyScorer() { super(APIScorerType.ANSWER_RELEVANCY); - setRequiredParams( - Arrays.asList( - ExampleParams.INPUT.getValue(), ExampleParams.ACTUAL_OUTPUT.getValue())); + setRequiredParams(Arrays.asList("input", "actual_output")); } public static APIScorer.Builder builder() { diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java index 3649bfe..dd4ca09 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java @@ -3,17 +3,12 @@ import java.util.Arrays; import com.judgmentlabs.judgeval.data.APIScorerType; -import com.judgmentlabs.judgeval.data.Example.ExampleParams; import com.judgmentlabs.judgeval.scorers.APIScorer; public class FaithfulnessScorer extends APIScorer { public FaithfulnessScorer() { super(APIScorerType.FAITHFULNESS); - setRequiredParams( - Arrays.asList( - ExampleParams.INPUT.getValue(), - ExampleParams.ACTUAL_OUTPUT.getValue(), - ExampleParams.RETRIEVAL_CONTEXT.getValue())); + setRequiredParams(Arrays.asList("input", "actual_output", "retrieval_context")); } public static APIScorer.Builder builder() { diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java index fe40999..96416b8 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java @@ -3,15 +3,12 @@ import java.util.Arrays; import com.judgmentlabs.judgeval.data.APIScorerType; -import com.judgmentlabs.judgeval.data.Example.ExampleParams; import com.judgmentlabs.judgeval.scorers.APIScorer; public class InstructionAdherenceScorer extends APIScorer { public InstructionAdherenceScorer() { super(APIScorerType.INSTRUCTION_ADHERENCE); - setRequiredParams( - Arrays.asList( - ExampleParams.INPUT.getValue(), ExampleParams.ACTUAL_OUTPUT.getValue())); + setRequiredParams(Arrays.asList("input", "actual_output")); setName("Instruction Adherence"); } From 668b9e0debfd836be17c2b9920aa873c43a02dc4 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Tue, 30 Sep 2025 14:34:20 -0700 Subject: [PATCH 11/14] remove example scorer --- .../judgeval/scorers/BaseScorer.java | 104 +----------------- .../judgeval/scorers/ExampleScorer.java | 20 ---- 2 files changed, 1 insertion(+), 123 deletions(-) delete mode 100644 src/main/java/com/judgmentlabs/judgeval/scorers/ExampleScorer.java diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java index daac522..3d31647 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java @@ -2,7 +2,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; @@ -10,9 +9,8 @@ public abstract class BaseScorer extends com.judgmentlabs.judgeval.internal.api. public BaseScorer() { super(); - setClassName(this.getClass().getSimpleName()); if (getName() == null) { - setName(getClassName()); + setName(this.getClass().getSimpleName()); } if (Boolean.TRUE.equals(getStrictMode())) { setThreshold(1.0); @@ -21,10 +19,6 @@ public BaseScorer() { public abstract ScorerConfig getScorerConfig(); - public void addModel(String model) { - setModel(model); - } - public boolean successCheck() { if (getError() != null) { return false; @@ -68,15 +62,6 @@ public void setName(String name) { super.setName(name); } - public String getClassName() { - Object className = super.getClassName(); - return className != null ? className.toString() : null; - } - - public void setClassName(String className) { - super.setClassName(className); - } - public Double getScore() { Object score = super.getScore(); if (score instanceof Number) { @@ -89,53 +74,6 @@ public void setScore(Double score) { super.setScore(score); } - public Map getScoreBreakdown() { - Object breakdown = super.getScoreBreakdown(); - if (breakdown instanceof Map) { - @SuppressWarnings("unchecked") - Map map = (Map) breakdown; - return map; - } - return null; - } - - public void setScoreBreakdown(Map scoreBreakdown) { - super.setScoreBreakdown(scoreBreakdown); - } - - public String getReason() { - Object reason = super.getReason(); - return reason != null ? reason.toString() : ""; - } - - public void setReason(String reason) { - super.setReason(reason); - } - - public Boolean getUsingNativeModel() { - Object usingNativeModel = super.getUsingNativeModel(); - if (usingNativeModel instanceof Boolean) { - return (Boolean) usingNativeModel; - } - return null; - } - - public void setUsingNativeModel(Boolean usingNativeModel) { - super.setUsingNativeModel(usingNativeModel); - } - - public Boolean getSuccess() { - Object success = super.getSuccess(); - if (success instanceof Boolean) { - return (Boolean) success; - } - return null; - } - - public void setSuccess(Boolean success) { - super.setSuccess(success); - } - public String getModel() { Object model = super.getModel(); return model != null ? model.toString() : null; @@ -145,14 +83,6 @@ public void setModel(String model) { super.setModel(model); } - public Object getModelClient() { - return super.getModelClient(); - } - - public void setModelClient(Object modelClient) { - super.setModelClient(modelClient); - } - public boolean isStrictMode() { Boolean strictMode = super.getStrictMode(); return strictMode != null ? strictMode : false; @@ -170,36 +100,4 @@ public String getError() { public void setError(String error) { super.setError(error); } - - public Map getAdditionalMetadata() { - Object metadata = super.getAdditionalMetadata(); - if (metadata instanceof Map) { - @SuppressWarnings("unchecked") - Map map = (Map) metadata; - return map; - } - return null; - } - - public void setAdditionalMetadata(Map additionalMetadata) { - super.setAdditionalMetadata(additionalMetadata); - } - - public String getUser() { - Object user = super.getUser(); - return user != null ? user.toString() : null; - } - - public void setUser(String user) { - super.setUser(user); - } - - public boolean isServerHosted() { - Boolean serverHosted = super.getServerHosted(); - return serverHosted != null ? serverHosted : false; - } - - public void setServerHosted(boolean serverHosted) { - super.setServerHosted(serverHosted); - } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/ExampleScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/ExampleScorer.java deleted file mode 100644 index edcbc55..0000000 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/ExampleScorer.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.judgmentlabs.judgeval.scorers; - -import java.util.ArrayList; -import java.util.List; - -import com.judgmentlabs.judgeval.data.Example; - -public abstract class ExampleScorer extends BaseScorer { - - public ExampleScorer() { - super(); - setScoreType("Custom"); - } - - public List getRequiredParams() { - return new ArrayList<>(); - } - - public abstract double scoreExample(Example example); -} From 9b6f68c571c2084c5cdfba0e4882f832a2b0e925 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Tue, 30 Sep 2025 14:48:30 -0700 Subject: [PATCH 12/14] simplify scorer --- .../judgeval/scorers/APIScorer.java | 22 +++- .../judgeval/scorers/BaseScorer.java | 116 +++--------------- .../judgmentlabs/judgeval/tracer/Tracer.java | 2 +- 3 files changed, 40 insertions(+), 100 deletions(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java index 32e24b6..e862530 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java @@ -8,7 +8,8 @@ import com.judgmentlabs.judgeval.data.APIScorerType; import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; -public class APIScorer extends BaseScorer { +public class APIScorer extends com.judgmentlabs.judgeval.internal.api.models.BaseScorer + implements BaseScorer { private APIScorerType scoreType; @JsonIgnore private List requiredParams; @@ -19,6 +20,9 @@ public APIScorer(APIScorerType scoreType) { setName(scoreType.toString()); setScoreType(scoreType.toString()); this.requiredParams = new java.util.ArrayList<>(); + if (Boolean.TRUE.equals(getStrictMode())) { + setThreshold(1.0); + } } public void setThreshold(double threshold) { @@ -41,6 +45,22 @@ public void setRequiredParams(List requiredParams) { this.requiredParams = requiredParams; } + // Convenience methods for type safety + public Double getThreshold() { + Double threshold = super.getThreshold(); + return threshold != null ? threshold : 0.5; + } + + public String getName() { + Object name = super.getName(); + return name != null ? name.toString() : null; + } + + public boolean isStrictMode() { + Boolean strictMode = super.getStrictMode(); + return strictMode != null ? strictMode : false; + } + @Override public ScorerConfig getScorerConfig() { ScorerConfig cfg = new ScorerConfig(); diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java index 3d31647..ce73512 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java @@ -1,103 +1,23 @@ package com.judgmentlabs.judgeval.scorers; -import java.util.ArrayList; -import java.util.List; - import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; -public abstract class BaseScorer extends com.judgmentlabs.judgeval.internal.api.models.BaseScorer { - - public BaseScorer() { - super(); - if (getName() == null) { - setName(this.getClass().getSimpleName()); - } - if (Boolean.TRUE.equals(getStrictMode())) { - setThreshold(1.0); - } - } - - public abstract ScorerConfig getScorerConfig(); - - public boolean successCheck() { - if (getError() != null) { - return false; - } - if (getScore() == null) { - return false; - } - Double threshold = getThreshold(); - Double score = getScore(); - return threshold != null && score != null && score >= threshold; - } - - public List getRequiredParams() { - return new ArrayList<>(); - } - - public String getScoreType() { - Object scoreType = super.getScoreType(); - return scoreType != null ? scoreType.toString() : null; - } - - public void setScoreType(String scoreType) { - super.setScoreType(scoreType); - } - - public Double getThreshold() { - Double threshold = super.getThreshold(); - return threshold != null ? threshold : 0.5; - } - - public void setThreshold(double threshold) { - super.setThreshold(threshold); - } - - public String getName() { - Object name = super.getName(); - return name != null ? name.toString() : null; - } - - public void setName(String name) { - super.setName(name); - } - - public Double getScore() { - Object score = super.getScore(); - if (score instanceof Number) { - return ((Number) score).doubleValue(); - } - return null; - } - - public void setScore(Double score) { - super.setScore(score); - } - - public String getModel() { - Object model = super.getModel(); - return model != null ? model.toString() : null; - } - - public void setModel(String model) { - super.setModel(model); - } - - public boolean isStrictMode() { - Boolean strictMode = super.getStrictMode(); - return strictMode != null ? strictMode : false; - } - - public void setStrictMode(boolean strictMode) { - super.setStrictMode(strictMode); - } - - public String getError() { - Object error = super.getError(); - return error != null ? error.toString() : null; - } - - public void setError(String error) { - super.setError(error); - } +/** + * Minimal interface for scorers used by BaseTracer. Only requires the essential methods needed for + * evaluation. + */ +public interface BaseScorer { + /** + * Gets the name of the scorer. + * + * @return the scorer name + */ + String getName(); + + /** + * Gets the scorer configuration for evaluation runs. + * + * @return the scorer configuration + */ + ScorerConfig getScorerConfig(); } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java index b3e4dc7..d5d394a 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java @@ -48,7 +48,7 @@ public void initialize() { .merge( Resource.create( Attributes.builder() - .put("service.name", "SqlAgent") + .put("service.name", configuration.projectName()) .put("telemetry.sdk.name", TRACER_NAME) .put("telemetry.sdk.version", Version.getVersion()) .build())); From e7d24aee2f71cbe36cb67ea16fb776040a8672da Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Tue, 30 Sep 2025 19:46:19 -0700 Subject: [PATCH 13/14] fix --- .../judgeval/scorers/APIScorer.java | 18 +----------------- .../prompt_scorer/PromptScorer.java | 2 +- .../prompt_scorer/TracePromptScorer.java | 2 +- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java index e862530..f4b2a74 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java @@ -45,29 +45,13 @@ public void setRequiredParams(List requiredParams) { this.requiredParams = requiredParams; } - // Convenience methods for type safety - public Double getThreshold() { - Double threshold = super.getThreshold(); - return threshold != null ? threshold : 0.5; - } - - public String getName() { - Object name = super.getName(); - return name != null ? name.toString() : null; - } - - public boolean isStrictMode() { - Boolean strictMode = super.getStrictMode(); - return strictMode != null ? strictMode : false; - } - @Override public ScorerConfig getScorerConfig() { ScorerConfig cfg = new ScorerConfig(); cfg.setScoreType(getScoreType()); cfg.setThreshold(getThreshold()); cfg.setName(getName()); - cfg.setStrictMode(isStrictMode()); + cfg.setStrictMode(getStrictMode()); cfg.setRequiredParams(getRequiredParams()); Map kwargs = new HashMap<>(); if (getAdditionalProperties() != null) kwargs.putAll(getAdditionalProperties()); diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java index f555641..d01cc96 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java @@ -113,7 +113,7 @@ public ScorerConfig getScorerConfig() { cfg.setScoreType(getScoreType()); cfg.setThreshold(getThreshold()); cfg.setName(getName()); - cfg.setStrictMode(isStrictMode()); + cfg.setStrictMode(getStrictMode()); cfg.setRequiredParams(getRequiredParams()); Map kwargs = new HashMap<>(); kwargs.put("prompt", prompt); diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java index ac2c5f9..94d0b03 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java @@ -115,7 +115,7 @@ public ScorerConfig getScorerConfig() { cfg.setScoreType(getScoreType()); cfg.setThreshold(getThreshold()); cfg.setName(getName()); - cfg.setStrictMode(isStrictMode()); + cfg.setStrictMode(getStrictMode()); cfg.setRequiredParams(getRequiredParams()); Map kwargs = new HashMap<>(); kwargs.put("prompt", prompt); From d99bf7a40dbcb58faa8bf20d19ccf3c4cbf9a3eb Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Thu, 2 Oct 2025 13:01:50 -0700 Subject: [PATCH 14/14] chore: add back overrides --- .../judgeval/scorers/APIScorer.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java index f4b2a74..a11e6f1 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java @@ -45,6 +45,24 @@ public void setRequiredParams(List requiredParams) { this.requiredParams = requiredParams; } + @Override + public Double getThreshold() { + Double threshold = super.getThreshold(); + return threshold != null ? threshold : 0.5; + } + + @Override + public String getName() { + Object name = super.getName(); + return name != null ? name.toString() : null; + } + + @Override + public Boolean getStrictMode() { + Boolean strictMode = super.getStrictMode(); + return strictMode != null ? strictMode : false; + } + @Override public ScorerConfig getScorerConfig() { ScorerConfig cfg = new ScorerConfig();