The Tracer class provides functionality to: - * - *
{@code
- * // Simple usage with defaults
- * Tracer tracer = Tracer.createDefault("my-project");
- *
- * // Get the span exporter for OpenTelemetry integration
- * SpanExporter exporter = tracer.getSpanExporter();
- *
- * // Set custom attributes on the current span
- * tracer.setAttribute("custom.key", "custom.value");
- *
- * // Evaluate a scorer with an example
- * Example example = new Example();
- * example.setAdditionalProperty("input", "user input");
- * example.setAdditionalProperty("output", "model output");
- *
- * BaseScorer scorer = new AnswerRelevancyScorer();
- * tracer.asyncEvaluate(scorer, example, "gpt-4");
- *
- * // Evaluate a scorer with trace context
- * tracer.asyncTraceEvaluate(scorer, "gpt-4");
- * }
- *
- * {@code
- * TracerConfiguration config = TracerConfiguration.builder()
- * .projectName("my-project")
- * .apiKey("custom-api-key")
- * .organizationId("custom-org-id")
- * .enableEvaluation(false)
- * .build();
- *
- * Tracer tracer = Tracer.createWithConfiguration(config);
- * }
- *
- * {@code
- * Tracer tracer = Tracer.createDefault("my-project");
- * SpanExporter exporter = tracer.getSpanExporter();
- *
- * SdkTracerProvider provider = SdkTracerProvider.builder()
- * .addSpanProcessor(BatchSpanProcessor.builder(exporter).build())
- * .build();
- *
- * OpenTelemetrySdk.builder()
- * .setTracerProvider(provider)
- * .buildAndRegisterGlobal();
- * }
+ * Main tracer for Judgment Labs distributed tracing and evaluation.
*
* @see TracerConfiguration
* @see SpanExporter
- * @see BaseScorer
- * @see Example
+ * @see com.judgmentlabs.judgeval.scorers.BaseScorer
+ * @see com.judgmentlabs.judgeval.data.Example
*/
-public final class Tracer {
- private final TracerConfiguration configuration;
- private final JudgmentSyncClient apiClient;
- private final ISerializer serializer;
- private final ObjectMapper jacksonMapper;
- private final String projectId;
+public final class Tracer extends BaseTracer {
private Tracer(
- TracerConfiguration configuration,
- JudgmentSyncClient apiClient,
- ISerializer serializer) {
- this.configuration = Objects.requireNonNull(configuration, "Configuration cannot be null");
- this.apiClient = Objects.requireNonNull(apiClient, "API client cannot be null");
- this.serializer = Objects.requireNonNull(serializer, "Serializer cannot be null");
- this.jacksonMapper = new ObjectMapper();
- this.projectId = resolveProjectId(configuration.projectName());
- if (this.projectId == null) {
- Logger.error(
- "Failed to resolve project "
- + configuration.projectName()
- + ", please create it first at https://app.judgmentlabs.ai/org/"
- + configuration.organizationId()
- + "/projects. Skipping Judgment export.");
- }
+ TracerConfiguration configuration, ISerializer serializer, boolean shouldInitialize) {
+ super(configuration, serializer, shouldInitialize);
}
public static TracerBuilder builder() {
@@ -131,348 +38,51 @@ public static Tracer createWithConfiguration(TracerConfiguration configuration)
return builder().configuration(configuration).build();
}
- /**
- * Gets the SpanExporter for integration with OpenTelemetry.
- *
- * If the project ID cannot be resolved (e.g., the project doesn't exist), this method - * returns a NoOpSpanExporter. - * - *
Example usage: - * - *
{@code
- * Tracer tracer = Tracer.createDefault("my-project");
- * SpanExporter exporter = tracer.getSpanExporter();
- *
- * SdkTracerProvider provider = SdkTracerProvider.builder()
- * .addSpanProcessor(BatchSpanProcessor.builder(exporter).build())
- * .build();
- * }
- *
- * @return a SpanExporter that sends spans to Judgment Labs, or a NoOpSpanExporter if project
- * resolution fails
- */
- public SpanExporter getSpanExporter() {
- if (projectId == null) {
- Logger.error(
- "Project not resolved; cannot create exporter, returning NoOpSpanExporter");
- return new NoOpSpanExporter();
- }
- return createJudgmentSpanExporter(projectId);
- }
-
- /**
- * Sets the span kind attribute on the current span.
- *
- * Common span kinds include: - * - *
This method serializes the value to JSON and sets it as a string attribute on the current - * span. This is useful for adding custom metadata to spans for better observability. - * - *
If there is no current span, this method does nothing. - * - * @param key the attribute key (must not be null) - * @param value the attribute value (will be serialized to JSON) - */ - public void setAttribute(String key, Object value) { - Optional.ofNullable(Span.current()) - .ifPresent(span -> span.setAttribute(key, serializer.serialize(value))); - } - - /** - * Sets a custom attribute on the current span with a specific type. - * - *
This method is similar to {@link #setAttribute(String, Object)} but allows you to specify - * the type for JSON serialization. This is useful when you need to preserve type information or - * when dealing with generic types. - * - *
If there is no current span, this method does nothing. - * - * @param key the attribute key (must not be null) - * @param value the attribute value (will be serialized to JSON) - * @param type the type to use for JSON serialization (must not be null) - */ - public void setAttribute(String key, Object value, Type type) { - Optional.ofNullable(Span.current()) - .ifPresent(span -> span.setAttribute(key, serializer.serialize(value, type))); - } - - /** - * Asynchronously evaluates a scorer with an example. - * - *
The evaluation includes: - * - *
If evaluation is disabled in the configuration, this method does nothing. The method - * respects OpenTelemetry's internal sampler - evaluation only runs if the current span is - * recording. - * - * @param scorer the scorer to use for evaluation (must not be null) - * @param example the example to evaluate (must not be null) - * @param model the model used for generation (can be null, will use default) - */ - public void asyncEvaluate(BaseScorer scorer, Example example, String model) { - try { - if (!configuration.enableEvaluation()) { - return; - } - - Span currentSpan = Span.current(); - if (currentSpan == null || !currentSpan.getSpanContext().isSampled()) { - return; - } - - SpanContext spanContext = currentSpan.getSpanContext(); - String traceId = spanContext.getTraceId(); - String spanId = spanContext.getSpanId(); - - Logger.info( - "asyncEvaluate: project=" - + configuration.projectName() - + ", traceId=" - + traceId - + ", spanId=" - + spanId - + ", scorer=" - + scorer.getName()); + /** Initializes the OpenTelemetry SDK with batch span processor and registers it globally. */ + @Override + public void initialize() { + SpanExporter spanExporter = getSpanExporter(); - ExampleEvaluationRun evaluationRun = - createEvaluationRun(scorer, example, model, traceId, spanId); - enqueueEvaluation(evaluationRun); - } catch (Exception e) { - Logger.error("Failed to evaluate scorer: " + e.getMessage()); - } - } - - public void asyncEvaluate(BaseScorer scorer, Example example) { - asyncEvaluate(scorer, example, null); - } + var resource = + Resource.getDefault() + .merge( + Resource.create( + Attributes.builder() + .put("service.name", configuration.projectName()) + .put("telemetry.sdk.name", TRACER_NAME) + .put("telemetry.sdk.version", Version.getVersion()) + .build())); - /** - * Asynchronously evaluates a scorer with trace and span context. - * - *
This method creates a trace evaluation run and sets it as an attribute on the current - * span. The evaluation will be processed when the span is exported. - * - *
If evaluation is disabled in the configuration, this method does nothing. The method
- * respects OpenTelemetry's internal sampler - evaluation only runs if the current span is
- * recording.
- *
- * @param scorer the scorer to use for evaluation (must not be null)
- * @param model the model used for generation (can be null, will use default)
- */
- public void asyncTraceEvaluate(BaseScorer scorer, String model) {
- try {
- if (!configuration.enableEvaluation()) {
- return;
- }
+ SdkTracerProvider tracerProvider =
+ SdkTracerProvider.builder()
+ .setResource(resource)
+ .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build())
+ .build();
- Span currentSpan = Span.current();
- if (currentSpan == null || !currentSpan.getSpanContext().isSampled()) {
- return;
- }
+ OpenTelemetry openTelemetry =
+ OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build();
- SpanContext spanContext = currentSpan.getSpanContext();
- String traceId = spanContext.getTraceId();
- String spanId = spanContext.getSpanId();
-
- Logger.info(
- "asyncTraceEvaluate: project="
- + configuration.projectName()
- + ", traceId="
- + traceId
- + ", spanId="
- + spanId
- + ", scorer="
- + scorer.getName());
-
- TraceEvaluationRun evaluationRun =
- createTraceEvaluationRun(scorer, model, traceId, spanId);
- try {
- String traceEvalJson = jacksonMapper.writeValueAsString(evaluationRun);
- currentSpan.setAttribute(
- OpenTelemetryKeys.AttributeKeys.PENDING_TRACE_EVAL, traceEvalJson);
- } catch (Exception e) {
- Logger.error("Failed to serialize trace evaluation: " + e.getMessage());
- }
- } catch (Exception e) {
- Logger.error("Failed to evaluate trace scorer: " + e.getMessage());
- }
+ GlobalOpenTelemetry.set(openTelemetry);
}
- public void asyncTraceEvaluate(BaseScorer scorer) {
- asyncTraceEvaluate(scorer, null);
- }
-
- public void setAttributes(Map Example usage:
- *
- * {@code
- * TracerConfiguration config = TracerConfiguration.builder()
- * .projectName("my-project")
- * .build();
- *
- * JudgmentSyncClient customClient = new JudgmentSyncClient(url, key, orgId);
- * ISerializer customSerializer = obj -> obj.toString();
- *
- * Tracer tracer = Tracer.builder()
- * .configuration(config)
- * .apiClient(customClient)
- * .serializer(customSerializer)
- * .build();
- * }
- */
+ /** Builder for creating Tracer instances. */
public static final class TracerBuilder {
private TracerConfiguration configuration;
- private JudgmentSyncClient apiClient;
private ISerializer serializer = new GsonSerializer();
+ private boolean initialize = false;
public TracerBuilder configuration(TracerConfiguration configuration) {
this.configuration = configuration;
return this;
}
- public TracerBuilder apiClient(JudgmentSyncClient apiClient) {
- this.apiClient = apiClient;
+ public TracerBuilder serializer(ISerializer serializer) {
+ this.serializer = serializer;
return this;
}
- public TracerBuilder serializer(ISerializer serializer) {
- this.serializer = serializer;
+ public TracerBuilder initialize(boolean initialize) {
+ this.initialize = initialize;
return this;
}
@@ -481,29 +91,7 @@ public Tracer build() {
throw new IllegalArgumentException("Configuration is required");
}
- JudgmentSyncClient client =
- apiClient != null
- ? apiClient
- : new JudgmentSyncClient(
- configuration.apiUrl(),
- configuration.apiKey(),
- configuration.organizationId());
-
- return new Tracer(configuration, client, serializer);
- }
- }
-
- private static class GsonSerializer implements ISerializer {
- private final Gson gson = new Gson();
-
- @Override
- public String serialize(Object obj) {
- return gson.toJson(obj);
- }
-
- @Override
- public String serialize(Object obj, Type type) {
- return gson.toJson(obj, type);
+ return new Tracer(configuration, serializer, initialize);
}
}
}
diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java
index 837fa84..8de675a 100644
--- a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java
+++ b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java
@@ -1,18 +1,11 @@
package com.judgmentlabs.judgeval.tracer.exporters;
import java.util.Collection;
-import java.util.List;
-import java.util.stream.Collectors;
-import com.judgmentlabs.judgeval.tracer.OpenTelemetryKeys;
import com.judgmentlabs.judgeval.utils.Logger;
-import io.opentelemetry.api.common.AttributeKey;
-import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter;
import io.opentelemetry.sdk.common.CompletableResultCode;
-import io.opentelemetry.sdk.resources.Resource;
-import io.opentelemetry.sdk.trace.data.DelegatingSpanData;
import io.opentelemetry.sdk.trace.data.SpanData;
import io.opentelemetry.sdk.trace.export.SpanExporter;
@@ -24,7 +17,6 @@
*/
public class JudgmentSpanExporter implements SpanExporter {
private final SpanExporter delegate;
- private final String projectId;
/**
* Creates a new JudgmentSpanExporter with the specified configuration.
@@ -45,8 +37,8 @@ public JudgmentSpanExporter(
.setEndpoint(endpoint)
.addHeader("Authorization", "Bearer " + apiKey)
.addHeader("X-Organization-Id", organizationId)
+ .addHeader("X-Project-Id", projectId)
.build();
- this.projectId = projectId;
}
/**
@@ -60,25 +52,8 @@ public static Builder builder() {
@Override
public CompletableResultCode export(Collection