From 3ce8c53dcc39f49b2a38fa654ff68ca9c389b5d1 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 17 Sep 2025 17:16:33 -0700 Subject: [PATCH 01/10] chore: tracer adjustments, logging, and error handling --- pom.xml | 2 +- .../judgmentlabs/judgeval/tracer/Tracer.java | 64 +++++++++++-------- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/pom.xml b/pom.xml index e4b9055..59c5a10 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.judgmentlabs judgeval-java - 0.1.9 + 0.2.0 jar Judgeval Java Java SDK for Judgeval diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java index b33a452..f10346a 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java @@ -102,6 +102,14 @@ private Tracer( this.apiClient = Objects.requireNonNull(apiClient, "API client cannot be null"); this.serializer = Objects.requireNonNull(serializer, "Serializer cannot be null"); this.projectId = resolveProjectId(configuration.projectName()); + if (this.projectId == null) { + Logger.error( + "Failed to resolve project " + + configuration.projectName() + + ", please create it first at https://app.judgmentlabs.ai/org/" + + configuration.organizationId() + + "/projects. Skipping Judgment export."); + } } public static TracerBuilder builder() { @@ -222,33 +230,37 @@ public void setAttribute(String key, Object value, Type type) { * @param model the model used for generation (can be null, will use default) */ public void asyncEvaluate(BaseScorer scorer, Example example, String model) { - if (!configuration.enableEvaluation()) { - return; - } - - Span currentSpan = Span.current(); - if (currentSpan == null || !currentSpan.getSpanContext().isSampled()) { - return; - } + try { + if (!configuration.enableEvaluation()) { + return; + } - SpanContext spanContext = currentSpan.getSpanContext(); - String traceId = spanContext.getTraceId(); - String spanId = spanContext.getSpanId(); - - Object transport = scorer.toTransport(); - Logger.info( - "asyncEvaluate: project=" - + configuration.projectName() - + ", traceId=" - + traceId - + ", spanId=" - + spanId - + ", scorerTransport=" - + (transport != null ? transport.getClass().getSimpleName() : "null")); + Span currentSpan = Span.current(); + if (currentSpan == null || !currentSpan.getSpanContext().isSampled()) { + return; + } - ExampleEvaluationRun evaluationRun = - createEvaluationRun(scorer, example, model, traceId, spanId); - enqueueEvaluation(evaluationRun); + SpanContext spanContext = currentSpan.getSpanContext(); + String traceId = spanContext.getTraceId(); + String spanId = spanContext.getSpanId(); + + Object transport = scorer.toTransport(); + Logger.info( + "asyncEvaluate: project=" + + configuration.projectName() + + ", traceId=" + + traceId + + ", spanId=" + + spanId + + ", scorerTransport=" + + (transport != null ? transport.getClass().getSimpleName() : "null")); + + ExampleEvaluationRun evaluationRun = + createEvaluationRun(scorer, example, model, traceId, spanId); + enqueueEvaluation(evaluationRun); + } catch (Exception e) { + Logger.error("Failed to evaluate scorer: " + e.getMessage()); + } } public void asyncEvaluate(BaseScorer scorer, Example example) { @@ -303,8 +315,6 @@ private String resolveProjectId(String name) { ResolveProjectNameResponse response = apiClient.projectsResolve(request); return Optional.ofNullable(response.getProjectId()).map(Object::toString).orElse(null); } catch (Exception e) { - Logger.error( - "Failed to resolve project ID for project '" + name + "': " + e.getMessage()); return null; } } From 301a3d9db58c1e4023199e3a17363975f1a39888 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 17 Sep 2025 17:21:31 -0700 Subject: [PATCH 02/10] chore: update api --- scripts/generate_client.py | 2 +- .../internal/api/JudgmentAsyncClient.java | 6 +- .../internal/api/JudgmentSyncClient.java | 6 +- ...st.java => FetchPromptScorersRequest.java} | 21 +- ...e.java => FetchPromptScorersResponse.java} | 21 +- .../internal/api/models/OtelTraceSpan.java | 376 ++++++++++++++++++ .../internal/api/models/ScorerData.java | 21 +- .../judgeval/internal/api/models/Tool.java | 116 ------ .../internal/api/models/TraceSpan.java | 285 ------------- .../internal/api/models/TraceUsage.java | 154 ------- .../scorers/api_scorers/PromptScorer.java | 13 +- 11 files changed, 429 insertions(+), 592 deletions(-) rename src/main/java/com/judgmentlabs/judgeval/internal/api/models/{FetchPromptScorerRequest.java => FetchPromptScorersRequest.java} (67%) rename src/main/java/com/judgmentlabs/judgeval/internal/api/models/{FetchPromptScorerResponse.java => FetchPromptScorersResponse.java} (65%) create mode 100644 src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java delete mode 100644 src/main/java/com/judgmentlabs/judgeval/internal/api/models/Tool.java delete mode 100644 src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceSpan.java delete mode 100644 src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceUsage.java diff --git a/scripts/generate_client.py b/scripts/generate_client.py index 91b1c65..d043351 100755 --- a/scripts/generate_client.py +++ b/scripts/generate_client.py @@ -13,7 +13,7 @@ "/add_to_run_eval_queue/", "/get_evaluation_status/", "/save_scorer/", - "/fetch_scorer/", + "/fetch_scorers/", "/scorer_exists/", "/projects/resolve/", ] diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java index 81895b6..6fed7da 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java @@ -172,9 +172,9 @@ public CompletableFuture saveScorer(SavePromptScorerRe .thenApply(this::handleResponse); } - public CompletableFuture fetchScorer( - FetchPromptScorerRequest payload) { - String url = buildUrl("/fetch_scorer/"); + public CompletableFuture fetchScorers( + FetchPromptScorersRequest payload) { + String url = buildUrl("/fetch_scorers/"); String jsonPayload; try { jsonPayload = mapper.writeValueAsString(payload); diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java index 8c2425f..5ea1b80 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java @@ -151,9 +151,9 @@ public SavePromptScorerResponse saveScorer(SavePromptScorerRequest payload) return mapper.readValue(response.body(), SavePromptScorerResponse.class); } - public FetchPromptScorerResponse fetchScorer(FetchPromptScorerRequest payload) + public FetchPromptScorersResponse fetchScorers(FetchPromptScorersRequest payload) throws IOException, InterruptedException { - String url = buildUrl("/fetch_scorer/"); + String url = buildUrl("/fetch_scorers/"); String jsonPayload = mapper.writeValueAsString(payload); HttpRequest request = HttpRequest.newBuilder() @@ -162,7 +162,7 @@ public FetchPromptScorerResponse fetchScorer(FetchPromptScorerRequest payload) .headers(buildHeaders()) .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); - return mapper.readValue(response.body(), FetchPromptScorerResponse.class); + return mapper.readValue(response.body(), FetchPromptScorersResponse.class); } public ResolveProjectNameResponse projectsResolve(ResolveProjectNameRequest payload) diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorerRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java similarity index 67% rename from src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorerRequest.java rename to src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java index cf5e620..49bd079 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorerRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java @@ -1,6 +1,7 @@ package com.judgmentlabs.judgeval.internal.api.models; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -8,9 +9,9 @@ import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; -public class FetchPromptScorerRequest { - @JsonProperty("name") - private String name; +public class FetchPromptScorersRequest { + @JsonProperty("names") + private List names; private Map additionalProperties = new HashMap<>(); @@ -24,25 +25,25 @@ public void setAdditionalProperty(String name, Object value) { additionalProperties.put(name, value); } - public String getName() { - return name; + public List getNames() { + return names; } - public void setName(String name) { - this.name = name; + public void setNames(List names) { + this.names = names; } @Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null || getClass() != obj.getClass()) return false; - FetchPromptScorerRequest other = (FetchPromptScorerRequest) obj; - return Objects.equals(name, other.name) + FetchPromptScorersRequest other = (FetchPromptScorersRequest) obj; + return Objects.equals(names, other.names) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash(name, Objects.hashCode(additionalProperties)); + return Objects.hash(names, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorerResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java similarity index 65% rename from src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorerResponse.java rename to src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java index 3ca4551..7bfe803 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorerResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java @@ -1,6 +1,7 @@ package com.judgmentlabs.judgeval.internal.api.models; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -8,9 +9,9 @@ import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; -public class FetchPromptScorerResponse { - @JsonProperty("scorer") - private PromptScorer scorer; +public class FetchPromptScorersResponse { + @JsonProperty("scorers") + private List scorers; private Map additionalProperties = new HashMap<>(); @@ -24,25 +25,25 @@ public void setAdditionalProperty(String name, Object value) { additionalProperties.put(name, value); } - public PromptScorer getScorer() { - return scorer; + public List getScorers() { + return scorers; } - public void setScorer(PromptScorer scorer) { - this.scorer = scorer; + public void setScorers(List scorers) { + this.scorers = scorers; } @Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null || getClass() != obj.getClass()) return false; - FetchPromptScorerResponse other = (FetchPromptScorerResponse) obj; - return Objects.equals(scorer, other.scorer) + FetchPromptScorersResponse other = (FetchPromptScorersResponse) obj; + return Objects.equals(scorers, other.scorers) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash(scorer, Objects.hashCode(additionalProperties)); + return Objects.hash(scorers, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java new file mode 100644 index 0000000..9755864 --- /dev/null +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java @@ -0,0 +1,376 @@ +package com.judgmentlabs.judgeval.internal.api.models; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class OtelTraceSpan { + @JsonProperty("organization_id") + private String organizationId; + + @JsonProperty("project_id") + private String projectId; + + @JsonProperty("user_id") + private String userId; + + @JsonProperty("timestamp") + private String timestamp; + + @JsonProperty("trace_id") + private String traceId; + + @JsonProperty("span_id") + private String spanId; + + @JsonProperty("parent_span_id") + private String parentSpanId; + + @JsonProperty("trace_state") + private String traceState; + + @JsonProperty("span_name") + private String spanName; + + @JsonProperty("span_kind") + private String spanKind; + + @JsonProperty("service_name") + private String serviceName; + + @JsonProperty("resource_attributes") + private Object resourceAttributes; + + @JsonProperty("span_attributes") + private Object spanAttributes; + + @JsonProperty("duration") + private Integer duration; + + @JsonProperty("status_code") + private String statusCode; + + @JsonProperty("status_message") + private String statusMessage; + + @JsonProperty("events") + private List events; + + @JsonProperty("links") + private List links; + + @JsonProperty("legacy_span_id") + private String legacySpanId; + + @JsonProperty("inputs") + private Object inputs; + + @JsonProperty("output") + private Object output; + + @JsonProperty("error") + private Object error; + + @JsonProperty("agent_id") + private String agentId; + + @JsonProperty("cumulative_llm_cost") + private Double cumulativeLlmCost; + + @JsonProperty("state_after") + private Object stateAfter; + + @JsonProperty("state_before") + private Object stateBefore; + + private Map additionalProperties = new HashMap<>(); + + @JsonAnyGetter + public Map getAdditionalProperties() { + return additionalProperties; + } + + @JsonAnySetter + public void setAdditionalProperty(String name, Object value) { + additionalProperties.put(name, value); + } + + public String getOrganizationId() { + return organizationId; + } + + public String getProjectId() { + return projectId; + } + + public String getUserId() { + return userId; + } + + public String getTimestamp() { + return timestamp; + } + + public String getTraceId() { + return traceId; + } + + public String getSpanId() { + return spanId; + } + + public String getParentSpanId() { + return parentSpanId; + } + + public String getTraceState() { + return traceState; + } + + public String getSpanName() { + return spanName; + } + + public String getSpanKind() { + return spanKind; + } + + public String getServiceName() { + return serviceName; + } + + public Object getResourceAttributes() { + return resourceAttributes; + } + + public Object getSpanAttributes() { + return spanAttributes; + } + + public Integer getDuration() { + return duration; + } + + public String getStatusCode() { + return statusCode; + } + + public String getStatusMessage() { + return statusMessage; + } + + public List getEvents() { + return events; + } + + public List getLinks() { + return links; + } + + public String getLegacySpanId() { + return legacySpanId; + } + + public Object getInputs() { + return inputs; + } + + public Object getOutput() { + return output; + } + + public Object getError() { + return error; + } + + public String getAgentId() { + return agentId; + } + + public Double getCumulativeLlmCost() { + return cumulativeLlmCost; + } + + public Object getStateAfter() { + return stateAfter; + } + + public Object getStateBefore() { + return stateBefore; + } + + public void setOrganizationId(String organizationId) { + this.organizationId = organizationId; + } + + public void setProjectId(String projectId) { + this.projectId = projectId; + } + + public void setUserId(String userId) { + this.userId = userId; + } + + public void setTimestamp(String timestamp) { + this.timestamp = timestamp; + } + + public void setTraceId(String traceId) { + this.traceId = traceId; + } + + public void setSpanId(String spanId) { + this.spanId = spanId; + } + + public void setParentSpanId(String parentSpanId) { + this.parentSpanId = parentSpanId; + } + + public void setTraceState(String traceState) { + this.traceState = traceState; + } + + public void setSpanName(String spanName) { + this.spanName = spanName; + } + + public void setSpanKind(String spanKind) { + this.spanKind = spanKind; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public void setResourceAttributes(Object resourceAttributes) { + this.resourceAttributes = resourceAttributes; + } + + public void setSpanAttributes(Object spanAttributes) { + this.spanAttributes = spanAttributes; + } + + public void setDuration(Integer duration) { + this.duration = duration; + } + + public void setStatusCode(String statusCode) { + this.statusCode = statusCode; + } + + public void setStatusMessage(String statusMessage) { + this.statusMessage = statusMessage; + } + + public void setEvents(List events) { + this.events = events; + } + + public void setLinks(List links) { + this.links = links; + } + + public void setLegacySpanId(String legacySpanId) { + this.legacySpanId = legacySpanId; + } + + public void setInputs(Object inputs) { + this.inputs = inputs; + } + + public void setOutput(Object output) { + this.output = output; + } + + public void setError(Object error) { + this.error = error; + } + + public void setAgentId(String agentId) { + this.agentId = agentId; + } + + public void setCumulativeLlmCost(Double cumulativeLlmCost) { + this.cumulativeLlmCost = cumulativeLlmCost; + } + + public void setStateAfter(Object stateAfter) { + this.stateAfter = stateAfter; + } + + public void setStateBefore(Object stateBefore) { + this.stateBefore = stateBefore; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + OtelTraceSpan other = (OtelTraceSpan) obj; + return Objects.equals(organizationId, other.organizationId) + && Objects.equals(projectId, other.projectId) + && Objects.equals(userId, other.userId) + && Objects.equals(timestamp, other.timestamp) + && Objects.equals(traceId, other.traceId) + && Objects.equals(spanId, other.spanId) + && Objects.equals(parentSpanId, other.parentSpanId) + && Objects.equals(traceState, other.traceState) + && Objects.equals(spanName, other.spanName) + && Objects.equals(spanKind, other.spanKind) + && Objects.equals(serviceName, other.serviceName) + && Objects.equals(resourceAttributes, other.resourceAttributes) + && Objects.equals(spanAttributes, other.spanAttributes) + && Objects.equals(duration, other.duration) + && Objects.equals(statusCode, other.statusCode) + && Objects.equals(statusMessage, other.statusMessage) + && Objects.equals(events, other.events) + && Objects.equals(links, other.links) + && Objects.equals(legacySpanId, other.legacySpanId) + && Objects.equals(inputs, other.inputs) + && Objects.equals(output, other.output) + && Objects.equals(error, other.error) + && Objects.equals(agentId, other.agentId) + && Objects.equals(cumulativeLlmCost, other.cumulativeLlmCost) + && Objects.equals(stateAfter, other.stateAfter) + && Objects.equals(stateBefore, other.stateBefore) + && Objects.equals(additionalProperties, other.additionalProperties); + } + + @Override + public int hashCode() { + return Objects.hash( + organizationId, + projectId, + userId, + timestamp, + traceId, + spanId, + parentSpanId, + traceState, + spanName, + spanKind, + serviceName, + resourceAttributes, + spanAttributes, + duration, + statusCode, + statusMessage, + events, + links, + legacySpanId, + inputs, + output, + error, + agentId, + cumulativeLlmCost, + stateAfter, + stateBefore, + Objects.hashCode(additionalProperties)); + } +} diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java index 3cf0bcd..62a7419 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java @@ -9,6 +9,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; public class ScorerData { + @JsonProperty("id") + private String id; + @JsonProperty("name") private String name; @@ -28,7 +31,7 @@ public class ScorerData { private Boolean strictMode; @JsonProperty("evaluation_model") - private Object evaluationModel; + private String evaluationModel; @JsonProperty("error") private String error; @@ -48,6 +51,10 @@ public void setAdditionalProperty(String name, Object value) { additionalProperties.put(name, value); } + public String getId() { + return id; + } + public String getName() { return name; } @@ -72,7 +79,7 @@ public Boolean getStrictMode() { return strictMode; } - public Object getEvaluationModel() { + public String getEvaluationModel() { return evaluationModel; } @@ -84,6 +91,10 @@ public Object getAdditionalMetadata() { return additionalMetadata; } + public void setId(String id) { + this.id = id; + } + public void setName(String name) { this.name = name; } @@ -108,7 +119,7 @@ public void setStrictMode(Boolean strictMode) { this.strictMode = strictMode; } - public void setEvaluationModel(Object evaluationModel) { + public void setEvaluationModel(String evaluationModel) { this.evaluationModel = evaluationModel; } @@ -125,7 +136,8 @@ public boolean equals(Object obj) { if (this == obj) return true; if (obj == null || getClass() != obj.getClass()) return false; ScorerData other = (ScorerData) obj; - return Objects.equals(name, other.name) + return Objects.equals(id, other.id) + && Objects.equals(name, other.name) && Objects.equals(threshold, other.threshold) && Objects.equals(success, other.success) && Objects.equals(score, other.score) @@ -140,6 +152,7 @@ public boolean equals(Object obj) { @Override public int hashCode() { return Objects.hash( + id, name, threshold, success, diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Tool.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Tool.java deleted file mode 100644 index d6b852d..0000000 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Tool.java +++ /dev/null @@ -1,116 +0,0 @@ -package com.judgmentlabs.judgeval.internal.api.models; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonProperty; - -public class Tool { - @JsonProperty("tool_name") - private String toolName; - - @JsonProperty("parameters") - private Object parameters; - - @JsonProperty("agent_name") - private String agentName; - - @JsonProperty("result_dependencies") - private List resultDependencies; - - @JsonProperty("action_dependencies") - private List actionDependencies; - - @JsonProperty("require_all") - private Boolean requireAll; - - private Map additionalProperties = new HashMap<>(); - - @JsonAnyGetter - public Map getAdditionalProperties() { - return additionalProperties; - } - - @JsonAnySetter - public void setAdditionalProperty(String name, Object value) { - additionalProperties.put(name, value); - } - - public String getToolName() { - return toolName; - } - - public Object getParameters() { - return parameters; - } - - public String getAgentName() { - return agentName; - } - - public List getResultDependencies() { - return resultDependencies; - } - - public List getActionDependencies() { - return actionDependencies; - } - - public Boolean getRequireAll() { - return requireAll; - } - - public void setToolName(String toolName) { - this.toolName = toolName; - } - - public void setParameters(Object parameters) { - this.parameters = parameters; - } - - public void setAgentName(String agentName) { - this.agentName = agentName; - } - - public void setResultDependencies(List resultDependencies) { - this.resultDependencies = resultDependencies; - } - - public void setActionDependencies(List actionDependencies) { - this.actionDependencies = actionDependencies; - } - - public void setRequireAll(Boolean requireAll) { - this.requireAll = requireAll; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - Tool other = (Tool) obj; - return Objects.equals(toolName, other.toolName) - && Objects.equals(parameters, other.parameters) - && Objects.equals(agentName, other.agentName) - && Objects.equals(resultDependencies, other.resultDependencies) - && Objects.equals(actionDependencies, other.actionDependencies) - && Objects.equals(requireAll, other.requireAll) - && Objects.equals(additionalProperties, other.additionalProperties); - } - - @Override - public int hashCode() { - return Objects.hash( - toolName, - parameters, - agentName, - resultDependencies, - actionDependencies, - requireAll, - Objects.hashCode(additionalProperties)); - } -} diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceSpan.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceSpan.java deleted file mode 100644 index 9689d99..0000000 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceSpan.java +++ /dev/null @@ -1,285 +0,0 @@ -package com.judgmentlabs.judgeval.internal.api.models; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonProperty; - -public class TraceSpan { - @JsonProperty("span_id") - private String spanId; - - @JsonProperty("trace_id") - private String traceId; - - @JsonProperty("function") - private String function; - - @JsonProperty("created_at") - private Object createdAt; - - @JsonProperty("parent_span_id") - private String parentSpanId; - - @JsonProperty("span_type") - private String spanType; - - @JsonProperty("inputs") - private Object inputs; - - @JsonProperty("error") - private Object error; - - @JsonProperty("output") - private Object output; - - @JsonProperty("usage") - private TraceUsage usage; - - @JsonProperty("duration") - private Double duration; - - @JsonProperty("expected_tools") - private List expectedTools; - - @JsonProperty("additional_metadata") - private Object additionalMetadata; - - @JsonProperty("has_evaluation") - private Boolean hasEvaluation; - - @JsonProperty("agent_name") - private String agentName; - - @JsonProperty("class_name") - private String className; - - @JsonProperty("state_before") - private Object stateBefore; - - @JsonProperty("state_after") - private Object stateAfter; - - @JsonProperty("update_id") - private Integer updateId; - - private Map additionalProperties = new HashMap<>(); - - @JsonAnyGetter - public Map getAdditionalProperties() { - return additionalProperties; - } - - @JsonAnySetter - public void setAdditionalProperty(String name, Object value) { - additionalProperties.put(name, value); - } - - public String getSpanId() { - return spanId; - } - - public String getTraceId() { - return traceId; - } - - public String getFunction() { - return function; - } - - public Object getCreatedAt() { - return createdAt; - } - - public String getParentSpanId() { - return parentSpanId; - } - - public String getSpanType() { - return spanType; - } - - public Object getInputs() { - return inputs; - } - - public Object getError() { - return error; - } - - public Object getOutput() { - return output; - } - - public TraceUsage getUsage() { - return usage; - } - - public Double getDuration() { - return duration; - } - - public List getExpectedTools() { - return expectedTools; - } - - public Object getAdditionalMetadata() { - return additionalMetadata; - } - - public Boolean getHasEvaluation() { - return hasEvaluation; - } - - public String getAgentName() { - return agentName; - } - - public String getClassName() { - return className; - } - - public Object getStateBefore() { - return stateBefore; - } - - public Object getStateAfter() { - return stateAfter; - } - - public Integer getUpdateId() { - return updateId; - } - - public void setSpanId(String spanId) { - this.spanId = spanId; - } - - public void setTraceId(String traceId) { - this.traceId = traceId; - } - - public void setFunction(String function) { - this.function = function; - } - - public void setCreatedAt(Object createdAt) { - this.createdAt = createdAt; - } - - public void setParentSpanId(String parentSpanId) { - this.parentSpanId = parentSpanId; - } - - public void setSpanType(String spanType) { - this.spanType = spanType; - } - - public void setInputs(Object inputs) { - this.inputs = inputs; - } - - public void setError(Object error) { - this.error = error; - } - - public void setOutput(Object output) { - this.output = output; - } - - public void setUsage(TraceUsage usage) { - this.usage = usage; - } - - public void setDuration(Double duration) { - this.duration = duration; - } - - public void setExpectedTools(List expectedTools) { - this.expectedTools = expectedTools; - } - - public void setAdditionalMetadata(Object additionalMetadata) { - this.additionalMetadata = additionalMetadata; - } - - public void setHasEvaluation(Boolean hasEvaluation) { - this.hasEvaluation = hasEvaluation; - } - - public void setAgentName(String agentName) { - this.agentName = agentName; - } - - public void setClassName(String className) { - this.className = className; - } - - public void setStateBefore(Object stateBefore) { - this.stateBefore = stateBefore; - } - - public void setStateAfter(Object stateAfter) { - this.stateAfter = stateAfter; - } - - public void setUpdateId(Integer updateId) { - this.updateId = updateId; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - TraceSpan other = (TraceSpan) obj; - return Objects.equals(spanId, other.spanId) - && Objects.equals(traceId, other.traceId) - && Objects.equals(function, other.function) - && Objects.equals(createdAt, other.createdAt) - && Objects.equals(parentSpanId, other.parentSpanId) - && Objects.equals(spanType, other.spanType) - && Objects.equals(inputs, other.inputs) - && Objects.equals(error, other.error) - && Objects.equals(output, other.output) - && Objects.equals(usage, other.usage) - && Objects.equals(duration, other.duration) - && Objects.equals(expectedTools, other.expectedTools) - && Objects.equals(additionalMetadata, other.additionalMetadata) - && Objects.equals(hasEvaluation, other.hasEvaluation) - && Objects.equals(agentName, other.agentName) - && Objects.equals(className, other.className) - && Objects.equals(stateBefore, other.stateBefore) - && Objects.equals(stateAfter, other.stateAfter) - && Objects.equals(updateId, other.updateId) - && Objects.equals(additionalProperties, other.additionalProperties); - } - - @Override - public int hashCode() { - return Objects.hash( - spanId, - traceId, - function, - createdAt, - parentSpanId, - spanType, - inputs, - error, - output, - usage, - duration, - expectedTools, - additionalMetadata, - hasEvaluation, - agentName, - className, - stateBefore, - stateAfter, - updateId, - Objects.hashCode(additionalProperties)); - } -} diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceUsage.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceUsage.java deleted file mode 100644 index 73d6d3e..0000000 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceUsage.java +++ /dev/null @@ -1,154 +0,0 @@ -package com.judgmentlabs.judgeval.internal.api.models; - -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonProperty; - -public class TraceUsage { - @JsonProperty("prompt_tokens") - private Integer promptTokens; - - @JsonProperty("completion_tokens") - private Integer completionTokens; - - @JsonProperty("cache_creation_input_tokens") - private Integer cacheCreationInputTokens; - - @JsonProperty("cache_read_input_tokens") - private Integer cacheReadInputTokens; - - @JsonProperty("total_tokens") - private Integer totalTokens; - - @JsonProperty("prompt_tokens_cost_usd") - private Double promptTokensCostUsd; - - @JsonProperty("completion_tokens_cost_usd") - private Double completionTokensCostUsd; - - @JsonProperty("total_cost_usd") - private Double totalCostUsd; - - @JsonProperty("model_name") - private String modelName; - - private Map additionalProperties = new HashMap<>(); - - @JsonAnyGetter - public Map getAdditionalProperties() { - return additionalProperties; - } - - @JsonAnySetter - public void setAdditionalProperty(String name, Object value) { - additionalProperties.put(name, value); - } - - public Integer getPromptTokens() { - return promptTokens; - } - - public Integer getCompletionTokens() { - return completionTokens; - } - - public Integer getCacheCreationInputTokens() { - return cacheCreationInputTokens; - } - - public Integer getCacheReadInputTokens() { - return cacheReadInputTokens; - } - - public Integer getTotalTokens() { - return totalTokens; - } - - public Double getPromptTokensCostUsd() { - return promptTokensCostUsd; - } - - public Double getCompletionTokensCostUsd() { - return completionTokensCostUsd; - } - - public Double getTotalCostUsd() { - return totalCostUsd; - } - - public String getModelName() { - return modelName; - } - - public void setPromptTokens(Integer promptTokens) { - this.promptTokens = promptTokens; - } - - public void setCompletionTokens(Integer completionTokens) { - this.completionTokens = completionTokens; - } - - public void setCacheCreationInputTokens(Integer cacheCreationInputTokens) { - this.cacheCreationInputTokens = cacheCreationInputTokens; - } - - public void setCacheReadInputTokens(Integer cacheReadInputTokens) { - this.cacheReadInputTokens = cacheReadInputTokens; - } - - public void setTotalTokens(Integer totalTokens) { - this.totalTokens = totalTokens; - } - - public void setPromptTokensCostUsd(Double promptTokensCostUsd) { - this.promptTokensCostUsd = promptTokensCostUsd; - } - - public void setCompletionTokensCostUsd(Double completionTokensCostUsd) { - this.completionTokensCostUsd = completionTokensCostUsd; - } - - public void setTotalCostUsd(Double totalCostUsd) { - this.totalCostUsd = totalCostUsd; - } - - public void setModelName(String modelName) { - this.modelName = modelName; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - TraceUsage other = (TraceUsage) obj; - return Objects.equals(promptTokens, other.promptTokens) - && Objects.equals(completionTokens, other.completionTokens) - && Objects.equals(cacheCreationInputTokens, other.cacheCreationInputTokens) - && Objects.equals(cacheReadInputTokens, other.cacheReadInputTokens) - && Objects.equals(totalTokens, other.totalTokens) - && Objects.equals(promptTokensCostUsd, other.promptTokensCostUsd) - && Objects.equals(completionTokensCostUsd, other.completionTokensCostUsd) - && Objects.equals(totalCostUsd, other.totalCostUsd) - && Objects.equals(modelName, other.modelName) - && Objects.equals(additionalProperties, other.additionalProperties); - } - - @Override - public int hashCode() { - return Objects.hash( - promptTokens, - completionTokens, - cacheCreationInputTokens, - cacheReadInputTokens, - totalTokens, - promptTokensCostUsd, - completionTokensCostUsd, - totalCostUsd, - modelName, - Objects.hashCode(additionalProperties)); - } -} diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/PromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/PromptScorer.java index bd15af2..0744ab8 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/PromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/PromptScorer.java @@ -1,6 +1,7 @@ package com.judgmentlabs.judgeval.scorers.api_scorers; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -10,8 +11,8 @@ import com.judgmentlabs.judgeval.data.APIScorerType; import com.judgmentlabs.judgeval.exceptions.JudgmentAPIError; import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; -import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorerRequest; -import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorerResponse; +import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersRequest; +import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersResponse; import com.judgmentlabs.judgeval.internal.api.models.SavePromptScorerRequest; import com.judgmentlabs.judgeval.internal.api.models.SavePromptScorerResponse; import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; @@ -76,12 +77,12 @@ public static PromptScorer get(String name, String judgmentApiKey, String organi try { JudgmentSyncClient client = new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); - FetchPromptScorerRequest request = new FetchPromptScorerRequest(); - request.setName(name); + FetchPromptScorersRequest request = new FetchPromptScorersRequest(); + request.setNames(Collections.singletonList(name)); - FetchPromptScorerResponse response = client.fetchScorer(request); + FetchPromptScorersResponse response = client.fetchScorers(request); com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorerConfig = - response.getScorer(); + response.getScorers().get(0); return new PromptScorer( client, From 654279f5e9ae7ac54da074c0b848651fa5771a25 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 17 Sep 2025 17:47:56 -0700 Subject: [PATCH 03/10] chore: updated prompt scorers --- .../judgeval/data/APIScorerType.java | 1 + .../scorers/api_scorers/PromptScorer.java | 293 ------------------ .../prompt_scorer/BasePromptScorer.java | 219 +++++++++++++ .../prompt_scorer/PromptScorer.java | 204 ++++++++++++ .../prompt_scorer/TracePromptScorer.java | 206 ++++++++++++ 5 files changed, 630 insertions(+), 293 deletions(-) delete mode 100644 src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/PromptScorer.java create mode 100644 src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java create mode 100644 src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java create mode 100644 src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java diff --git a/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java b/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java index d06b13c..49e9af2 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java @@ -2,6 +2,7 @@ public enum APIScorerType { PROMPT_SCORER("Prompt Scorer"), + TRACE_PROMPT_SCORER("Trace Prompt Scorer"), FAITHFULNESS("Faithfulness"), ANSWER_RELEVANCY("Answer Relevancy"), ANSWER_CORRECTNESS("Answer Correctness"), diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/PromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/PromptScorer.java deleted file mode 100644 index 0744ab8..0000000 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/PromptScorer.java +++ /dev/null @@ -1,293 +0,0 @@ -package com.judgmentlabs.judgeval.scorers.api_scorers; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.judgmentlabs.judgeval.Env; -import com.judgmentlabs.judgeval.data.APIScorerType; -import com.judgmentlabs.judgeval.exceptions.JudgmentAPIError; -import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; -import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersRequest; -import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersResponse; -import com.judgmentlabs.judgeval.internal.api.models.SavePromptScorerRequest; -import com.judgmentlabs.judgeval.internal.api.models.SavePromptScorerResponse; -import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; -import com.judgmentlabs.judgeval.internal.api.models.ScorerExistsRequest; -import com.judgmentlabs.judgeval.internal.api.models.ScorerExistsResponse; -import com.judgmentlabs.judgeval.scorers.APIScorer; - -public class PromptScorer extends APIScorer { - @JsonProperty("prompt") - private String prompt; - - @JsonProperty("options") - private Map options; - - @JsonIgnore private JudgmentSyncClient client; - - public PromptScorer(String name, String prompt, double threshold, Map options) { - this( - new JudgmentSyncClient( - Env.JUDGMENT_API_URL, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID), - name, - prompt, - threshold, - options); - } - - public PromptScorer( - JudgmentSyncClient client, - String name, - String prompt, - double threshold, - Map options) { - super(APIScorerType.PROMPT_SCORER); - this.client = client; - setName(name); - this.prompt = prompt; - this.options = options; - setThreshold(threshold); - } - - @Override - public Object toTransport() { - ScorerConfig cfg = new ScorerConfig(); - cfg.setScoreType(getScoreType()); - cfg.setThreshold(getThreshold()); - cfg.setName(getName()); - cfg.setStrictMode(isStrictMode()); - cfg.setRequiredParams(getRequiredParams()); - Map kwargs = new HashMap<>(); - kwargs.put("prompt", prompt); - if (options != null) kwargs.put("options", options); - if (getAdditionalProperties() != null) kwargs.putAll(getAdditionalProperties()); - cfg.setKwargs(kwargs); - return cfg; - } - - public static PromptScorer get(String name) { - return get(name, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); - } - - public static PromptScorer get(String name, String judgmentApiKey, String organizationId) { - try { - JudgmentSyncClient client = - new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); - FetchPromptScorersRequest request = new FetchPromptScorersRequest(); - request.setNames(Collections.singletonList(name)); - - FetchPromptScorersResponse response = client.fetchScorers(request); - com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorerConfig = - response.getScorers().get(0); - - return new PromptScorer( - client, - name, - scorerConfig.getPrompt(), - scorerConfig.getThreshold(), - (Map) scorerConfig.getOptions()); - } catch (IOException | InterruptedException e) { - throw new JudgmentAPIError(500, "Failed to fetch prompt scorer: " + e.getMessage()); - } - } - - public static PromptScorer create(String name, String prompt) { - return create(name, prompt, 0.5, null); - } - - public static PromptScorer create(String name, String prompt, double threshold) { - return create(name, prompt, threshold, null); - } - - public static PromptScorer create( - String name, String prompt, double threshold, Map options) { - return create(name, prompt, threshold, options, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); - } - - public static PromptScorer create( - String name, - String prompt, - double threshold, - Map options, - String judgmentApiKey, - String organizationId) { - try { - JudgmentSyncClient client = - new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); - - ScorerExistsRequest existsRequest = new ScorerExistsRequest(); - existsRequest.setName(name); - - ScorerExistsResponse existsResponse = client.scorerExists(existsRequest); - boolean exists = Boolean.TRUE.equals(existsResponse.getExists()); - - if (exists) { - throw new JudgmentAPIError( - 400, - "Scorer with name " - + name - + " already exists. Either use the existing scorer with the get() method or use a new name."); - } - - pushPromptScorer(client, name, prompt, threshold, options); - return new PromptScorer(client, name, prompt, threshold, options); - } catch (IOException | InterruptedException e) { - throw new JudgmentAPIError(500, "Failed to create prompt scorer: " + e.getMessage()); - } - } - - public void setThreshold(double threshold) { - super.setThreshold(threshold); - pushPromptScorer(); - } - - public void setPrompt(String prompt) { - this.prompt = prompt; - pushPromptScorer(); - } - - public void setOptions(Map options) { - this.options = options; - pushPromptScorer(); - } - - public void appendToPrompt(String promptAddition) { - this.prompt += promptAddition; - pushPromptScorer(); - } - - public Double getThreshold() { - return super.getThreshold(); - } - - public String getPrompt() { - return prompt; - } - - public Map getOptions() { - return options != null ? new HashMap<>(options) : null; - } - - public String getScorerName() { - return getName(); - } - - public Map getConfig() { - Map config = new HashMap<>(); - config.put("name", getName()); - config.put("prompt", prompt); - config.put("threshold", getThreshold()); - config.put("options", options); - return config; - } - - private void pushPromptScorer() { - pushPromptScorer(this.client, getName(), prompt, getThreshold(), options); - } - - private static String pushPromptScorer( - JudgmentSyncClient client, - String name, - String prompt, - double threshold, - Map options) { - try { - SavePromptScorerRequest request = new SavePromptScorerRequest(); - request.setName(name); - request.setPrompt(prompt); - request.setThreshold(threshold); - request.setOptions(options); - request.setIsTrace(false); - - SavePromptScorerResponse response = client.saveScorer(request); - return response != null ? response.getName() : null; - } catch (IOException | InterruptedException e) { - throw new JudgmentAPIError(500, "Failed to save prompt scorer: " + e.getMessage()); - } - } - - @Override - public String toString() { - return "PromptScorer(name=" - + getName() - + ", prompt=" - + prompt - + ", threshold=" - + getThreshold() - + ", options=" - + options - + ")"; - } - - public static Builder builder() { - return new Builder(); - } - - public static Builder builder(String name, String prompt) { - return new Builder(name, prompt); - } - - public static final class Builder { - private String name; - private String prompt; - private double threshold = 0.5; - private Map options; - private JudgmentSyncClient client; - - private Builder() {} - - private Builder(String name, String prompt) { - this.name = name; - this.prompt = prompt; - } - - public Builder name(String name) { - this.name = name; - return this; - } - - public Builder prompt(String prompt) { - this.prompt = prompt; - return this; - } - - public Builder threshold(double threshold) { - this.threshold = threshold; - return this; - } - - public Builder options(Map options) { - this.options = options; - return this; - } - - public Builder option(String key, Double value) { - if (this.options == null) { - this.options = new HashMap<>(); - } - this.options.put(key, value); - return this; - } - - public Builder client(JudgmentSyncClient client) { - this.client = client; - return this; - } - - public PromptScorer build() { - if (name == null || prompt == null) { - throw new IllegalArgumentException("Name and prompt are required"); - } - - if (client != null) { - return new PromptScorer(client, name, prompt, threshold, options); - } else { - return new PromptScorer(name, prompt, threshold, options); - } - } - } -} diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java new file mode 100644 index 0000000..e0d666f --- /dev/null +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java @@ -0,0 +1,219 @@ +package com.judgmentlabs.judgeval.scorers.api_scorers.prompt_scorer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import com.judgmentlabs.judgeval.Env; +import com.judgmentlabs.judgeval.data.APIScorerType; +import com.judgmentlabs.judgeval.exceptions.JudgmentAPIError; +import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; +import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersRequest; +import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersResponse; +import com.judgmentlabs.judgeval.internal.api.models.SavePromptScorerRequest; +import com.judgmentlabs.judgeval.internal.api.models.SavePromptScorerResponse; +import com.judgmentlabs.judgeval.internal.api.models.ScorerExistsRequest; +import com.judgmentlabs.judgeval.internal.api.models.ScorerExistsResponse; +import com.judgmentlabs.judgeval.scorers.APIScorer; +import com.judgmentlabs.judgeval.utils.Logger; + +public abstract class BasePromptScorer extends APIScorer { + protected String prompt; + protected Map options; + protected String judgmentApiKey; + protected String organizationId; + + protected BasePromptScorer( + APIScorerType scoreType, + String name, + String prompt, + double threshold, + Map options, + String judgmentApiKey, + String organizationId) { + super(scoreType); + this.prompt = prompt; + this.options = options; + this.judgmentApiKey = judgmentApiKey; + this.organizationId = organizationId; + setName(name); + setThreshold(threshold); + } + + public static boolean scorerExists(String name, String judgmentApiKey, String organizationId) { + try { + JudgmentSyncClient client = + new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); + ScorerExistsRequest request = new ScorerExistsRequest(); + request.setName(name); + ScorerExistsResponse response = client.scorerExists(request); + return Boolean.TRUE.equals(response.getExists()); + } catch (JudgmentAPIError e) { + if (e.getStatusCode() == 500) { + throw new JudgmentAPIError( + e.getStatusCode(), + "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " + + e.getMessage()); + } + throw new JudgmentAPIError( + e.getStatusCode(), "Failed to check if scorer exists: " + e.getMessage()); + } catch (IOException | InterruptedException e) { + throw new JudgmentAPIError( + 500, + "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " + + e.getMessage()); + } + } + + public static com.judgmentlabs.judgeval.internal.api.models.PromptScorer fetchPromptScorer( + String name, String judgmentApiKey, String organizationId) { + try { + JudgmentSyncClient client = + new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); + FetchPromptScorersRequest request = new FetchPromptScorersRequest(); + request.setNames(java.util.Collections.singletonList(name)); + + FetchPromptScorersResponse response = client.fetchScorers(request); + + if (response.getScorers() == null || response.getScorers().isEmpty()) { + throw new JudgmentAPIError( + 404, "Failed to fetch prompt scorer '" + name + "': not found"); + } + + return response.getScorers().get(0); + } catch (JudgmentAPIError e) { + if (e.getStatusCode() == 500) { + throw new JudgmentAPIError( + e.getStatusCode(), + "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " + + e.getMessage()); + } + throw new JudgmentAPIError( + e.getStatusCode(), + "Failed to fetch prompt scorer '" + name + "': " + e.getMessage()); + } catch (IOException | InterruptedException e) { + throw new JudgmentAPIError( + 500, + "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " + + e.getMessage()); + } + } + + public static String pushPromptScorer( + String name, + String prompt, + double threshold, + Map options, + String judgmentApiKey, + String organizationId, + Boolean isTrace) { + try { + JudgmentSyncClient client = + new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); + SavePromptScorerRequest request = new SavePromptScorerRequest(); + request.setName(name); + request.setPrompt(prompt); + request.setThreshold(threshold); + Map apiOptions = null; + if (options != null) { + apiOptions = new HashMap<>(); + for (Map.Entry entry : options.entrySet()) { + apiOptions.put(entry.getKey(), entry.getValue()); + } + } + request.setOptions(apiOptions); + request.setIsTrace(isTrace); + + SavePromptScorerResponse response = client.saveScorer(request); + return response != null ? response.getName() : null; + } catch (JudgmentAPIError e) { + if (e.getStatusCode() == 500) { + throw new JudgmentAPIError( + e.getStatusCode(), + "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " + + e.getMessage()); + } + throw new JudgmentAPIError( + e.getStatusCode(), "Failed to save prompt scorer: " + e.getMessage()); + } catch (IOException | InterruptedException e) { + throw new JudgmentAPIError( + 500, + "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " + + e.getMessage()); + } + } + + public void setThreshold(double threshold) { + super.setThreshold(threshold); + pushPromptScorer(); + } + + public void setPrompt(String prompt) { + this.prompt = prompt; + pushPromptScorer(); + Logger.info("Successfully updated prompt for " + getName()); + } + + public void setOptions(Map options) { + this.options = options; + pushPromptScorer(); + Logger.info("Successfully updated options for " + getName()); + } + + public void appendToPrompt(String promptAddition) { + this.prompt += promptAddition; + pushPromptScorer(); + Logger.info("Successfully appended to prompt for " + getName()); + } + + public Double getThreshold() { + return super.getThreshold(); + } + + public String getPrompt() { + return prompt; + } + + public Map getOptions() { + return options != null ? new HashMap<>(options) : null; + } + + public String getScorerName() { + return getName(); + } + + public Map getConfig() { + Map config = new HashMap<>(); + config.put("name", getName()); + config.put("prompt", prompt); + config.put("threshold", getThreshold()); + config.put("options", options); + return config; + } + + protected void pushPromptScorer() { + pushPromptScorer( + getName(), + prompt, + getThreshold(), + options, + judgmentApiKey, + organizationId, + isTrace()); + } + + protected abstract boolean isTrace(); + + @Override + public String toString() { + return "PromptScorer(name=" + + getName() + + ", prompt=" + + prompt + + ", threshold=" + + getThreshold() + + ", options=" + + options + + ")"; + } +} diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java new file mode 100644 index 0000000..857decd --- /dev/null +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java @@ -0,0 +1,204 @@ +package com.judgmentlabs.judgeval.scorers.api_scorers.prompt_scorer; + +import java.util.HashMap; +import java.util.Map; + +import com.judgmentlabs.judgeval.Env; +import com.judgmentlabs.judgeval.data.APIScorerType; +import com.judgmentlabs.judgeval.exceptions.JudgmentAPIError; +import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; + +public class PromptScorer extends BasePromptScorer { + + public PromptScorer(String name, String prompt, double threshold, Map options) { + super( + APIScorerType.PROMPT_SCORER, + name, + prompt, + threshold, + options, + Env.JUDGMENT_API_KEY, + Env.JUDGMENT_ORG_ID); + } + + public PromptScorer( + String name, + String prompt, + double threshold, + Map options, + String judgmentApiKey, + String organizationId) { + super( + APIScorerType.PROMPT_SCORER, + name, + prompt, + threshold, + options, + judgmentApiKey, + organizationId); + } + + public static PromptScorer get(String name) { + return get(name, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); + } + + public static PromptScorer get(String name, String judgmentApiKey, String organizationId) { + com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorerConfig = + fetchPromptScorer(name, judgmentApiKey, organizationId); + + if (Boolean.TRUE.equals(scorerConfig.getIsTrace())) { + throw new JudgmentAPIError(400, "Scorer with name " + name + " is not a PromptScorer"); + } + + Map options = null; + if (scorerConfig.getOptions() != null) { + if (scorerConfig.getOptions() instanceof Map) { + @SuppressWarnings("unchecked") + Map rawOptions = (Map) scorerConfig.getOptions(); + options = new HashMap<>(); + for (Map.Entry entry : rawOptions.entrySet()) { + if (entry.getValue() instanceof Number) { + options.put(entry.getKey(), ((Number) entry.getValue()).doubleValue()); + } + } + } + } + + return new PromptScorer( + name, + scorerConfig.getPrompt(), + scorerConfig.getThreshold() != null ? scorerConfig.getThreshold() : 0.5, + options, + judgmentApiKey, + organizationId); + } + + public static PromptScorer create(String name, String prompt) { + return create(name, prompt, 0.5, null); + } + + public static PromptScorer create(String name, String prompt, double threshold) { + return create(name, prompt, threshold, null); + } + + public static PromptScorer create( + String name, String prompt, double threshold, Map options) { + return create(name, prompt, threshold, options, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); + } + + public static PromptScorer create( + String name, + String prompt, + double threshold, + Map options, + String judgmentApiKey, + String organizationId) { + if (!scorerExists(name, judgmentApiKey, organizationId)) { + pushPromptScorer( + name, prompt, threshold, options, judgmentApiKey, organizationId, false); + return new PromptScorer( + name, prompt, threshold, options, judgmentApiKey, organizationId); + } else { + throw new JudgmentAPIError( + 400, + "Scorer with name " + + name + + " already exists. Either use the existing scorer with the get() method or use a new name."); + } + } + + @Override + public Object toTransport() { + ScorerConfig cfg = new ScorerConfig(); + cfg.setScoreType(getScoreType()); + cfg.setThreshold(getThreshold()); + cfg.setName(getName()); + cfg.setStrictMode(isStrictMode()); + cfg.setRequiredParams(getRequiredParams()); + Map kwargs = new HashMap<>(); + kwargs.put("prompt", prompt); + if (options != null) { + kwargs.put("options", options); + } + if (getAdditionalProperties() != null) { + kwargs.putAll(getAdditionalProperties()); + } + cfg.setKwargs(kwargs); + return cfg; + } + + @Override + protected boolean isTrace() { + return false; + } + + public static Builder builder() { + return new Builder(); + } + + public static Builder builder(String name, String prompt) { + return new Builder(name, prompt); + } + + public static final class Builder { + private String name; + private String prompt; + private double threshold = 0.5; + private Map options; + private String judgmentApiKey = Env.JUDGMENT_API_KEY; + private String organizationId = Env.JUDGMENT_ORG_ID; + + private Builder() {} + + private Builder(String name, String prompt) { + this.name = name; + this.prompt = prompt; + } + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder prompt(String prompt) { + this.prompt = prompt; + return this; + } + + public Builder threshold(double threshold) { + this.threshold = threshold; + return this; + } + + public Builder options(Map options) { + this.options = options; + return this; + } + + public Builder option(String key, Double value) { + if (this.options == null) { + this.options = new HashMap<>(); + } + this.options.put(key, value); + return this; + } + + public Builder judgmentApiKey(String judgmentApiKey) { + this.judgmentApiKey = judgmentApiKey; + return this; + } + + public Builder organizationId(String organizationId) { + this.organizationId = organizationId; + return this; + } + + public PromptScorer build() { + if (name == null || prompt == null) { + throw new IllegalArgumentException("Name and prompt are required"); + } + return new PromptScorer( + name, prompt, threshold, options, judgmentApiKey, organizationId); + } + } +} diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java new file mode 100644 index 0000000..7b68516 --- /dev/null +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java @@ -0,0 +1,206 @@ +package com.judgmentlabs.judgeval.scorers.api_scorers.prompt_scorer; + +import java.util.HashMap; +import java.util.Map; + +import com.judgmentlabs.judgeval.Env; +import com.judgmentlabs.judgeval.data.APIScorerType; +import com.judgmentlabs.judgeval.exceptions.JudgmentAPIError; +import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; + +public class TracePromptScorer extends BasePromptScorer { + + public TracePromptScorer( + String name, String prompt, double threshold, Map options) { + super( + APIScorerType.TRACE_PROMPT_SCORER, + name, + prompt, + threshold, + options, + Env.JUDGMENT_API_KEY, + Env.JUDGMENT_ORG_ID); + } + + public TracePromptScorer( + String name, + String prompt, + double threshold, + Map options, + String judgmentApiKey, + String organizationId) { + super( + APIScorerType.TRACE_PROMPT_SCORER, + name, + prompt, + threshold, + options, + judgmentApiKey, + organizationId); + } + + public static TracePromptScorer get(String name) { + return get(name, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); + } + + public static TracePromptScorer get(String name, String judgmentApiKey, String organizationId) { + com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorerConfig = + fetchPromptScorer(name, judgmentApiKey, organizationId); + + if (!Boolean.TRUE.equals(scorerConfig.getIsTrace())) { + throw new JudgmentAPIError( + 400, "Scorer with name " + name + " is not a TracePromptScorer"); + } + + Map options = null; + if (scorerConfig.getOptions() != null) { + if (scorerConfig.getOptions() instanceof Map) { + @SuppressWarnings("unchecked") + Map rawOptions = (Map) scorerConfig.getOptions(); + options = new HashMap<>(); + for (Map.Entry entry : rawOptions.entrySet()) { + if (entry.getValue() instanceof Number) { + options.put(entry.getKey(), ((Number) entry.getValue()).doubleValue()); + } + } + } + } + + return new TracePromptScorer( + name, + scorerConfig.getPrompt(), + scorerConfig.getThreshold() != null ? scorerConfig.getThreshold() : 0.5, + options, + judgmentApiKey, + organizationId); + } + + public static TracePromptScorer create(String name, String prompt) { + return create(name, prompt, 0.5, null); + } + + public static TracePromptScorer create(String name, String prompt, double threshold) { + return create(name, prompt, threshold, null); + } + + public static TracePromptScorer create( + String name, String prompt, double threshold, Map options) { + return create(name, prompt, threshold, options, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); + } + + public static TracePromptScorer create( + String name, + String prompt, + double threshold, + Map options, + String judgmentApiKey, + String organizationId) { + if (!scorerExists(name, judgmentApiKey, organizationId)) { + pushPromptScorer( + name, prompt, threshold, options, judgmentApiKey, organizationId, true); + return new TracePromptScorer( + name, prompt, threshold, options, judgmentApiKey, organizationId); + } else { + throw new JudgmentAPIError( + 400, + "Scorer with name " + + name + + " already exists. Either use the existing scorer with the get() method or use a new name."); + } + } + + @Override + public Object toTransport() { + ScorerConfig cfg = new ScorerConfig(); + cfg.setScoreType(getScoreType()); + cfg.setThreshold(getThreshold()); + cfg.setName(getName()); + cfg.setStrictMode(isStrictMode()); + cfg.setRequiredParams(getRequiredParams()); + Map kwargs = new HashMap<>(); + kwargs.put("prompt", prompt); + if (options != null) { + kwargs.put("options", options); + } + if (getAdditionalProperties() != null) { + kwargs.putAll(getAdditionalProperties()); + } + cfg.setKwargs(kwargs); + return cfg; + } + + @Override + protected boolean isTrace() { + return true; + } + + public static Builder builder() { + return new Builder(); + } + + public static Builder builder(String name, String prompt) { + return new Builder(name, prompt); + } + + public static final class Builder { + private String name; + private String prompt; + private double threshold = 0.5; + private Map options; + private String judgmentApiKey = Env.JUDGMENT_API_KEY; + private String organizationId = Env.JUDGMENT_ORG_ID; + + private Builder() {} + + private Builder(String name, String prompt) { + this.name = name; + this.prompt = prompt; + } + + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder prompt(String prompt) { + this.prompt = prompt; + return this; + } + + public Builder threshold(double threshold) { + this.threshold = threshold; + return this; + } + + public Builder options(Map options) { + this.options = options; + return this; + } + + public Builder option(String key, Double value) { + if (this.options == null) { + this.options = new HashMap<>(); + } + this.options.put(key, value); + return this; + } + + public Builder judgmentApiKey(String judgmentApiKey) { + this.judgmentApiKey = judgmentApiKey; + return this; + } + + public Builder organizationId(String organizationId) { + this.organizationId = organizationId; + return this; + } + + public TracePromptScorer build() { + if (name == null || prompt == null) { + throw new IllegalArgumentException("Name and prompt are required"); + } + return new TracePromptScorer( + name, prompt, threshold, options, judgmentApiKey, organizationId); + } + } +} From 4a9b5df9000b2a9b2f5c5b43349f91757d82a3c5 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 17 Sep 2025 18:16:53 -0700 Subject: [PATCH 04/10] chore: match example behavior: --- src/main/java/com/judgmentlabs/judgeval/data/Example.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/data/Example.java b/src/main/java/com/judgmentlabs/judgeval/data/Example.java index 6bf0ddb..e26451c 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/Example.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/Example.java @@ -2,6 +2,7 @@ import java.time.Instant; import java.util.Map; +import java.util.UUID; public class Example extends com.judgmentlabs.judgeval.internal.api.models.Example { @@ -28,7 +29,7 @@ public String getValue() { public Example() { super(); - setExampleId(""); + setExampleId(UUID.randomUUID().toString()); setCreatedAt(Instant.now().toString()); setName(null); } From b4544761af873cc933d27097fa2a64f07e7b5178 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 17 Sep 2025 19:11:00 -0700 Subject: [PATCH 05/10] chore: trace evaluate --- .../judgeval/tracer/OpenTelemetryKeys.java | 1 + .../judgmentlabs/judgeval/tracer/Tracer.java | 72 +++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/OpenTelemetryKeys.java b/src/main/java/com/judgmentlabs/judgeval/tracer/OpenTelemetryKeys.java index 56cd9a6..b5016e7 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/OpenTelemetryKeys.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/OpenTelemetryKeys.java @@ -16,6 +16,7 @@ public static final class AttributeKeys { public static final String JUDGMENT_CUMULATIVE_LLM_COST = "judgment.cumulative_llm_cost"; public static final String JUDGMENT_STATE_BEFORE = "judgment.state_before"; public static final String JUDGMENT_STATE_AFTER = "judgment.state_after"; + public static final String PENDING_TRACE_EVAL = "judgment.pending_trace_eval"; private AttributeKeys() {} } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java index f10346a..d9d72c7 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java @@ -13,6 +13,7 @@ import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameRequest; import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameResponse; +import com.judgmentlabs.judgeval.internal.api.models.TraceEvaluationRun; import com.judgmentlabs.judgeval.scorers.BaseScorer; import com.judgmentlabs.judgeval.tracer.exporters.JudgmentSpanExporter; import com.judgmentlabs.judgeval.tracer.exporters.NoOpSpanExporter; @@ -267,6 +268,59 @@ public void asyncEvaluate(BaseScorer scorer, Example example) { asyncEvaluate(scorer, example, null); } + /** + * Asynchronously evaluates a scorer with trace and span context. + * + *

This method creates a trace evaluation run and sets it as an attribute on the current + * span. The evaluation will be processed when the span is exported. + * + *

If evaluation is disabled in the configuration, this method does nothing. The method + * respects OpenTelemetry's internal sampler - evaluation only runs if the current span is + * recording. + * + * @param scorer the scorer to use for evaluation (must not be null) + * @param model the model used for generation (can be null, will use default) + */ + public void asyncTraceEvaluate(BaseScorer scorer, String model) { + try { + if (!configuration.enableEvaluation()) { + return; + } + + Span currentSpan = Span.current(); + if (currentSpan == null || !currentSpan.getSpanContext().isSampled()) { + return; + } + + SpanContext spanContext = currentSpan.getSpanContext(); + String traceId = spanContext.getTraceId(); + String spanId = spanContext.getSpanId(); + + Object transport = scorer.toTransport(); + Logger.info( + "asyncTraceEvaluate: project=" + + configuration.projectName() + + ", traceId=" + + traceId + + ", spanId=" + + spanId + + ", scorerTransport=" + + (transport != null ? transport.getClass().getSimpleName() : "null")); + + TraceEvaluationRun evaluationRun = + createTraceEvaluationRun(scorer, model, traceId, spanId); + currentSpan.setAttribute( + OpenTelemetryKeys.AttributeKeys.PENDING_TRACE_EVAL, + serializer.serialize(evaluationRun)); + } catch (Exception e) { + Logger.error("Failed to evaluate trace scorer: " + e.getMessage()); + } + } + + public void asyncTraceEvaluate(BaseScorer scorer) { + asyncTraceEvaluate(scorer, null); + } + public void setAttributes(Map attributes) { if (attributes == null) { return; @@ -350,6 +404,24 @@ private ExampleEvaluationRun createEvaluationRun( return evaluationRun; } + private TraceEvaluationRun createTraceEvaluationRun( + BaseScorer scorer, String model, String traceId, String spanId) { + String evalName = + "async_trace_evaluate_" + (spanId != null ? spanId : System.currentTimeMillis()); + String modelName = model != null ? model : Env.JUDGMENT_DEFAULT_GPT_MODEL; + + TraceEvaluationRun evaluationRun = new TraceEvaluationRun(); + evaluationRun.setProjectName(configuration.projectName()); + evaluationRun.setEvalName(evalName); + evaluationRun.setJudgmentScorers( + List.of( + (com.judgmentlabs.judgeval.internal.api.models.ScorerConfig) + scorer.toTransport())); + evaluationRun.setModel(modelName); + evaluationRun.setTraceAndSpanIds(List.of(List.of(traceId, spanId))); + return evaluationRun; + } + private void enqueueEvaluation(ExampleEvaluationRun evaluationRun) { try { apiClient.addToRunEvalQueue(evaluationRun); From dc2cff7f1f42aa62a73c32b6a7c8cd93ab889be2 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 17 Sep 2025 19:29:42 -0700 Subject: [PATCH 06/10] feat: TraceEvaluationRun --- .../judgeval/data/TraceEvaluationRun.java | 298 ++++++++++++++++++ .../judgmentlabs/judgeval/tracer/Tracer.java | 25 +- 2 files changed, 311 insertions(+), 12 deletions(-) create mode 100644 src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java diff --git a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java new file mode 100644 index 0000000..a625d24 --- /dev/null +++ b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java @@ -0,0 +1,298 @@ +package com.judgmentlabs.judgeval.data; + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Objects; +import java.util.UUID; + +import com.judgmentlabs.judgeval.internal.api.models.BaseScorer; +import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; + +public class TraceEvaluationRun + extends com.judgmentlabs.judgeval.internal.api.models.TraceEvaluationRun { + + private String organizationId; + + public TraceEvaluationRun() { + super(); + setId(UUID.randomUUID().toString()); + setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); + } + + public static TraceEvaluationRun createWithLocalScorers( + String projectName, + String evalName, + List localScorers, + String model, + String organizationId, + List> traceAndSpanIds) { + TraceEvaluationRun eval = new TraceEvaluationRun(); + eval.setProjectName(projectName); + eval.setEvalName(evalName); + eval.setModel(model); + eval.setOrganizationId(organizationId); + eval.setTraceAndSpanIds(convertTraceAndSpanIds(traceAndSpanIds)); + + if (localScorers != null) { + List customScorers = new java.util.ArrayList<>(); + for (BaseScorer scorer : localScorers) { + customScorers.add(scorer); + } + eval.setCustomScorers(customScorers); + eval.setJudgmentScorers(new java.util.ArrayList<>()); + } + + eval.validateScorerLists(); + return eval; + } + + public static TraceEvaluationRun createWithApiScorers( + String projectName, + String evalName, + List apiScorers, + String model, + String organizationId, + List> traceAndSpanIds) { + TraceEvaluationRun eval = new TraceEvaluationRun(); + eval.setProjectName(projectName); + eval.setEvalName(evalName); + eval.setModel(model); + eval.setOrganizationId(organizationId); + eval.setTraceAndSpanIds(convertTraceAndSpanIds(traceAndSpanIds)); + + if (apiScorers != null) { + eval.setCustomScorers(new java.util.ArrayList<>()); + eval.setJudgmentScorers(apiScorers); + } + + eval.validateScorerLists(); + return eval; + } + + public TraceEvaluationRun( + String projectName, + String evalName, + List scorers, + String model, + String organizationId, + List> traceAndSpanIds) { + this(); + setProjectName(projectName); + setEvalName(evalName); + setModel(model); + setOrganizationId(organizationId); + setTraceAndSpanIds(convertTraceAndSpanIds(traceAndSpanIds)); + + if (scorers != null) { + List customScorers = new java.util.ArrayList<>(); + List judgmentScorers = new java.util.ArrayList<>(); + + for (Object scorer : scorers) { + if (scorer instanceof com.judgmentlabs.judgeval.scorers.BaseScorer) { + customScorers.add((BaseScorer) scorer); + } else if (scorer instanceof ScorerConfig) { + judgmentScorers.add((ScorerConfig) scorer); + } + } + + setCustomScorers(customScorers); + setJudgmentScorers(judgmentScorers); + } + + validateScorerLists(); + } + + private static List> convertTraceAndSpanIds(List> traceAndSpanIds) { + if (traceAndSpanIds == null || traceAndSpanIds.isEmpty()) { + throw new IllegalArgumentException( + "Trace and span IDs are required for trace evaluations."); + } + + List> converted = new java.util.ArrayList<>(); + for (List pair : traceAndSpanIds) { + if (pair == null || pair.size() != 2) { + throw new IllegalArgumentException( + "Each trace and span ID pair must contain exactly 2 elements."); + } + converted.add(List.of(pair.get(0), pair.get(1))); + } + return converted; + } + + private void validateScorerLists() { + List customScorers = getCustomScorers(); + List judgmentScorers = getJudgmentScorers(); + + if ((customScorers == null || customScorers.isEmpty()) + && (judgmentScorers == null || judgmentScorers.isEmpty())) { + throw new IllegalArgumentException( + "At least one of custom_scorers or judgment_scorers must be provided."); + } + + if (customScorers != null + && !customScorers.isEmpty() + && judgmentScorers != null + && !judgmentScorers.isEmpty()) { + throw new IllegalArgumentException( + "Only one of custom_scorers or judgment_scorers can be provided, not both."); + } + } + + public void setOrganizationId(String organizationId) { + this.organizationId = organizationId; + } + + public String getOrganizationId() { + return organizationId; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + if (!super.equals(obj)) return false; + TraceEvaluationRun other = (TraceEvaluationRun) obj; + return Objects.equals(organizationId, other.organizationId); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), organizationId); + } + + public static Builder builder() { + return new Builder(); + } + + public static Builder builder(String projectName, String evalName) { + return new Builder(projectName, evalName); + } + + public static final class Builder { + private String projectName; + private String evalName; + private List localScorers; + private List apiScorers; + private List mixedScorers; + private String model; + private String organizationId; + private List> traceAndSpanIds; + + private Builder() {} + + private Builder(String projectName, String evalName) { + this.projectName = projectName; + this.evalName = evalName; + } + + public Builder projectName(String projectName) { + this.projectName = projectName; + return this; + } + + public Builder evalName(String evalName) { + this.evalName = evalName; + return this; + } + + public Builder localScorers(List localScorers) { + this.localScorers = localScorers; + return this; + } + + public Builder localScorer(BaseScorer scorer) { + if (this.localScorers == null) { + this.localScorers = new java.util.ArrayList<>(); + } + this.localScorers.add(scorer); + return this; + } + + public Builder apiScorers(List apiScorers) { + this.apiScorers = apiScorers; + return this; + } + + public Builder apiScorer(ScorerConfig scorer) { + if (this.apiScorers == null) { + this.apiScorers = new java.util.ArrayList<>(); + } + this.apiScorers.add(scorer); + return this; + } + + public Builder scorers(List scorers) { + this.mixedScorers = scorers; + return this; + } + + public Builder scorer(Object scorer) { + if (this.mixedScorers == null) { + this.mixedScorers = new java.util.ArrayList<>(); + } + this.mixedScorers.add(scorer); + return this; + } + + public Builder model(String model) { + this.model = model; + return this; + } + + public Builder organizationId(String organizationId) { + this.organizationId = organizationId; + return this; + } + + public Builder traceAndSpanIds(List> traceAndSpanIds) { + this.traceAndSpanIds = traceAndSpanIds; + return this; + } + + public Builder traceAndSpanId(String traceId, String spanId) { + if (this.traceAndSpanIds == null) { + this.traceAndSpanIds = new java.util.ArrayList<>(); + } + this.traceAndSpanIds.add(List.of(traceId, spanId)); + return this; + } + + public TraceEvaluationRun build() { + if (projectName == null || projectName.trim().isEmpty()) { + throw new IllegalArgumentException("Project name is required"); + } + if (evalName == null || evalName.trim().isEmpty()) { + throw new IllegalArgumentException("Evaluation name is required"); + } + if (traceAndSpanIds == null || traceAndSpanIds.isEmpty()) { + throw new IllegalArgumentException( + "At least one trace and span ID pair is required"); + } + + if (localScorers != null && !localScorers.isEmpty()) { + return createWithLocalScorers( + projectName, + evalName, + localScorers, + model, + organizationId, + traceAndSpanIds); + } else if (apiScorers != null && !apiScorers.isEmpty()) { + return createWithApiScorers( + projectName, evalName, apiScorers, model, organizationId, traceAndSpanIds); + } else if (mixedScorers != null && !mixedScorers.isEmpty()) { + return new TraceEvaluationRun( + projectName, + evalName, + mixedScorers, + model, + organizationId, + traceAndSpanIds); + } else { + throw new IllegalArgumentException("At least one scorer is required"); + } + } + } +} diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java index d9d72c7..455739b 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java @@ -10,10 +10,10 @@ import com.judgmentlabs.judgeval.Env; import com.judgmentlabs.judgeval.data.Example; import com.judgmentlabs.judgeval.data.ExampleEvaluationRun; +import com.judgmentlabs.judgeval.data.TraceEvaluationRun; import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameRequest; import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameResponse; -import com.judgmentlabs.judgeval.internal.api.models.TraceEvaluationRun; import com.judgmentlabs.judgeval.scorers.BaseScorer; import com.judgmentlabs.judgeval.tracer.exporters.JudgmentSpanExporter; import com.judgmentlabs.judgeval.tracer.exporters.NoOpSpanExporter; @@ -53,7 +53,10 @@ * example.setAdditionalProperty("output", "model output"); * * BaseScorer scorer = new AnswerRelevancyScorer(); - * tracer.asyncEvaluate(scorer, example, "gpt-4", 1.0); + * tracer.asyncEvaluate(scorer, example, "gpt-4"); + * + * // Evaluate a scorer with trace context + * tracer.asyncTraceEvaluate(scorer, "gpt-4"); * } * *

Advanced Configuration

@@ -410,16 +413,14 @@ private TraceEvaluationRun createTraceEvaluationRun( "async_trace_evaluate_" + (spanId != null ? spanId : System.currentTimeMillis()); String modelName = model != null ? model : Env.JUDGMENT_DEFAULT_GPT_MODEL; - TraceEvaluationRun evaluationRun = new TraceEvaluationRun(); - evaluationRun.setProjectName(configuration.projectName()); - evaluationRun.setEvalName(evalName); - evaluationRun.setJudgmentScorers( - List.of( - (com.judgmentlabs.judgeval.internal.api.models.ScorerConfig) - scorer.toTransport())); - evaluationRun.setModel(modelName); - evaluationRun.setTraceAndSpanIds(List.of(List.of(traceId, spanId))); - return evaluationRun; + return TraceEvaluationRun.builder() + .projectName(configuration.projectName()) + .evalName(evalName) + .scorer(scorer.toTransport()) + .model(modelName) + .organizationId(configuration.organizationId()) + .traceAndSpanId(traceId, spanId) + .build(); } private void enqueueEvaluation(ExampleEvaluationRun evaluationRun) { From ae7fcb9893af2cd596b16eb18165745638541b44 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 17 Sep 2025 19:58:19 -0700 Subject: [PATCH 07/10] chore: fix toTransport, remove simplify custom scorer logic --- .../judgeval/data/ExampleEvaluationRun.java | 172 +++--------------- .../judgeval/data/TraceEvaluationRun.java | 172 +++--------------- .../judgeval/scorers/APIScorer.java | 2 +- .../judgeval/scorers/BaseScorer.java | 4 +- .../prompt_scorer/PromptScorer.java | 2 +- .../prompt_scorer/TracePromptScorer.java | 2 +- .../judgmentlabs/judgeval/tracer/Tracer.java | 10 +- 7 files changed, 54 insertions(+), 310 deletions(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java index 4ee1629..c9bd55d 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java @@ -7,7 +7,6 @@ import java.util.Objects; import java.util.UUID; -import com.judgmentlabs.judgeval.internal.api.models.BaseScorer; import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; public class ExampleEvaluationRun @@ -21,117 +20,22 @@ public ExampleEvaluationRun() { setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); } - public static ExampleEvaluationRun createWithLocalScorers( - String projectName, - String evalName, - List examples, - List localScorers, - String model, - String organizationId) { - ExampleEvaluationRun eval = new ExampleEvaluationRun(); - eval.setProjectName(projectName); - eval.setEvalName(evalName); - @SuppressWarnings("unchecked") - List internalExamples = - (List) (List) examples; - eval.setExamples(internalExamples); - eval.setModel(model); - eval.setOrganizationId(organizationId); - - if (localScorers != null) { - List customScorers = new java.util.ArrayList<>(); - - for (com.judgmentlabs.judgeval.scorers.BaseScorer scorer : localScorers) { - customScorers.add((BaseScorer) scorer); - } - - eval.setCustomScorers(customScorers); - eval.setJudgmentScorers(new java.util.ArrayList<>()); - } - - eval.validateScorerLists(); - return eval; - } - - public static ExampleEvaluationRun createWithApiScorers( - String projectName, - String evalName, - List examples, - List apiScorers, - String model, - String organizationId) { - ExampleEvaluationRun eval = new ExampleEvaluationRun(); - eval.setProjectName(projectName); - eval.setEvalName(evalName); - @SuppressWarnings("unchecked") - List internalExamples = - (List) (List) examples; - eval.setExamples(internalExamples); - eval.setModel(model); - eval.setOrganizationId(organizationId); - - if (apiScorers != null) { - eval.setCustomScorers(new java.util.ArrayList<>()); - eval.setJudgmentScorers(apiScorers); - } - - eval.validateScorerLists(); - return eval; - } - public ExampleEvaluationRun( String projectName, String evalName, List examples, - List scorers, + List scorers, String model, String organizationId) { this(); setProjectName(projectName); setEvalName(evalName); @SuppressWarnings("unchecked") - List internalExamples = - (List) (List) examples; + List internalExamples = (List) (List) examples; setExamples(internalExamples); setModel(model); setOrganizationId(organizationId); - - if (scorers != null) { - List customScorers = new java.util.ArrayList<>(); - List judgmentScorers = new java.util.ArrayList<>(); - - for (Object scorer : scorers) { - if (scorer instanceof com.judgmentlabs.judgeval.scorers.BaseScorer) { - customScorers.add((BaseScorer) scorer); - } else if (scorer instanceof ScorerConfig) { - judgmentScorers.add((ScorerConfig) scorer); - } - } - - setCustomScorers(customScorers); - setJudgmentScorers(judgmentScorers); - } - - validateScorerLists(); - } - - private void validateScorerLists() { - List customScorers = getCustomScorers(); - List judgmentScorers = getJudgmentScorers(); - - if ((customScorers == null || customScorers.isEmpty()) - && (judgmentScorers == null || judgmentScorers.isEmpty())) { - throw new IllegalArgumentException( - "At least one of custom_scorers or judgment_scorers must be provided."); - } - - if (customScorers != null - && !customScorers.isEmpty() - && judgmentScorers != null - && !judgmentScorers.isEmpty()) { - throw new IllegalArgumentException( - "Only one of custom_scorers or judgment_scorers can be provided, not both."); - } + setJudgmentScorers(scorers); } public void setOrganizationId(String organizationId) { @@ -144,9 +48,12 @@ public String getOrganizationId() { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - if (!super.equals(obj)) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; + if (!super.equals(obj)) + return false; ExampleEvaluationRun other = (ExampleEvaluationRun) obj; return Objects.equals(organizationId, other.organizationId); } @@ -168,13 +75,12 @@ public static final class Builder { private String projectName; private String evalName; private List examples; - private List localScorers; - private List apiScorers; - private List mixedScorers; + private List scorers; private String model; private String organizationId; - private Builder() {} + private Builder() { + } private Builder(String projectName, String evalName) { this.projectName = projectName; @@ -204,43 +110,16 @@ public Builder example(Example example) { return this; } - public Builder localScorers( - List localScorers) { - this.localScorers = localScorers; + public Builder scorers(List scorers) { + this.scorers = scorers; return this; } - public Builder localScorer(com.judgmentlabs.judgeval.scorers.BaseScorer scorer) { - if (this.localScorers == null) { - this.localScorers = new java.util.ArrayList<>(); + public Builder scorer(ScorerConfig scorer) { + if (this.scorers == null) { + this.scorers = new java.util.ArrayList<>(); } - this.localScorers.add(scorer); - return this; - } - - public Builder apiScorers(List apiScorers) { - this.apiScorers = apiScorers; - return this; - } - - public Builder apiScorer(ScorerConfig scorer) { - if (this.apiScorers == null) { - this.apiScorers = new java.util.ArrayList<>(); - } - this.apiScorers.add(scorer); - return this; - } - - public Builder scorers(List scorers) { - this.mixedScorers = scorers; - return this; - } - - public Builder scorer(Object scorer) { - if (this.mixedScorers == null) { - this.mixedScorers = new java.util.ArrayList<>(); - } - this.mixedScorers.add(scorer); + this.scorers.add(scorer); return this; } @@ -264,19 +143,12 @@ public ExampleEvaluationRun build() { if (examples == null || examples.isEmpty()) { throw new IllegalArgumentException("At least one example is required"); } - - if (localScorers != null && !localScorers.isEmpty()) { - return createWithLocalScorers( - projectName, evalName, examples, localScorers, model, organizationId); - } else if (apiScorers != null && !apiScorers.isEmpty()) { - return createWithApiScorers( - projectName, evalName, examples, apiScorers, model, organizationId); - } else if (mixedScorers != null && !mixedScorers.isEmpty()) { - return new ExampleEvaluationRun( - projectName, evalName, examples, mixedScorers, model, organizationId); - } else { + if (scorers == null || scorers.isEmpty()) { throw new IllegalArgumentException("At least one scorer is required"); } + + return new ExampleEvaluationRun( + projectName, evalName, examples, scorers, model, organizationId); } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java index a625d24..a9cea51 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java @@ -7,7 +7,6 @@ import java.util.Objects; import java.util.UUID; -import com.judgmentlabs.judgeval.internal.api.models.BaseScorer; import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; public class TraceEvaluationRun @@ -21,60 +20,10 @@ public TraceEvaluationRun() { setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); } - public static TraceEvaluationRun createWithLocalScorers( - String projectName, - String evalName, - List localScorers, - String model, - String organizationId, - List> traceAndSpanIds) { - TraceEvaluationRun eval = new TraceEvaluationRun(); - eval.setProjectName(projectName); - eval.setEvalName(evalName); - eval.setModel(model); - eval.setOrganizationId(organizationId); - eval.setTraceAndSpanIds(convertTraceAndSpanIds(traceAndSpanIds)); - - if (localScorers != null) { - List customScorers = new java.util.ArrayList<>(); - for (BaseScorer scorer : localScorers) { - customScorers.add(scorer); - } - eval.setCustomScorers(customScorers); - eval.setJudgmentScorers(new java.util.ArrayList<>()); - } - - eval.validateScorerLists(); - return eval; - } - - public static TraceEvaluationRun createWithApiScorers( - String projectName, - String evalName, - List apiScorers, - String model, - String organizationId, - List> traceAndSpanIds) { - TraceEvaluationRun eval = new TraceEvaluationRun(); - eval.setProjectName(projectName); - eval.setEvalName(evalName); - eval.setModel(model); - eval.setOrganizationId(organizationId); - eval.setTraceAndSpanIds(convertTraceAndSpanIds(traceAndSpanIds)); - - if (apiScorers != null) { - eval.setCustomScorers(new java.util.ArrayList<>()); - eval.setJudgmentScorers(apiScorers); - } - - eval.validateScorerLists(); - return eval; - } - public TraceEvaluationRun( String projectName, String evalName, - List scorers, + List scorers, String model, String organizationId, List> traceAndSpanIds) { @@ -84,24 +33,7 @@ public TraceEvaluationRun( setModel(model); setOrganizationId(organizationId); setTraceAndSpanIds(convertTraceAndSpanIds(traceAndSpanIds)); - - if (scorers != null) { - List customScorers = new java.util.ArrayList<>(); - List judgmentScorers = new java.util.ArrayList<>(); - - for (Object scorer : scorers) { - if (scorer instanceof com.judgmentlabs.judgeval.scorers.BaseScorer) { - customScorers.add((BaseScorer) scorer); - } else if (scorer instanceof ScorerConfig) { - judgmentScorers.add((ScorerConfig) scorer); - } - } - - setCustomScorers(customScorers); - setJudgmentScorers(judgmentScorers); - } - - validateScorerLists(); + setJudgmentScorers(scorers); } private static List> convertTraceAndSpanIds(List> traceAndSpanIds) { @@ -121,25 +53,6 @@ private static List> convertTraceAndSpanIds(List> trac return converted; } - private void validateScorerLists() { - List customScorers = getCustomScorers(); - List judgmentScorers = getJudgmentScorers(); - - if ((customScorers == null || customScorers.isEmpty()) - && (judgmentScorers == null || judgmentScorers.isEmpty())) { - throw new IllegalArgumentException( - "At least one of custom_scorers or judgment_scorers must be provided."); - } - - if (customScorers != null - && !customScorers.isEmpty() - && judgmentScorers != null - && !judgmentScorers.isEmpty()) { - throw new IllegalArgumentException( - "Only one of custom_scorers or judgment_scorers can be provided, not both."); - } - } - public void setOrganizationId(String organizationId) { this.organizationId = organizationId; } @@ -150,9 +63,12 @@ public String getOrganizationId() { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - if (!super.equals(obj)) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; + if (!super.equals(obj)) + return false; TraceEvaluationRun other = (TraceEvaluationRun) obj; return Objects.equals(organizationId, other.organizationId); } @@ -173,14 +89,13 @@ public static Builder builder(String projectName, String evalName) { public static final class Builder { private String projectName; private String evalName; - private List localScorers; - private List apiScorers; - private List mixedScorers; + private List scorers; private String model; private String organizationId; private List> traceAndSpanIds; - private Builder() {} + private Builder() { + } private Builder(String projectName, String evalName) { this.projectName = projectName; @@ -197,42 +112,16 @@ public Builder evalName(String evalName) { return this; } - public Builder localScorers(List localScorers) { - this.localScorers = localScorers; + public Builder scorers(List scorers) { + this.scorers = scorers; return this; } - public Builder localScorer(BaseScorer scorer) { - if (this.localScorers == null) { - this.localScorers = new java.util.ArrayList<>(); + public Builder scorer(ScorerConfig scorer) { + if (this.scorers == null) { + this.scorers = new java.util.ArrayList<>(); } - this.localScorers.add(scorer); - return this; - } - - public Builder apiScorers(List apiScorers) { - this.apiScorers = apiScorers; - return this; - } - - public Builder apiScorer(ScorerConfig scorer) { - if (this.apiScorers == null) { - this.apiScorers = new java.util.ArrayList<>(); - } - this.apiScorers.add(scorer); - return this; - } - - public Builder scorers(List scorers) { - this.mixedScorers = scorers; - return this; - } - - public Builder scorer(Object scorer) { - if (this.mixedScorers == null) { - this.mixedScorers = new java.util.ArrayList<>(); - } - this.mixedScorers.add(scorer); + this.scorers.add(scorer); return this; } @@ -266,33 +155,16 @@ public TraceEvaluationRun build() { if (evalName == null || evalName.trim().isEmpty()) { throw new IllegalArgumentException("Evaluation name is required"); } + if (scorers == null || scorers.isEmpty()) { + throw new IllegalArgumentException("At least one scorer is required"); + } if (traceAndSpanIds == null || traceAndSpanIds.isEmpty()) { throw new IllegalArgumentException( "At least one trace and span ID pair is required"); } - if (localScorers != null && !localScorers.isEmpty()) { - return createWithLocalScorers( - projectName, - evalName, - localScorers, - model, - organizationId, - traceAndSpanIds); - } else if (apiScorers != null && !apiScorers.isEmpty()) { - return createWithApiScorers( - projectName, evalName, apiScorers, model, organizationId, traceAndSpanIds); - } else if (mixedScorers != null && !mixedScorers.isEmpty()) { - return new TraceEvaluationRun( - projectName, - evalName, - mixedScorers, - model, - organizationId, - traceAndSpanIds); - } else { - throw new IllegalArgumentException("At least one scorer is required"); - } + return new TraceEvaluationRun( + projectName, evalName, scorers, model, organizationId, traceAndSpanIds); } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java index bbe90ec..186025e 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java @@ -42,7 +42,7 @@ public void setRequiredParams(List requiredParams) { } @Override - public Object toTransport() { + public ScorerConfig toTransport() { ScorerConfig cfg = new ScorerConfig(); cfg.setScoreType(getScoreType()); cfg.setThreshold(getThreshold()); diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java index ff3d698..fc26dd2 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java @@ -4,6 +4,8 @@ import java.util.List; import java.util.Map; +import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; + public abstract class BaseScorer extends com.judgmentlabs.judgeval.internal.api.models.BaseScorer { public BaseScorer() { @@ -17,7 +19,7 @@ public BaseScorer() { } } - public abstract Object toTransport(); + public abstract ScorerConfig toTransport(); public void addModel(String model) { setModel(model); diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java index 857decd..1b74b12 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java @@ -108,7 +108,7 @@ public static PromptScorer create( } @Override - public Object toTransport() { + public ScorerConfig toTransport() { ScorerConfig cfg = new ScorerConfig(); cfg.setScoreType(getScoreType()); cfg.setThreshold(getThreshold()); diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java index 7b68516..4439fc8 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java @@ -110,7 +110,7 @@ public static TracePromptScorer create( } @Override - public Object toTransport() { + public ScorerConfig toTransport() { ScorerConfig cfg = new ScorerConfig(); cfg.setScoreType(getScoreType()); cfg.setThreshold(getThreshold()); diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java index 455739b..1c0f268 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java @@ -248,7 +248,6 @@ public void asyncEvaluate(BaseScorer scorer, Example example, String model) { String traceId = spanContext.getTraceId(); String spanId = spanContext.getSpanId(); - Object transport = scorer.toTransport(); Logger.info( "asyncEvaluate: project=" + configuration.projectName() @@ -256,8 +255,8 @@ public void asyncEvaluate(BaseScorer scorer, Example example, String model) { + traceId + ", spanId=" + spanId - + ", scorerTransport=" - + (transport != null ? transport.getClass().getSimpleName() : "null")); + + ", scorer=" + + scorer.getName()); ExampleEvaluationRun evaluationRun = createEvaluationRun(scorer, example, model, traceId, spanId); @@ -299,7 +298,6 @@ public void asyncTraceEvaluate(BaseScorer scorer, String model) { String traceId = spanContext.getTraceId(); String spanId = spanContext.getSpanId(); - Object transport = scorer.toTransport(); Logger.info( "asyncTraceEvaluate: project=" + configuration.projectName() @@ -307,8 +305,8 @@ public void asyncTraceEvaluate(BaseScorer scorer, String model) { + traceId + ", spanId=" + spanId - + ", scorerTransport=" - + (transport != null ? transport.getClass().getSimpleName() : "null")); + + ", scorer=" + + scorer.getName()); TraceEvaluationRun evaluationRun = createTraceEvaluationRun(scorer, model, traceId, spanId); From 94b1ea3edf9334da62d4d7636444b932d1774ad1 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 17 Sep 2025 20:05:00 -0700 Subject: [PATCH 08/10] fix: add back empty custom scorer list --- .../com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java | 1 + .../java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java index c9bd55d..9c15ce4 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java @@ -35,6 +35,7 @@ public ExampleEvaluationRun( setExamples(internalExamples); setModel(model); setOrganizationId(organizationId); + setCustomScorers(new java.util.ArrayList<>()); setJudgmentScorers(scorers); } diff --git a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java index a9cea51..e5e5767 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java @@ -33,6 +33,7 @@ public TraceEvaluationRun( setModel(model); setOrganizationId(organizationId); setTraceAndSpanIds(convertTraceAndSpanIds(traceAndSpanIds)); + setCustomScorers(new java.util.ArrayList<>()); setJudgmentScorers(scorers); } From f99dbe052f8d5ffb6fa97597daf340a84da033a8 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 17 Sep 2025 20:10:11 -0700 Subject: [PATCH 09/10] chore: fix serializer --- .../judgeval/data/ExampleEvaluationRun.java | 15 ++++++--------- .../judgeval/data/TraceEvaluationRun.java | 12 ++++-------- .../com/judgmentlabs/judgeval/tracer/Tracer.java | 13 ++++++++++--- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java index 9c15ce4..fb64636 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java @@ -31,7 +31,8 @@ public ExampleEvaluationRun( setProjectName(projectName); setEvalName(evalName); @SuppressWarnings("unchecked") - List internalExamples = (List) (List) examples; + List internalExamples = + (List) (List) examples; setExamples(internalExamples); setModel(model); setOrganizationId(organizationId); @@ -49,12 +50,9 @@ public String getOrganizationId() { @Override public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null || getClass() != obj.getClass()) - return false; - if (!super.equals(obj)) - return false; + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + if (!super.equals(obj)) return false; ExampleEvaluationRun other = (ExampleEvaluationRun) obj; return Objects.equals(organizationId, other.organizationId); } @@ -80,8 +78,7 @@ public static final class Builder { private String model; private String organizationId; - private Builder() { - } + private Builder() {} private Builder(String projectName, String evalName) { this.projectName = projectName; diff --git a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java index e5e5767..9f86a86 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java @@ -64,12 +64,9 @@ public String getOrganizationId() { @Override public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null || getClass() != obj.getClass()) - return false; - if (!super.equals(obj)) - return false; + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + if (!super.equals(obj)) return false; TraceEvaluationRun other = (TraceEvaluationRun) obj; return Objects.equals(organizationId, other.organizationId); } @@ -95,8 +92,7 @@ public static final class Builder { private String organizationId; private List> traceAndSpanIds; - private Builder() { - } + private Builder() {} private Builder(String projectName, String evalName) { this.projectName = projectName; diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java index 1c0f268..7b305fd 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java @@ -6,6 +6,7 @@ import java.util.Objects; import java.util.Optional; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.judgmentlabs.judgeval.Env; import com.judgmentlabs.judgeval.data.Example; @@ -96,6 +97,7 @@ public final class Tracer { private final TracerConfiguration configuration; private final JudgmentSyncClient apiClient; private final ISerializer serializer; + private final ObjectMapper jacksonMapper; private final String projectId; private Tracer( @@ -105,6 +107,7 @@ private Tracer( this.configuration = Objects.requireNonNull(configuration, "Configuration cannot be null"); this.apiClient = Objects.requireNonNull(apiClient, "API client cannot be null"); this.serializer = Objects.requireNonNull(serializer, "Serializer cannot be null"); + this.jacksonMapper = new ObjectMapper(); this.projectId = resolveProjectId(configuration.projectName()); if (this.projectId == null) { Logger.error( @@ -310,9 +313,13 @@ public void asyncTraceEvaluate(BaseScorer scorer, String model) { TraceEvaluationRun evaluationRun = createTraceEvaluationRun(scorer, model, traceId, spanId); - currentSpan.setAttribute( - OpenTelemetryKeys.AttributeKeys.PENDING_TRACE_EVAL, - serializer.serialize(evaluationRun)); + try { + String traceEvalJson = jacksonMapper.writeValueAsString(evaluationRun); + currentSpan.setAttribute( + OpenTelemetryKeys.AttributeKeys.PENDING_TRACE_EVAL, traceEvalJson); + } catch (Exception e) { + Logger.error("Failed to serialize trace evaluation: " + e.getMessage()); + } } catch (Exception e) { Logger.error("Failed to evaluate trace scorer: " + e.getMessage()); } From caca58be9d2caf4273a28db150758b4afa3759e5 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 17 Sep 2025 20:13:26 -0700 Subject: [PATCH 10/10] feat: working trace eval --- .../judgeval/data/TraceEvaluationRun.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java index 9f86a86..691223c 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java @@ -18,6 +18,7 @@ public TraceEvaluationRun() { super(); setId(UUID.randomUUID().toString()); setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); + setIsOffline(false); } public TraceEvaluationRun( @@ -64,9 +65,12 @@ public String getOrganizationId() { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - if (!super.equals(obj)) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; + if (!super.equals(obj)) + return false; TraceEvaluationRun other = (TraceEvaluationRun) obj; return Objects.equals(organizationId, other.organizationId); } @@ -92,7 +96,8 @@ public static final class Builder { private String organizationId; private List> traceAndSpanIds; - private Builder() {} + private Builder() { + } private Builder(String projectName, String evalName) { this.projectName = projectName;