From 4ee1a69da92e6983ace25cab7b16691ee64f0b38 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 29 Oct 2025 13:28:07 -0700 Subject: [PATCH 01/11] chore: consistant otel keys in setAttribute + add genai keys --- pom.xml | 2 +- .../judgeval/tracer/BaseTracer.java | 75 +++++++++++++------ .../judgeval/tracer/OpenTelemetryKeys.java | 16 ++++ 3 files changed, 68 insertions(+), 25 deletions(-) diff --git a/pom.xml b/pom.xml index 56c452e..4ef23fa 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.judgmentlabs judgeval-java - 0.2.1 + 0.2.2 jar Judgeval Java Java SDK for Judgeval diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java index 36b8773..abf454b 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java @@ -4,7 +4,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; @@ -76,26 +75,54 @@ public SpanExporter getSpanExporter() { return createJudgmentSpanExporter(projectId); } - /** Sets the span kind attribute on the current span. Common kinds: "span", "llm", "tool". */ public void setSpanKind(String kind) { - Optional.ofNullable(Span.current()) - .filter(span -> kind != null) - .ifPresent( - span -> - span.setAttribute( - OpenTelemetryKeys.AttributeKeys.JUDGMENT_SPAN_KIND, kind)); + if (kind == null) { + return; + } + withCurrentSpan( + span -> + span.setAttribute( + OpenTelemetryKeys.AttributeKeys.JUDGMENT_SPAN_KIND, kind)); + } + + private static void withCurrentSpan(java.util.function.Consumer action) { + Span span = Span.current(); + if (span != null) { + action.accept(span); + } } - /** Sets a custom attribute on the current span. Value is serialized to JSON. */ public void setAttribute(String key, Object value) { - Optional.ofNullable(Span.current()) - .ifPresent(span -> span.setAttribute(key, serializer.serialize(value))); + if (value == null) { + return; + } + setAttribute(key, value, value.getClass()); } - /** Sets a custom attribute on the current span with specific type for JSON serialization. */ public void setAttribute(String key, Object value, Type type) { - Optional.ofNullable(Span.current()) - .ifPresent(span -> span.setAttribute(key, serializer.serialize(value, type))); + if (value == null) { + return; + } + withCurrentSpan(span -> span.setAttribute(key, serializer.serialize(value, type))); + } + + public void setAttribute(String key, String value) { + if (value == null) { + return; + } + withCurrentSpan(span -> span.setAttribute(key, value)); + } + + public void setAttribute(String key, long value) { + withCurrentSpan(span -> span.setAttribute(key, value)); + } + + public void setAttribute(String key, double value) { + withCurrentSpan(span -> span.setAttribute(key, value)); + } + + public void setAttribute(String key, boolean value) { + withCurrentSpan(span -> span.setAttribute(key, value)); } /** Asynchronously evaluates a scorer with an example using current trace context. */ @@ -184,13 +211,7 @@ public void setAttributes(Map attributes) { if (attributes == null) { return; } - Optional.ofNullable(Span.current()) - .ifPresent( - span -> - attributes.forEach( - (key, value) -> - span.setAttribute( - key, serializer.serialize(value)))); + withCurrentSpan(span -> attributes.forEach(this::setAttribute)); } public void setLLMSpan() { @@ -266,7 +287,8 @@ protected String resolveProjectId(String name) { ResolveProjectNameRequest request = new ResolveProjectNameRequest(); request.setProjectName(name); ResolveProjectNameResponse response = apiClient.projectsResolve(request); - return Optional.ofNullable(response.getProjectId()).map(Object::toString).orElse(null); + Object projectId = response.getProjectId(); + return projectId != null ? projectId.toString() : null; } catch (Exception e) { return null; } @@ -332,12 +354,17 @@ protected static class GsonSerializer implements ISerializer { @Override public String serialize(Object obj) { - return gson.toJson(obj); + return serialize(obj, obj.getClass()); } @Override public String serialize(Object obj, Type type) { - return gson.toJson(obj, type); + try { + return gson.toJson(obj, type); + } catch (Exception e) { + Logger.error("Failed to serialize object: " + e.getMessage()); + return obj != null ? obj.toString() : null; + } } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/OpenTelemetryKeys.java b/src/main/java/com/judgmentlabs/judgeval/tracer/OpenTelemetryKeys.java index b5016e7..aade76f 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/OpenTelemetryKeys.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/OpenTelemetryKeys.java @@ -18,6 +18,22 @@ public static final class AttributeKeys { public static final String JUDGMENT_STATE_AFTER = "judgment.state_after"; public static final String PENDING_TRACE_EVAL = "judgment.pending_trace_eval"; + public static final String GEN_AI_PROMPT = "gen_ai.prompt"; + public static final String GEN_AI_COMPLETION = "gen_ai.completion"; + public static final String GEN_AI_REQUEST_MODEL = "gen_ai.request.model"; + public static final String GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"; + public static final String GEN_AI_SYSTEM = "gen_ai.system"; + public static final String GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"; + public static final String GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"; + public static final String GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS = + "gen_ai.usage.cache_creation_input_tokens"; + public static final String GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS = + "gen_ai.usage.cache_read_input_tokens"; + public static final String GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"; + public static final String GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"; + public static final String GEN_AI_RESPONSE_FINISH_REASONS = + "gen_ai.response.finish_reasons"; + private AttributeKeys() {} } From 420922a84942357066ef1972bbff1e3dbcbdb381 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 29 Oct 2025 14:37:58 -0700 Subject: [PATCH 02/11] chore: format --- .cursorrules | 227 ++++++++++ .idea/codeStyles/codeStyleConfig.xml | 5 + .idea/vcs.xml | 6 + .idea/workspace.xml | 66 +++ .vscode/java-formatter.xml | 337 ++++++++++++++ .vscode/settings.json | 3 +- pom.xml | 17 +- scripts/generate_client.py | 122 ++--- .../judgeval/data/APIScorerType.java | 20 +- .../judgeval/data/ExampleEvaluationRun.java | 22 +- .../judgeval/data/ScoringResult.java | 9 +- .../judgeval/data/TraceEvaluationRun.java | 22 +- .../internal/api/JudgmentAsyncClient.java | 105 ++--- .../internal/api/JudgmentSyncClient.java | 116 ++--- .../internal/api/models/BaseScorer.java | 54 +-- .../internal/api/models/EvalResults.java | 18 +- .../internal/api/models/EvalResultsFetch.java | 15 +- .../judgeval/internal/api/models/Example.java | 8 +- .../api/models/ExampleEvaluationRun.java | 45 +- .../api/models/FetchPromptScorersRequest.java | 20 +- .../models/FetchPromptScorersResponse.java | 11 +- .../internal/api/models/OtelTraceSpan.java | 174 +------ .../internal/api/models/PromptScorer.java | 104 ++++- .../api/models/ResolveProjectNameRequest.java | 11 +- .../models/ResolveProjectNameResponse.java | 11 +- .../api/models/SavePromptScorerRequest.java | 66 ++- .../api/models/SavePromptScorerResponse.java | 37 +- .../internal/api/models/ScorerConfig.java | 39 +- .../internal/api/models/ScorerData.java | 45 +- .../api/models/ScorerExistsRequest.java | 11 +- .../api/models/ScorerExistsResponse.java | 11 +- .../internal/api/models/ScoringResult.java | 34 +- .../api/models/TraceEvaluationRun.java | 54 +-- .../judgeval/scorers/APIScorer.java | 6 +- .../prompt_scorer/BasePromptScorer.java | 71 +-- .../prompt_scorer/PromptScorer.java | 67 +-- .../prompt_scorer/TracePromptScorer.java | 74 +-- .../judgeval/tracer/BaseTracer.java | 424 +++++++++++------- .../judgmentlabs/judgeval/tracer/Tracer.java | 89 ++-- .../judgeval/tracer/TracerConfiguration.java | 77 ++-- .../exporters/JudgmentSpanExporter.java | 89 ++-- .../tracer/exporters/NoOpSpanExporter.java | 20 + .../judgmentlabs/judgeval/utils/Logger.java | 6 +- 43 files changed, 1686 insertions(+), 1082 deletions(-) create mode 100644 .cursorrules create mode 100644 .idea/codeStyles/codeStyleConfig.xml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml create mode 100644 .vscode/java-formatter.xml diff --git a/.cursorrules b/.cursorrules new file mode 100644 index 0000000..8422f18 --- /dev/null +++ b/.cursorrules @@ -0,0 +1,227 @@ +# Judgeval Java - Internal API Patterns + +## Context + +The `com.judgmentlabs.judgeval.internal` package contains internal API code that is auto-generated from OpenAPI specifications using `scripts/generate_client.py`, but is explicitly modifiable for internal improvements. This code is NOT part of the public API and can be refactored, cleaned up, and optimized as needed. + +## Internal API Models (`internal/api/models/`) + +### Structure Pattern +- **Package**: `com.judgmentlabs.judgeval.internal.api.models` +- **Naming**: PascalCase class names matching OpenAPI schema names (e.g., `EvalResultsFetch`, `ResolveProjectNameResponse`) +- **Purpose**: Jackson-annotated POJOs for serializing/deserializing JSON requests and responses + +### Field Patterns +- **Field Declaration**: + - Private fields with camelCase naming (e.g., `experimentRunId`, `projectName`) + - Jackson `@JsonProperty` annotation mapping snake_case JSON keys to camelCase fields + - Use wrapper types (String, Integer, Boolean, List, etc.) - never primitives for nullable fields + - All fields are nullable by default (no `@NotNull` annotations) + +### Additional Properties Pattern +All model classes MUST include: +```java +private Map additionalProperties = new HashMap<>(); + +@JsonAnyGetter +public Map getAdditionalProperties() { + return additionalProperties; +} + +@JsonAnySetter +public void setAdditionalProperty(String name, Object value) { + additionalProperties.put(name, value); +} +``` +This allows unknown JSON fields to be preserved during deserialization. + +### Getter/Setter Pattern +- **Getters**: Public methods following JavaBean convention `getFieldName()` returning field type +- **Setters**: Public methods following JavaBean convention `setFieldName(Type fieldName)` returning void +- **Order**: Fields declared first, then additionalProperties, then getters, then setters + +### Equals/HashCode Pattern +- **Equals**: Must include all fields plus `additionalProperties` +- **Implementation**: Use `Objects.equals()` for all field comparisons +- **HashCode**: Use `Objects.hash()` for all fields, with `Objects.hashCode(additionalProperties)` for the Map +- **Null Safety**: Both methods handle null fields safely via `Objects.equals()` + +### Imports +Standard imports: +- `java.util.HashMap` +- `java.util.List` (if using collections) +- `java.util.Map` +- `java.util.Objects` +- `com.fasterxml.jackson.annotation.JsonAnyGetter` +- `com.fasterxml.jackson.annotation.JsonAnySetter` +- `com.fasterxml.jackson.annotation.JsonProperty` + +## Internal API Clients (`internal/api/`) + +### Client Classes +- **JudgmentSyncClient**: Synchronous HTTP client for blocking API calls +- **JudgmentAsyncClient**: Asynchronous HTTP client for non-blocking API calls +- Both share identical structure and helper methods, differing only in execution model + +### Constructor Pattern +```java +public JudgmentSyncClient(String baseUrl, String apiKey, String organizationId) { + this.baseUrl = baseUrl; + this.apiKey = apiKey; + this.organizationId = organizationId; + this.client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build(); + this.mapper = new ObjectMapper(); +} +``` + +### Helper Methods Pattern + +#### buildUrl(String path, Map queryParams) +- Builds full URL from baseUrl + path +- Appends query string from map if non-empty +- Query string format: `key1=value1&key2=value2` +- Returns complete URL string + +#### buildUrl(String path) +- Convenience overload calling `buildUrl(path, new HashMap<>())` + +#### buildHeaders() +- Validates `apiKey` and `organizationId` are not null (throws `IllegalArgumentException`) +- Returns String array with: + - `"Content-Type"`, `"application/json"` + - `"Authorization"`, `"Bearer " + apiKey` + - `"X-Organization-Id"`, `organizationId` + +#### handleResponse(HttpResponse response) +- **Sync**: `throws IOException` +- **Async**: No throws clause (unchecked exceptions) +- Checks status code >= 400, throws `RuntimeException` with status and body +- Attempts to deserialize response body using `mapper.readValue(response.body(), new TypeReference() {})` +- Catches parsing exceptions and wraps in `RuntimeException("Failed to parse response", e)` +- **Issue**: TypeReference generic type erasure makes this unreliable - should use specific class when known + +### Method Naming Pattern +- Method names derived from API path: `/fetch_experiment_run/` → `fetchExperimentRun` +- Path segments separated by underscores/forward slashes become camelCase +- HTTP method determines method signature (GET = no body, POST = has payload) + +### Sync Client Method Pattern +```java +public ReturnType methodName(RequestType payload) throws IOException, InterruptedException { + String url = buildUrl("/api/path/"); + String jsonPayload = mapper.writeValueAsString(payload); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + return handleResponse(response); // or mapper.readValue(response.body(), SpecificType.class); +} +``` + +### Async Client Method Pattern +```java +public CompletableFuture methodName(RequestType payload) { + String url = buildUrl("/api/path/"); + String jsonPayload; + try { + jsonPayload = mapper.writeValueAsString(payload); + } catch (Exception e) { + throw new RuntimeException("Failed to serialize payload", e); + } + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); + return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) + .thenApply(this::handleResponse); +} +``` + +### Return Type Patterns +- **Consistency Issue**: Some methods return `Object`, others return specific response types +- **Preferred**: Return specific response model types when known (e.g., `ScorerExistsResponse`, `ResolveProjectNameResponse`) +- **Fallback**: Use `Object` only when response schema is truly unknown/variable +- **Async**: Return type wrapped in `CompletableFuture` + +### Error Handling Standards + +#### Sync Client +- `IOException` and `InterruptedException` propagate (method signature) +- `RuntimeException` for HTTP errors (4xx/5xx) - thrown by `handleResponse` +- `RuntimeException` for deserialization errors - thrown by `handleResponse` or `mapper.readValue` + +#### Async Client +- All exceptions caught internally and wrapped in `RuntimeException` +- Serialization errors caught immediately and rethrown before async operation +- HTTP/deserialization errors thrown from `handleResponse` in CompletableFuture chain +- No checked exceptions in method signatures + +### Query Parameters Pattern +For GET requests or POST requests with query params: +```java +Map queryParams = new HashMap<>(); +queryParams.put("param_name", paramValue); +String url = buildUrl("/api/path/", queryParams); +``` + +## Code Quality Expectations + +### Documentation +- Add JavaDoc to all public methods in client classes +- Document parameters, return types, and thrown exceptions +- Explain complex business logic or non-obvious patterns + +### Null Safety +- Use `Optional` internally for nullable values when appropriate +- Use `@NotNull` annotations for parameters that must not be null (only if needed for external safety) +- Validate constructor parameters that must not be null + +### Error Messages +- Error messages should be descriptive and include context +- HTTP errors should include status code and response body +- Serialization errors should indicate the operation that failed + +### Code Organization +- Group related methods together +- Keep helper methods private +- Maintain consistent method ordering (constructors, helpers, public API methods) +- Extract common patterns into reusable helpers when duplicated across sync/async clients + +### Type Safety +- Prefer specific types over `Object` for return values +- Use proper generic types for collections +- Avoid raw types and unchecked casts + +### Consistency +- Sync and async clients should have identical API (only differing by CompletableFuture wrapper) +- Method names should be consistent across both clients +- Helper methods should behave identically in both clients + +## Refactoring Opportunities + +### Known Issues +1. **TypeReference generic erasure**: `handleResponse` uses `TypeReference()` which doesn't preserve type information. Consider passing `Class` or using Jackson's `TypeFactory`. + +2. **Return type inconsistency**: Some sync methods return `Object`, others return specific types. Standardize based on OpenAPI response schemas. + +3. **Code duplication**: Sync and async clients share ~90% of code. Consider extracting common base class or using composition. + +4. **Error handling**: Async client wraps serialization errors, sync client doesn't. Consider consistent error handling strategy. + +5. **Missing validation**: No validation of request payloads before serialization. Consider adding validation for required fields. + +6. **Missing documentation**: No JavaDoc comments on any client methods. Should document all public methods. + +### Recommended Improvements +- Extract common client logic to base class `BaseJudgmentClient` +- Standardize return types based on OpenAPI response schemas +- Add request/response validation +- Improve error handling with custom exception types +- Add comprehensive JavaDoc documentation +- Consider using builder pattern for request construction +- Add retry logic for transient failures +- Add request/response logging + diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..a55e7a1 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..c531a7e --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1761770774206 + + + + \ No newline at end of file diff --git a/.vscode/java-formatter.xml b/.vscode/java-formatter.xml new file mode 100644 index 0000000..a14f80e --- /dev/null +++ b/.vscode/java-formatter.xml @@ -0,0 +1,337 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 4f81299..49b061d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,4 @@ { - "java.configuration.updateBuildConfiguration": "automatic" + "java.configuration.updateBuildConfiguration": "automatic", + "java.format.settings.url": ".vscode/java-formatter.xml" } diff --git a/pom.xml b/pom.xml index 4ef23fa..9ae9091 100644 --- a/pom.xml +++ b/pom.xml @@ -111,6 +111,12 @@ gson 2.10.1 + + org.jetbrains + annotations + 24.1.0 + provided + @@ -207,13 +213,13 @@ ${spotless.version} - - 1.19.2 - - + + 4.28 + .vscode/java-formatter.xml + - java,javax,org,com, + java,javax,org,com @@ -225,7 +231,6 @@ ${checkstyle.version} checkstyle.xml - UTF-8 true false diff --git a/scripts/generate_client.py b/scripts/generate_client.py index d043351..51afec6 100755 --- a/scripts/generate_client.py +++ b/scripts/generate_client.py @@ -193,19 +193,25 @@ def get_java_type(schema: Dict[str, Any]) -> str: def generate_model_class(className: str, schema: Dict[str, Any]) -> str: + required_fields = set(schema.get("required", [])) + has_required = bool(required_fields) + lines = [ "package com.judgmentlabs.judgeval.internal.api.models;", "", - "import com.fasterxml.jackson.annotation.JsonProperty;", - "import com.fasterxml.jackson.annotation.JsonAnySetter;", "import com.fasterxml.jackson.annotation.JsonAnyGetter;", + "import com.fasterxml.jackson.annotation.JsonAnySetter;", + "import com.fasterxml.jackson.annotation.JsonProperty;", + "import java.util.HashMap;", "import java.util.List;", "import java.util.Map;", - "import java.util.HashMap;", "import java.util.Objects;", - "", - f"public class {className} {{", ] + + if has_required: + lines.append("import org.jetbrains.annotations.NotNull;") + + lines.extend(["", f"public class {className} {{"]) fields = [] getters = [] @@ -217,13 +223,14 @@ def generate_model_class(className: str, schema: Dict[str, Any]) -> str: for field_name, property_schema in schema["properties"].items(): java_type = get_java_type(property_schema) camel_case_name = to_camel_case(field_name) + is_required = field_name in required_fields - fields.extend( - [ - f' @JsonProperty("{field_name}")', - f" private {java_type} {camel_case_name};", - ] - ) + field_lines = [f' @JsonProperty("{field_name}")'] + if is_required: + field_lines.append(" @NotNull") + field_lines.append(f" private {java_type} {camel_case_name};") + + fields.extend(field_lines) getters.extend( [ @@ -233,9 +240,10 @@ def generate_model_class(className: str, schema: Dict[str, Any]) -> str: ] ) + setter_param = f"@NotNull {java_type} {camel_case_name}" if is_required else f"{java_type} {camel_case_name}" setters.extend( [ - f" public void set{to_class_name(camel_case_name)}({java_type} {camel_case_name}) {{", + f" public void set{to_class_name(camel_case_name)}({setter_param}) {{", f" this.{camel_case_name} = {camel_case_name};", " }", ] @@ -309,10 +317,10 @@ def generate_method_signature( for param in query_params: if param["required"]: - params.append(f"String {param['name']}") + params.append(f"@NotNull String {param['name']}") if request_type: - params.append(f"{request_type} payload") + params.append(f"@NotNull {request_type} payload") for param in query_params: if not param["required"]: @@ -344,12 +352,9 @@ def generate_method_body( if param["required"]: lines.append(f' queryParams.put("{param_name}", {param_name});') else: - lines.extend( - [ - f" if ({param_name} != null) {{", - f' queryParams.put("{param_name}", {param_name});', - " }", - ] + param_key = param["name"] + lines.append( + f' Optional.ofNullable({param_name}).ifPresent(v -> queryParams.put("{param_key}", v));' ) lines.append( @@ -422,8 +427,8 @@ def generate_client_class( imports = [ "package com.judgmentlabs.judgeval.internal.api;", "", - "import com.fasterxml.jackson.databind.ObjectMapper;", "import com.fasterxml.jackson.core.type.TypeReference;", + "import com.fasterxml.jackson.databind.ObjectMapper;", "import java.io.IOException;", "import java.net.URI;", "import java.net.http.HttpClient;", @@ -431,6 +436,9 @@ def generate_client_class( "import java.net.http.HttpResponse;", "import java.util.HashMap;", "import java.util.Map;", + "import java.util.Objects;", + "import java.util.Optional;", + "import org.jetbrains.annotations.NotNull;", "import com.judgmentlabs.judgeval.internal.api.models.*;", ] @@ -446,10 +454,10 @@ def generate_client_class( " private final String apiKey;", " private final String organizationId;", "", - f" public {className}(String baseUrl, String apiKey, String organizationId) {{", - " this.baseUrl = baseUrl;", - " this.apiKey = apiKey;", - " this.organizationId = organizationId;", + f" public {className}(@NotNull String baseUrl, @NotNull String apiKey, @NotNull String organizationId) {{", + " this.baseUrl = Objects.requireNonNull(baseUrl, \"Base URL cannot be null\");", + " this.apiKey = Objects.requireNonNull(apiKey, \"API key cannot be null\");", + " this.organizationId = Objects.requireNonNull(organizationId, \"Organization ID cannot be null\");", " this.client = HttpClient.newBuilder()", " .version(HttpClient.Version.HTTP_1_1)", " .build();", @@ -473,9 +481,6 @@ def generate_client_class( " }", "", " private String[] buildHeaders() {", - " if (apiKey == null || organizationId == null) {", - ' throw new IllegalArgumentException("API key and organization ID cannot be null");', - " }", " return new String[] {", ' "Content-Type",', ' "application/json",', @@ -489,42 +494,39 @@ def generate_client_class( ] throws_clause = "" if is_async else " throws IOException" - lines.extend( - [ - f" private T handleResponse(HttpResponse response){throws_clause} {{", - " if (response.statusCode() >= 400) {", - ' throw new RuntimeException("HTTP Error: " + response.statusCode() + " - " + response.body());', - " }", - " try {", - " return mapper.readValue(response.body(), new TypeReference() {});", - " } catch (Exception e) {", - ' throw new RuntimeException("Failed to parse response", e);', - " }", - " }", - "", - ] - ) + lines.append(f" private T handleResponse(HttpResponse response){throws_clause} {{") + lines.append(" if (response.statusCode() >= 400) {") + lines.append(f' throw new RuntimeException("HTTP Error: " + response.statusCode() + " - " + response.body());') + lines.append(" }") + lines.append(" try {") + lines.append(" return mapper.readValue(response.body(), new TypeReference() {});") + lines.append(" } catch (Exception e) {") + lines.append(' throw new RuntimeException("Failed to parse response", e);') + lines.append(" }") + lines.append(" }") + lines.append("") for method_info in methods: - signature = generate_method_signature( - method_info["name"], - method_info["request_type"], - method_info["query_params"], - method_info["response_type"], - is_async, + lines.append( + generate_method_signature( + method_info["name"], + method_info["request_type"], + method_info["query_params"], + method_info["response_type"], + is_async, + ) ) - lines.append(signature) - - body = generate_method_body( - method_info["name"], - method_info["path"], - method_info["method"], - method_info["request_type"], - method_info["query_params"], - method_info["response_type"], - is_async, + lines.append( + generate_method_body( + method_info["name"], + method_info["path"], + method_info["method"], + method_info["request_type"], + method_info["query_params"], + method_info["response_type"], + is_async, + ) ) - lines.append(body) lines.append(" }") lines.append("") diff --git a/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java b/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java index 49e9af2..0f11b45 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java @@ -1,18 +1,14 @@ package com.judgmentlabs.judgeval.data; public enum APIScorerType { - PROMPT_SCORER("Prompt Scorer"), - TRACE_PROMPT_SCORER("Trace Prompt Scorer"), - FAITHFULNESS("Faithfulness"), - ANSWER_RELEVANCY("Answer Relevancy"), - ANSWER_CORRECTNESS("Answer Correctness"), - INSTRUCTION_ADHERENCE("Instruction Adherence"), - EXECUTION_ORDER("Execution Order"), - DERAILMENT("Derailment"), - TOOL_ORDER("Tool Order"), - CLASSIFIER("Classifier"), - TOOL_DEPENDENCY("Tool Dependency"), - CUSTOM("Custom"); + PROMPT_SCORER("Prompt Scorer"), TRACE_PROMPT_SCORER("Trace Prompt Scorer"), FAITHFULNESS( + "Faithfulness"), ANSWER_RELEVANCY("Answer Relevancy"), ANSWER_CORRECTNESS( + "Answer Correctness"), INSTRUCTION_ADHERENCE( + "Instruction Adherence"), EXECUTION_ORDER( + "Execution Order"), DERAILMENT("Derailment"), TOOL_ORDER( + "Tool Order"), CLASSIFIER( + "Classifier"), TOOL_DEPENDENCY( + "Tool Dependency"), CUSTOM("Custom"); private final String value; diff --git a/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java index fb64636..ae4b31f 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java @@ -20,13 +20,8 @@ public ExampleEvaluationRun() { setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); } - public ExampleEvaluationRun( - String projectName, - String evalName, - List examples, - List scorers, - String model, - String organizationId) { + public ExampleEvaluationRun(String projectName, String evalName, List examples, + List scorers, String model, String organizationId) { this(); setProjectName(projectName); setEvalName(evalName); @@ -50,9 +45,12 @@ public String getOrganizationId() { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - if (!super.equals(obj)) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; + if (!super.equals(obj)) + return false; ExampleEvaluationRun other = (ExampleEvaluationRun) obj; return Objects.equals(organizationId, other.organizationId); } @@ -145,8 +143,8 @@ public ExampleEvaluationRun build() { throw new IllegalArgumentException("At least one scorer is required"); } - return new ExampleEvaluationRun( - projectName, evalName, examples, scorers, model, organizationId); + return new ExampleEvaluationRun(projectName, evalName, examples, scorers, model, + organizationId); } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java b/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java index 8df8b2a..096a141 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java @@ -23,8 +23,7 @@ public Builder success(Boolean success) { public Builder scorersData(List scorersData) { @SuppressWarnings("unchecked") List internalList = - (List) - (List) scorersData; + (List) (List) scorersData; result.setScorersData(internalList); return this; } @@ -38,8 +37,10 @@ public Builder scorerData(ScorerData scorerData) { } public Builder dataObject(Example dataObject) { - // Store Example in additional properties since setDataObject expects TraceSpan - // This indicates a potential API design issue - ScoringResult may be + // Store Example in additional properties since setDataObject + // expects TraceSpan + // This indicates a potential API design issue - ScoringResult may + // be // trace-specific if (dataObject != null) { result.setAdditionalProperty("example", dataObject); diff --git a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java index 311b31a..526e640 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java @@ -21,13 +21,8 @@ public TraceEvaluationRun() { setIsOffline(false); } - public TraceEvaluationRun( - String projectName, - String evalName, - List scorers, - String model, - String organizationId, - List> traceAndSpanIds) { + public TraceEvaluationRun(String projectName, String evalName, List scorers, + String model, String organizationId, List> traceAndSpanIds) { this(); setProjectName(projectName); setEvalName(evalName); @@ -65,9 +60,12 @@ public String getOrganizationId() { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - if (!super.equals(obj)) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; + if (!super.equals(obj)) + return false; TraceEvaluationRun other = (TraceEvaluationRun) obj; return Objects.equals(organizationId, other.organizationId); } @@ -161,8 +159,8 @@ public TraceEvaluationRun build() { "At least one trace and span ID pair is required"); } - return new TraceEvaluationRun( - projectName, evalName, scorers, model, organizationId, traceAndSpanIds); + return new TraceEvaluationRun(projectName, evalName, scorers, model, organizationId, + traceAndSpanIds); } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java index 6fed7da..1749154 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java @@ -6,8 +6,11 @@ import java.net.http.HttpResponse; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.concurrent.CompletableFuture; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.judgmentlabs.judgeval.internal.api.models.*; @@ -19,10 +22,12 @@ public class JudgmentAsyncClient { private final String apiKey; private final String organizationId; - public JudgmentAsyncClient(String baseUrl, String apiKey, String organizationId) { - this.baseUrl = baseUrl; - this.apiKey = apiKey; - this.organizationId = organizationId; + public JudgmentAsyncClient(@NotNull String baseUrl, @NotNull String apiKey, + @NotNull String organizationId) { + this.baseUrl = Objects.requireNonNull(baseUrl, "Base URL cannot be null"); + this.apiKey = Objects.requireNonNull(apiKey, "API key cannot be null"); + this.organizationId = + Objects.requireNonNull(organizationId, "Organization ID cannot be null"); this.client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build(); this.mapper = new ObjectMapper(); } @@ -45,17 +50,9 @@ private String buildUrl(String path) { } private String[] buildHeaders() { - if (apiKey == null || organizationId == null) { - throw new IllegalArgumentException("API key and organization ID cannot be null"); - } - return new String[] { - "Content-Type", - "application/json", - "Authorization", - "Bearer " + apiKey, - "X-Organization-Id", - organizationId - }; + return new String[] {"Content-Type", "application/json", "Authorization", + "Bearer " + apiKey, + "X-Organization-Id", organizationId}; } private T handleResponse(HttpResponse response) { @@ -70,7 +67,7 @@ private T handleResponse(HttpResponse response) { } } - public CompletableFuture addToRunEvalQueue(ExampleEvaluationRun payload) { + public CompletableFuture addToRunEvalQueue(@NotNull ExampleEvaluationRun payload) { String url = buildUrl("/add_to_run_eval_queue/"); String jsonPayload; try { @@ -79,16 +76,13 @@ public CompletableFuture addToRunEvalQueue(ExampleEvaluationRun payload) throw new RuntimeException("Failed to serialize payload", e); } HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)).headers(buildHeaders()).build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } - public CompletableFuture logEvalResults(EvalResults payload) { + public CompletableFuture logEvalResults(@NotNull EvalResults payload) { String url = buildUrl("/log_eval_results/"); String jsonPayload; try { @@ -97,16 +91,13 @@ public CompletableFuture logEvalResults(EvalResults payload) { throw new RuntimeException("Failed to serialize payload", e); } HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)).headers(buildHeaders()).build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } - public CompletableFuture fetchExperimentRun(EvalResultsFetch payload) { + public CompletableFuture fetchExperimentRun(@NotNull EvalResultsFetch payload) { String url = buildUrl("/fetch_experiment_run/"); String jsonPayload; try { @@ -115,28 +106,14 @@ public CompletableFuture fetchExperimentRun(EvalResultsFetch payload) { throw new RuntimeException("Failed to serialize payload", e); } HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); - return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) - .thenApply(this::handleResponse); - } - - public CompletableFuture getEvaluationStatus( - String experiment_run_id, String project_name) { - Map queryParams = new HashMap<>(); - queryParams.put("experiment_run_id", experiment_run_id); - queryParams.put("project_name", project_name); - String url = buildUrl("/get_evaluation_status/", queryParams); - HttpRequest request = - HttpRequest.newBuilder().GET().uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)).headers(buildHeaders()).build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } - public CompletableFuture scorerExists(ScorerExistsRequest payload) { + public CompletableFuture scorerExists( + @NotNull ScorerExistsRequest payload) { String url = buildUrl("/scorer_exists/"); String jsonPayload; try { @@ -145,16 +122,14 @@ public CompletableFuture scorerExists(ScorerExistsRequest throw new RuntimeException("Failed to serialize payload", e); } HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)).headers(buildHeaders()).build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } - public CompletableFuture saveScorer(SavePromptScorerRequest payload) { + public CompletableFuture saveScorer( + @NotNull SavePromptScorerRequest payload) { String url = buildUrl("/save_scorer/"); String jsonPayload; try { @@ -163,17 +138,14 @@ public CompletableFuture saveScorer(SavePromptScorerRe throw new RuntimeException("Failed to serialize payload", e); } HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)).headers(buildHeaders()).build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } public CompletableFuture fetchScorers( - FetchPromptScorersRequest payload) { + @NotNull FetchPromptScorersRequest payload) { String url = buildUrl("/fetch_scorers/"); String jsonPayload; try { @@ -182,17 +154,14 @@ public CompletableFuture fetchScorers( throw new RuntimeException("Failed to serialize payload", e); } HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)).headers(buildHeaders()).build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } public CompletableFuture projectsResolve( - ResolveProjectNameRequest payload) { + @NotNull ResolveProjectNameRequest payload) { String url = buildUrl("/projects/resolve/"); String jsonPayload; try { @@ -201,12 +170,10 @@ public CompletableFuture projectsResolve( throw new RuntimeException("Failed to serialize payload", e); } HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)).headers(buildHeaders()).build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } + } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java index 5ea1b80..995bd68 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java @@ -7,10 +7,23 @@ import java.net.http.HttpResponse; import java.util.HashMap; import java.util.Map; +import java.util.Objects; + +import org.jetbrains.annotations.NotNull; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.judgmentlabs.judgeval.internal.api.models.*; +import com.judgmentlabs.judgeval.internal.api.models.EvalResults; +import com.judgmentlabs.judgeval.internal.api.models.EvalResultsFetch; +import com.judgmentlabs.judgeval.internal.api.models.ExampleEvaluationRun; +import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersRequest; +import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersResponse; +import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameRequest; +import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameResponse; +import com.judgmentlabs.judgeval.internal.api.models.SavePromptScorerRequest; +import com.judgmentlabs.judgeval.internal.api.models.SavePromptScorerResponse; +import com.judgmentlabs.judgeval.internal.api.models.ScorerExistsRequest; +import com.judgmentlabs.judgeval.internal.api.models.ScorerExistsResponse; public class JudgmentSyncClient { private final HttpClient client; @@ -19,10 +32,12 @@ public class JudgmentSyncClient { private final String apiKey; private final String organizationId; - public JudgmentSyncClient(String baseUrl, String apiKey, String organizationId) { - this.baseUrl = baseUrl; - this.apiKey = apiKey; - this.organizationId = organizationId; + public JudgmentSyncClient(@NotNull String baseUrl, @NotNull String apiKey, + @NotNull String organizationId) { + this.baseUrl = Objects.requireNonNull(baseUrl, "Base URL cannot be null"); + this.apiKey = Objects.requireNonNull(apiKey, "API key cannot be null"); + this.organizationId = + Objects.requireNonNull(organizationId, "Organization ID cannot be null"); this.client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build(); this.mapper = new ObjectMapper(); } @@ -45,17 +60,9 @@ private String buildUrl(String path) { } private String[] buildHeaders() { - if (apiKey == null || organizationId == null) { - throw new IllegalArgumentException("API key and organization ID cannot be null"); - } - return new String[] { - "Content-Type", - "application/json", - "Authorization", - "Bearer " + apiKey, - "X-Organization-Id", - organizationId - }; + return new String[] {"Content-Type", "application/json", "Authorization", + "Bearer " + apiKey, + "X-Organization-Id", organizationId}; } private T handleResponse(HttpResponse response) throws IOException { @@ -70,112 +77,81 @@ private T handleResponse(HttpResponse response) throws IOException { } } - public Object addToRunEvalQueue(ExampleEvaluationRun payload) + public Object addToRunEvalQueue(@NotNull ExampleEvaluationRun payload) throws IOException, InterruptedException { String url = buildUrl("/add_to_run_eval_queue/"); String jsonPayload = mapper.writeValueAsString(payload); HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)).headers(buildHeaders()).build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return handleResponse(response); } - public Object logEvalResults(EvalResults payload) throws IOException, InterruptedException { + public Object logEvalResults(@NotNull EvalResults payload) + throws IOException, InterruptedException { String url = buildUrl("/log_eval_results/"); String jsonPayload = mapper.writeValueAsString(payload); HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)).headers(buildHeaders()).build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return handleResponse(response); } - public Object fetchExperimentRun(EvalResultsFetch payload) + public Object fetchExperimentRun(@NotNull EvalResultsFetch payload) throws IOException, InterruptedException { String url = buildUrl("/fetch_experiment_run/"); String jsonPayload = mapper.writeValueAsString(payload); HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); - return handleResponse(response); - } - - public Object getEvaluationStatus(String experiment_run_id, String project_name) - throws IOException, InterruptedException { - Map queryParams = new HashMap<>(); - queryParams.put("experiment_run_id", experiment_run_id); - queryParams.put("project_name", project_name); - String url = buildUrl("/get_evaluation_status/", queryParams); - HttpRequest request = - HttpRequest.newBuilder().GET().uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)).headers(buildHeaders()).build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return handleResponse(response); } - public ScorerExistsResponse scorerExists(ScorerExistsRequest payload) + public ScorerExistsResponse scorerExists(@NotNull ScorerExistsRequest payload) throws IOException, InterruptedException { String url = buildUrl("/scorer_exists/"); String jsonPayload = mapper.writeValueAsString(payload); HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)).headers(buildHeaders()).build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return mapper.readValue(response.body(), ScorerExistsResponse.class); } - public SavePromptScorerResponse saveScorer(SavePromptScorerRequest payload) + public SavePromptScorerResponse saveScorer(@NotNull SavePromptScorerRequest payload) throws IOException, InterruptedException { String url = buildUrl("/save_scorer/"); String jsonPayload = mapper.writeValueAsString(payload); HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)).headers(buildHeaders()).build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return mapper.readValue(response.body(), SavePromptScorerResponse.class); } - public FetchPromptScorersResponse fetchScorers(FetchPromptScorersRequest payload) + public FetchPromptScorersResponse fetchScorers(@NotNull FetchPromptScorersRequest payload) throws IOException, InterruptedException { String url = buildUrl("/fetch_scorers/"); String jsonPayload = mapper.writeValueAsString(payload); HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)).headers(buildHeaders()).build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return mapper.readValue(response.body(), FetchPromptScorersResponse.class); } - public ResolveProjectNameResponse projectsResolve(ResolveProjectNameRequest payload) + public ResolveProjectNameResponse projectsResolve(@NotNull ResolveProjectNameRequest payload) throws IOException, InterruptedException { String url = buildUrl("/projects/resolve/"); String jsonPayload = mapper.writeValueAsString(payload); HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)).headers(buildHeaders()).build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return mapper.readValue(response.body(), ResolveProjectNameResponse.class); } + } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java index 72811fb..5b42a44 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java @@ -4,56 +4,44 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class BaseScorer { @JsonProperty("score_type") + @NotNull private String scoreType; - @JsonProperty("threshold") private Double threshold; - @JsonProperty("name") private String name; - @JsonProperty("class_name") private String className; - @JsonProperty("score") private Double score; - @JsonProperty("score_breakdown") private Object scoreBreakdown; - @JsonProperty("reason") private String reason; - @JsonProperty("using_native_model") private Boolean usingNativeModel; - @JsonProperty("success") private Boolean success; - @JsonProperty("model") private String model; - @JsonProperty("model_client") private Object modelClient; - @JsonProperty("strict_mode") private Boolean strictMode; - @JsonProperty("error") private String error; - @JsonProperty("additional_metadata") private Object additionalMetadata; - @JsonProperty("user") private String user; - @JsonProperty("server_hosted") private Boolean serverHosted; @@ -133,7 +121,7 @@ public Boolean getServerHosted() { return serverHosted; } - public void setScoreType(String scoreType) { + public void setScoreType(@NotNull String scoreType) { this.scoreType = scoreType; } @@ -199,19 +187,19 @@ public void setServerHosted(Boolean serverHosted) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; BaseScorer other = (BaseScorer) obj; return Objects.equals(scoreType, other.scoreType) && Objects.equals(threshold, other.threshold) - && Objects.equals(name, other.name) - && Objects.equals(className, other.className) + && Objects.equals(name, other.name) && Objects.equals(className, other.className) && Objects.equals(score, other.score) && Objects.equals(scoreBreakdown, other.scoreBreakdown) && Objects.equals(reason, other.reason) && Objects.equals(usingNativeModel, other.usingNativeModel) - && Objects.equals(success, other.success) - && Objects.equals(model, other.model) + && Objects.equals(success, other.success) && Objects.equals(model, other.model) && Objects.equals(modelClient, other.modelClient) && Objects.equals(strictMode, other.strictMode) && Objects.equals(error, other.error) @@ -223,23 +211,9 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return Objects.hash( - scoreType, - threshold, - name, - className, - score, - scoreBreakdown, - reason, - usingNativeModel, - success, - model, - modelClient, - strictMode, - error, - additionalMetadata, - user, - serverHosted, - Objects.hashCode(additionalProperties)); + return Objects.hash(scoreType, threshold, name, className, score, scoreBreakdown, reason, + usingNativeModel, success, model, modelClient, strictMode, error, + additionalMetadata, user, + serverHosted, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java index 7a5e5c5..25de32f 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java @@ -5,15 +5,18 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class EvalResults { @JsonProperty("results") + @NotNull private List results; - @JsonProperty("run") + @NotNull private Object run; private Map additionalProperties = new HashMap<>(); @@ -36,21 +39,22 @@ public Object getRun() { return run; } - public void setResults(List results) { + public void setResults(@NotNull List results) { this.results = results; } - public void setRun(Object run) { + public void setRun(@NotNull Object run) { this.run = run; } @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; EvalResults other = (EvalResults) obj; - return Objects.equals(results, other.results) - && Objects.equals(run, other.run) + return Objects.equals(results, other.results) && Objects.equals(run, other.run) && Objects.equals(additionalProperties, other.additionalProperties); } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java index 92ef79c..b6a0b35 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java @@ -4,15 +4,18 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class EvalResultsFetch { @JsonProperty("experiment_run_id") + @NotNull private String experimentRunId; - @JsonProperty("project_name") + @NotNull private String projectName; private Map additionalProperties = new HashMap<>(); @@ -35,18 +38,20 @@ public String getProjectName() { return projectName; } - public void setExperimentRunId(String experimentRunId) { + public void setExperimentRunId(@NotNull String experimentRunId) { this.experimentRunId = experimentRunId; } - public void setProjectName(String projectName) { + public void setProjectName(@NotNull String projectName) { this.projectName = projectName; } @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; EvalResultsFetch other = (EvalResultsFetch) obj; return Objects.equals(experimentRunId, other.experimentRunId) && Objects.equals(projectName, other.projectName) diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java index 23ca12b..e160933 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java @@ -11,10 +11,8 @@ public class Example { @JsonProperty("example_id") private String exampleId; - @JsonProperty("created_at") private String createdAt; - @JsonProperty("name") private String name; @@ -56,8 +54,10 @@ public void setName(String name) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; Example other = (Example) obj; return Objects.equals(exampleId, other.exampleId) && Objects.equals(createdAt, other.createdAt) diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java index e942be4..3cc0edd 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java @@ -5,6 +5,8 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; @@ -12,31 +14,25 @@ public class ExampleEvaluationRun { @JsonProperty("id") private String id; - @JsonProperty("project_name") + @NotNull private String projectName; - @JsonProperty("eval_name") + @NotNull private String evalName; - @JsonProperty("custom_scorers") private List customScorers; - @JsonProperty("judgment_scorers") private List judgmentScorers; - @JsonProperty("model") private String model; - @JsonProperty("created_at") private String createdAt; - @JsonProperty("examples") + @NotNull private List examples; - @JsonProperty("trace_span_id") private String traceSpanId; - @JsonProperty("trace_id") private String traceId; @@ -96,11 +92,11 @@ public void setId(String id) { this.id = id; } - public void setProjectName(String projectName) { + public void setProjectName(@NotNull String projectName) { this.projectName = projectName; } - public void setEvalName(String evalName) { + public void setEvalName(@NotNull String evalName) { this.evalName = evalName; } @@ -120,7 +116,7 @@ public void setCreatedAt(String createdAt) { this.createdAt = createdAt; } - public void setExamples(List examples) { + public void setExamples(@NotNull List examples) { this.examples = examples; } @@ -134,16 +130,16 @@ public void setTraceId(String traceId) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ExampleEvaluationRun other = (ExampleEvaluationRun) obj; - return Objects.equals(id, other.id) - && Objects.equals(projectName, other.projectName) + return Objects.equals(id, other.id) && Objects.equals(projectName, other.projectName) && Objects.equals(evalName, other.evalName) && Objects.equals(customScorers, other.customScorers) && Objects.equals(judgmentScorers, other.judgmentScorers) - && Objects.equals(model, other.model) - && Objects.equals(createdAt, other.createdAt) + && Objects.equals(model, other.model) && Objects.equals(createdAt, other.createdAt) && Objects.equals(examples, other.examples) && Objects.equals(traceSpanId, other.traceSpanId) && Objects.equals(traceId, other.traceId) @@ -152,17 +148,8 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return Objects.hash( - id, - projectName, - evalName, - customScorers, - judgmentScorers, - model, + return Objects.hash(id, projectName, evalName, customScorers, judgmentScorers, model, createdAt, - examples, - traceSpanId, - traceId, - Objects.hashCode(additionalProperties)); + examples, traceSpanId, traceId, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java index 49bd079..79f6be3 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java @@ -12,6 +12,8 @@ public class FetchPromptScorersRequest { @JsonProperty("names") private List names; + @JsonProperty("is_trace") + private Boolean isTrace; private Map additionalProperties = new HashMap<>(); @@ -29,21 +31,31 @@ public List getNames() { return names; } + public Boolean getIsTrace() { + return isTrace; + } + public void setNames(List names) { this.names = names; } + public void setIsTrace(Boolean isTrace) { + this.isTrace = isTrace; + } + @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; FetchPromptScorersRequest other = (FetchPromptScorersRequest) obj; - return Objects.equals(names, other.names) + return Objects.equals(names, other.names) && Objects.equals(isTrace, other.isTrace) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash(names, Objects.hashCode(additionalProperties)); + return Objects.hash(names, isTrace, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java index 7bfe803..478b9ef 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java @@ -5,12 +5,15 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class FetchPromptScorersResponse { @JsonProperty("scorers") + @NotNull private List scorers; private Map additionalProperties = new HashMap<>(); @@ -29,14 +32,16 @@ public List getScorers() { return scorers; } - public void setScorers(List scorers) { + public void setScorers(@NotNull List scorers) { this.scorers = scorers; } @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; FetchPromptScorersResponse other = (FetchPromptScorersResponse) obj; return Objects.equals(scorers, other.scorers) && Objects.equals(additionalProperties, other.additionalProperties); diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java index 9755864..076c555 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java @@ -5,89 +5,55 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class OtelTraceSpan { @JsonProperty("organization_id") + @NotNull private String organizationId; - @JsonProperty("project_id") private String projectId; - @JsonProperty("user_id") + @NotNull private String userId; - @JsonProperty("timestamp") + @NotNull private String timestamp; - @JsonProperty("trace_id") + @NotNull private String traceId; - @JsonProperty("span_id") + @NotNull private String spanId; - @JsonProperty("parent_span_id") private String parentSpanId; - @JsonProperty("trace_state") private String traceState; - @JsonProperty("span_name") private String spanName; - @JsonProperty("span_kind") private String spanKind; - @JsonProperty("service_name") private String serviceName; - @JsonProperty("resource_attributes") private Object resourceAttributes; - @JsonProperty("span_attributes") private Object spanAttributes; - @JsonProperty("duration") private Integer duration; - @JsonProperty("status_code") - private String statusCode; - + private Integer statusCode; @JsonProperty("status_message") private String statusMessage; - @JsonProperty("events") private List events; - @JsonProperty("links") private List links; - @JsonProperty("legacy_span_id") - private String legacySpanId; - - @JsonProperty("inputs") - private Object inputs; - - @JsonProperty("output") - private Object output; - - @JsonProperty("error") - private Object error; - - @JsonProperty("agent_id") - private String agentId; - - @JsonProperty("cumulative_llm_cost") - private Double cumulativeLlmCost; - - @JsonProperty("state_after") - private Object stateAfter; - - @JsonProperty("state_before") - private Object stateBefore; - private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -156,7 +122,7 @@ public Integer getDuration() { return duration; } - public String getStatusCode() { + public Integer getStatusCode() { return statusCode; } @@ -172,39 +138,7 @@ public List getLinks() { return links; } - public String getLegacySpanId() { - return legacySpanId; - } - - public Object getInputs() { - return inputs; - } - - public Object getOutput() { - return output; - } - - public Object getError() { - return error; - } - - public String getAgentId() { - return agentId; - } - - public Double getCumulativeLlmCost() { - return cumulativeLlmCost; - } - - public Object getStateAfter() { - return stateAfter; - } - - public Object getStateBefore() { - return stateBefore; - } - - public void setOrganizationId(String organizationId) { + public void setOrganizationId(@NotNull String organizationId) { this.organizationId = organizationId; } @@ -212,19 +146,19 @@ public void setProjectId(String projectId) { this.projectId = projectId; } - public void setUserId(String userId) { + public void setUserId(@NotNull String userId) { this.userId = userId; } - public void setTimestamp(String timestamp) { + public void setTimestamp(@NotNull String timestamp) { this.timestamp = timestamp; } - public void setTraceId(String traceId) { + public void setTraceId(@NotNull String traceId) { this.traceId = traceId; } - public void setSpanId(String spanId) { + public void setSpanId(@NotNull String spanId) { this.spanId = spanId; } @@ -260,7 +194,7 @@ public void setDuration(Integer duration) { this.duration = duration; } - public void setStatusCode(String statusCode) { + public void setStatusCode(Integer statusCode) { this.statusCode = statusCode; } @@ -276,42 +210,12 @@ public void setLinks(List links) { this.links = links; } - public void setLegacySpanId(String legacySpanId) { - this.legacySpanId = legacySpanId; - } - - public void setInputs(Object inputs) { - this.inputs = inputs; - } - - public void setOutput(Object output) { - this.output = output; - } - - public void setError(Object error) { - this.error = error; - } - - public void setAgentId(String agentId) { - this.agentId = agentId; - } - - public void setCumulativeLlmCost(Double cumulativeLlmCost) { - this.cumulativeLlmCost = cumulativeLlmCost; - } - - public void setStateAfter(Object stateAfter) { - this.stateAfter = stateAfter; - } - - public void setStateBefore(Object stateBefore) { - this.stateBefore = stateBefore; - } - @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; OtelTraceSpan other = (OtelTraceSpan) obj; return Objects.equals(organizationId, other.organizationId) && Objects.equals(projectId, other.projectId) @@ -329,48 +233,16 @@ public boolean equals(Object obj) { && Objects.equals(duration, other.duration) && Objects.equals(statusCode, other.statusCode) && Objects.equals(statusMessage, other.statusMessage) - && Objects.equals(events, other.events) - && Objects.equals(links, other.links) - && Objects.equals(legacySpanId, other.legacySpanId) - && Objects.equals(inputs, other.inputs) - && Objects.equals(output, other.output) - && Objects.equals(error, other.error) - && Objects.equals(agentId, other.agentId) - && Objects.equals(cumulativeLlmCost, other.cumulativeLlmCost) - && Objects.equals(stateAfter, other.stateAfter) - && Objects.equals(stateBefore, other.stateBefore) + && Objects.equals(events, other.events) && Objects.equals(links, other.links) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash( - organizationId, - projectId, - userId, - timestamp, - traceId, - spanId, + return Objects.hash(organizationId, projectId, userId, timestamp, traceId, spanId, parentSpanId, - traceState, - spanName, - spanKind, - serviceName, - resourceAttributes, - spanAttributes, + traceState, spanName, spanKind, serviceName, resourceAttributes, spanAttributes, duration, - statusCode, - statusMessage, - events, - links, - legacySpanId, - inputs, - output, - error, - agentId, - cumulativeLlmCost, - stateAfter, - stateBefore, - Objects.hashCode(additionalProperties)); + statusCode, statusMessage, events, links, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java index 8327092..c2d8ba2 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java @@ -4,31 +4,45 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class PromptScorer { + @JsonProperty("id") + @NotNull + private String id; + @JsonProperty("user_id") + @NotNull + private String userId; + @JsonProperty("organization_id") + @NotNull + private String organizationId; @JsonProperty("name") + @NotNull private String name; - @JsonProperty("prompt") + @NotNull private String prompt; - @JsonProperty("threshold") + @NotNull private Double threshold; - + @JsonProperty("model") + private String model; @JsonProperty("options") private Object options; - + @JsonProperty("description") + private String description; @JsonProperty("created_at") private String createdAt; - @JsonProperty("updated_at") private String updatedAt; - @JsonProperty("is_trace") private Boolean isTrace; + @JsonProperty("is_bucket_rubric") + private Boolean isBucketRubric; private Map additionalProperties = new HashMap<>(); @@ -42,6 +56,18 @@ public void setAdditionalProperty(String name, Object value) { additionalProperties.put(name, value); } + public String getId() { + return id; + } + + public String getUserId() { + return userId; + } + + public String getOrganizationId() { + return organizationId; + } + public String getName() { return name; } @@ -54,10 +80,18 @@ public Double getThreshold() { return threshold; } + public String getModel() { + return model; + } + public Object getOptions() { return options; } + public String getDescription() { + return description; + } + public String getCreatedAt() { return createdAt; } @@ -70,22 +104,46 @@ public Boolean getIsTrace() { return isTrace; } - public void setName(String name) { + public Boolean getIsBucketRubric() { + return isBucketRubric; + } + + public void setId(@NotNull String id) { + this.id = id; + } + + public void setUserId(@NotNull String userId) { + this.userId = userId; + } + + public void setOrganizationId(@NotNull String organizationId) { + this.organizationId = organizationId; + } + + public void setName(@NotNull String name) { this.name = name; } - public void setPrompt(String prompt) { + public void setPrompt(@NotNull String prompt) { this.prompt = prompt; } - public void setThreshold(Double threshold) { + public void setThreshold(@NotNull Double threshold) { this.threshold = threshold; } + public void setModel(String model) { + this.model = model; + } + public void setOptions(Object options) { this.options = options; } + public void setDescription(String description) { + this.description = description; + } + public void setCreatedAt(String createdAt) { this.createdAt = createdAt; } @@ -98,31 +156,35 @@ public void setIsTrace(Boolean isTrace) { this.isTrace = isTrace; } + public void setIsBucketRubric(Boolean isBucketRubric) { + this.isBucketRubric = isBucketRubric; + } + @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; PromptScorer other = (PromptScorer) obj; - return Objects.equals(name, other.name) + return Objects.equals(id, other.id) && Objects.equals(userId, other.userId) + && Objects.equals(organizationId, other.organizationId) + && Objects.equals(name, other.name) && Objects.equals(prompt, other.prompt) && Objects.equals(threshold, other.threshold) - && Objects.equals(options, other.options) + && Objects.equals(model, other.model) && Objects.equals(options, other.options) + && Objects.equals(description, other.description) && Objects.equals(createdAt, other.createdAt) && Objects.equals(updatedAt, other.updatedAt) && Objects.equals(isTrace, other.isTrace) + && Objects.equals(isBucketRubric, other.isBucketRubric) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash( - name, - prompt, - threshold, - options, - createdAt, - updatedAt, - isTrace, + return Objects.hash(id, userId, organizationId, name, prompt, threshold, model, options, + description, createdAt, updatedAt, isTrace, isBucketRubric, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java index cf1cb41..5398b07 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java @@ -4,12 +4,15 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class ResolveProjectNameRequest { @JsonProperty("project_name") + @NotNull private String projectName; private Map additionalProperties = new HashMap<>(); @@ -28,14 +31,16 @@ public String getProjectName() { return projectName; } - public void setProjectName(String projectName) { + public void setProjectName(@NotNull String projectName) { this.projectName = projectName; } @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ResolveProjectNameRequest other = (ResolveProjectNameRequest) obj; return Objects.equals(projectName, other.projectName) && Objects.equals(additionalProperties, other.additionalProperties); diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java index d668f8b..94cc197 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java @@ -4,12 +4,15 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class ResolveProjectNameResponse { @JsonProperty("project_id") + @NotNull private String projectId; private Map additionalProperties = new HashMap<>(); @@ -28,14 +31,16 @@ public String getProjectId() { return projectId; } - public void setProjectId(String projectId) { + public void setProjectId(@NotNull String projectId) { this.projectId = projectId; } @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ResolveProjectNameResponse other = (ResolveProjectNameResponse) obj; return Objects.equals(projectId, other.projectId) && Objects.equals(additionalProperties, other.additionalProperties); diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java index 62be159..dc19cc3 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java @@ -4,25 +4,30 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class SavePromptScorerRequest { @JsonProperty("name") + @NotNull private String name; - @JsonProperty("prompt") + @NotNull private String prompt; - @JsonProperty("threshold") + @NotNull private Double threshold; - - @JsonProperty("options") - private Object options; - + @JsonProperty("model") + private String model; @JsonProperty("is_trace") private Boolean isTrace; + @JsonProperty("options") + private Object options; + @JsonProperty("description") + private String description; private Map additionalProperties = new HashMap<>(); @@ -48,50 +53,67 @@ public Double getThreshold() { return threshold; } - public Object getOptions() { - return options; + public String getModel() { + return model; } public Boolean getIsTrace() { return isTrace; } - public void setName(String name) { + public Object getOptions() { + return options; + } + + public String getDescription() { + return description; + } + + public void setName(@NotNull String name) { this.name = name; } - public void setPrompt(String prompt) { + public void setPrompt(@NotNull String prompt) { this.prompt = prompt; } - public void setThreshold(Double threshold) { + public void setThreshold(@NotNull Double threshold) { this.threshold = threshold; } - public void setOptions(Object options) { - this.options = options; + public void setModel(String model) { + this.model = model; } public void setIsTrace(Boolean isTrace) { this.isTrace = isTrace; } + public void setOptions(Object options) { + this.options = options; + } + + public void setDescription(String description) { + this.description = description; + } + @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; SavePromptScorerRequest other = (SavePromptScorerRequest) obj; - return Objects.equals(name, other.name) - && Objects.equals(prompt, other.prompt) - && Objects.equals(threshold, other.threshold) - && Objects.equals(options, other.options) - && Objects.equals(isTrace, other.isTrace) + return Objects.equals(name, other.name) && Objects.equals(prompt, other.prompt) + && Objects.equals(threshold, other.threshold) && Objects.equals(model, other.model) + && Objects.equals(isTrace, other.isTrace) && Objects.equals(options, other.options) + && Objects.equals(description, other.description) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash( - name, prompt, threshold, options, isTrace, Objects.hashCode(additionalProperties)); + return Objects.hash(name, prompt, threshold, model, isTrace, options, description, + Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java index e126892..8748242 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java @@ -4,16 +4,16 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class SavePromptScorerResponse { - @JsonProperty("message") - private String message; - - @JsonProperty("name") - private String name; + @JsonProperty("scorer_response") + @NotNull + private PromptScorer scorerResponse; private Map additionalProperties = new HashMap<>(); @@ -27,34 +27,27 @@ public void setAdditionalProperty(String name, Object value) { additionalProperties.put(name, value); } - public String getMessage() { - return message; - } - - public String getName() { - return name; - } - - public void setMessage(String message) { - this.message = message; + public PromptScorer getScorerResponse() { + return scorerResponse; } - public void setName(String name) { - this.name = name; + public void setScorerResponse(@NotNull PromptScorer scorerResponse) { + this.scorerResponse = scorerResponse; } @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; SavePromptScorerResponse other = (SavePromptScorerResponse) obj; - return Objects.equals(message, other.message) - && Objects.equals(name, other.name) + return Objects.equals(scorerResponse, other.scorerResponse) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash(message, name, Objects.hashCode(additionalProperties)); + return Objects.hash(scorerResponse, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java index d312ee1..da2cd2c 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java @@ -5,26 +5,26 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class ScorerConfig { @JsonProperty("score_type") + @NotNull private String scoreType; - @JsonProperty("name") private String name; - @JsonProperty("threshold") private Double threshold; - + @JsonProperty("model") + private String model; @JsonProperty("strict_mode") private Boolean strictMode; - @JsonProperty("required_params") private List requiredParams; - @JsonProperty("kwargs") private Object kwargs; @@ -52,6 +52,10 @@ public Double getThreshold() { return threshold; } + public String getModel() { + return model; + } + public Boolean getStrictMode() { return strictMode; } @@ -64,7 +68,7 @@ public Object getKwargs() { return kwargs; } - public void setScoreType(String scoreType) { + public void setScoreType(@NotNull String scoreType) { this.scoreType = scoreType; } @@ -76,6 +80,10 @@ public void setThreshold(Double threshold) { this.threshold = threshold; } + public void setModel(String model) { + this.model = model; + } + public void setStrictMode(Boolean strictMode) { this.strictMode = strictMode; } @@ -90,12 +98,13 @@ public void setKwargs(Object kwargs) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ScorerConfig other = (ScorerConfig) obj; - return Objects.equals(scoreType, other.scoreType) - && Objects.equals(name, other.name) - && Objects.equals(threshold, other.threshold) + return Objects.equals(scoreType, other.scoreType) && Objects.equals(name, other.name) + && Objects.equals(threshold, other.threshold) && Objects.equals(model, other.model) && Objects.equals(strictMode, other.strictMode) && Objects.equals(requiredParams, other.requiredParams) && Objects.equals(kwargs, other.kwargs) @@ -104,13 +113,7 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return Objects.hash( - scoreType, - name, - threshold, - strictMode, - requiredParams, - kwargs, + return Objects.hash(scoreType, name, threshold, model, strictMode, requiredParams, kwargs, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java index 62a7419..a72c72b 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java @@ -4,6 +4,8 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; @@ -11,31 +13,25 @@ public class ScorerData { @JsonProperty("id") private String id; - @JsonProperty("name") + @NotNull private String name; - @JsonProperty("threshold") + @NotNull private Double threshold; - @JsonProperty("success") + @NotNull private Boolean success; - @JsonProperty("score") private Double score; - @JsonProperty("reason") private String reason; - @JsonProperty("strict_mode") private Boolean strictMode; - @JsonProperty("evaluation_model") private String evaluationModel; - @JsonProperty("error") private String error; - @JsonProperty("additional_metadata") private Object additionalMetadata; @@ -95,15 +91,15 @@ public void setId(String id) { this.id = id; } - public void setName(String name) { + public void setName(@NotNull String name) { this.name = name; } - public void setThreshold(Double threshold) { + public void setThreshold(@NotNull Double threshold) { this.threshold = threshold; } - public void setSuccess(Boolean success) { + public void setSuccess(@NotNull Boolean success) { this.success = success; } @@ -133,15 +129,15 @@ public void setAdditionalMetadata(Object additionalMetadata) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ScorerData other = (ScorerData) obj; - return Objects.equals(id, other.id) - && Objects.equals(name, other.name) + return Objects.equals(id, other.id) && Objects.equals(name, other.name) && Objects.equals(threshold, other.threshold) && Objects.equals(success, other.success) - && Objects.equals(score, other.score) - && Objects.equals(reason, other.reason) + && Objects.equals(score, other.score) && Objects.equals(reason, other.reason) && Objects.equals(strictMode, other.strictMode) && Objects.equals(evaluationModel, other.evaluationModel) && Objects.equals(error, other.error) @@ -151,17 +147,8 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return Objects.hash( - id, - name, - threshold, - success, - score, - reason, - strictMode, + return Objects.hash(id, name, threshold, success, score, reason, strictMode, evaluationModel, - error, - additionalMetadata, - Objects.hashCode(additionalProperties)); + error, additionalMetadata, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java index 809e4d4..540b646 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java @@ -4,12 +4,15 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class ScorerExistsRequest { @JsonProperty("name") + @NotNull private String name; private Map additionalProperties = new HashMap<>(); @@ -28,14 +31,16 @@ public String getName() { return name; } - public void setName(String name) { + public void setName(@NotNull String name) { this.name = name; } @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ScorerExistsRequest other = (ScorerExistsRequest) obj; return Objects.equals(name, other.name) && Objects.equals(additionalProperties, other.additionalProperties); diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java index 1dafef1..8bd564d 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java @@ -4,12 +4,15 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class ScorerExistsResponse { @JsonProperty("exists") + @NotNull private Boolean exists; private Map additionalProperties = new HashMap<>(); @@ -28,14 +31,16 @@ public Boolean getExists() { return exists; } - public void setExists(Boolean exists) { + public void setExists(@NotNull Boolean exists) { this.exists = exists; } @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ScorerExistsResponse other = (ScorerExistsResponse) obj; return Objects.equals(exists, other.exists) && Objects.equals(additionalProperties, other.additionalProperties); diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java index 215d06f..27bb357 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java @@ -5,29 +5,27 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class ScoringResult { @JsonProperty("success") + @NotNull private Boolean success; - @JsonProperty("scorers_data") + @NotNull private List scorersData; - @JsonProperty("name") private String name; - @JsonProperty("data_object") private Object dataObject; - @JsonProperty("trace_id") private String traceId; - @JsonProperty("run_duration") private Double runDuration; - @JsonProperty("evaluation_cost") private Double evaluationCost; @@ -71,11 +69,11 @@ public Double getEvaluationCost() { return evaluationCost; } - public void setSuccess(Boolean success) { + public void setSuccess(@NotNull Boolean success) { this.success = success; } - public void setScorersData(List scorersData) { + public void setScorersData(@NotNull List scorersData) { this.scorersData = scorersData; } @@ -101,13 +99,14 @@ public void setEvaluationCost(Double evaluationCost) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ScoringResult other = (ScoringResult) obj; return Objects.equals(success, other.success) && Objects.equals(scorersData, other.scorersData) - && Objects.equals(name, other.name) - && Objects.equals(dataObject, other.dataObject) + && Objects.equals(name, other.name) && Objects.equals(dataObject, other.dataObject) && Objects.equals(traceId, other.traceId) && Objects.equals(runDuration, other.runDuration) && Objects.equals(evaluationCost, other.evaluationCost) @@ -116,14 +115,7 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return Objects.hash( - success, - scorersData, - name, - dataObject, - traceId, - runDuration, - evaluationCost, - Objects.hashCode(additionalProperties)); + return Objects.hash(success, scorersData, name, dataObject, traceId, runDuration, + evaluationCost, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java index 0961e3d..af949aa 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java @@ -5,6 +5,8 @@ import java.util.Map; import java.util.Objects; +import org.jetbrains.annotations.NotNull; + import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; @@ -12,30 +14,27 @@ public class TraceEvaluationRun { @JsonProperty("id") private String id; - @JsonProperty("project_name") + @NotNull private String projectName; - @JsonProperty("eval_name") + @NotNull private String evalName; - @JsonProperty("custom_scorers") private List customScorers; - @JsonProperty("judgment_scorers") private List judgmentScorers; - @JsonProperty("model") private String model; - @JsonProperty("created_at") private String createdAt; - @JsonProperty("trace_and_span_ids") + @NotNull private List> traceAndSpanIds; - @JsonProperty("is_offline") private Boolean isOffline; + @JsonProperty("is_bucket_run") + private Boolean isBucketRun; private Map additionalProperties = new HashMap<>(); @@ -85,15 +84,19 @@ public Boolean getIsOffline() { return isOffline; } + public Boolean getIsBucketRun() { + return isBucketRun; + } + public void setId(String id) { this.id = id; } - public void setProjectName(String projectName) { + public void setProjectName(@NotNull String projectName) { this.projectName = projectName; } - public void setEvalName(String evalName) { + public void setEvalName(@NotNull String evalName) { this.evalName = evalName; } @@ -113,7 +116,7 @@ public void setCreatedAt(String createdAt) { this.createdAt = createdAt; } - public void setTraceAndSpanIds(List> traceAndSpanIds) { + public void setTraceAndSpanIds(@NotNull List> traceAndSpanIds) { this.traceAndSpanIds = traceAndSpanIds; } @@ -121,35 +124,32 @@ public void setIsOffline(Boolean isOffline) { this.isOffline = isOffline; } + public void setIsBucketRun(Boolean isBucketRun) { + this.isBucketRun = isBucketRun; + } + @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; TraceEvaluationRun other = (TraceEvaluationRun) obj; - return Objects.equals(id, other.id) - && Objects.equals(projectName, other.projectName) + return Objects.equals(id, other.id) && Objects.equals(projectName, other.projectName) && Objects.equals(evalName, other.evalName) && Objects.equals(customScorers, other.customScorers) && Objects.equals(judgmentScorers, other.judgmentScorers) - && Objects.equals(model, other.model) - && Objects.equals(createdAt, other.createdAt) + && Objects.equals(model, other.model) && Objects.equals(createdAt, other.createdAt) && Objects.equals(traceAndSpanIds, other.traceAndSpanIds) && Objects.equals(isOffline, other.isOffline) + && Objects.equals(isBucketRun, other.isBucketRun) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash( - id, - projectName, - evalName, - customScorers, - judgmentScorers, - model, + return Objects.hash(id, projectName, evalName, customScorers, judgmentScorers, model, createdAt, - traceAndSpanIds, - isOffline, - Objects.hashCode(additionalProperties)); + traceAndSpanIds, isOffline, isBucketRun, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java index a11e6f1..952d614 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java @@ -12,7 +12,8 @@ public class APIScorer extends com.judgmentlabs.judgeval.internal.api.models.Bas implements BaseScorer { private APIScorerType scoreType; - @JsonIgnore private List requiredParams; + @JsonIgnore + private List requiredParams; public APIScorer(APIScorerType scoreType) { super(); @@ -72,7 +73,8 @@ public ScorerConfig getScorerConfig() { cfg.setStrictMode(getStrictMode()); cfg.setRequiredParams(getRequiredParams()); Map kwargs = new HashMap<>(); - if (getAdditionalProperties() != null) kwargs.putAll(getAdditionalProperties()); + if (getAdditionalProperties() != null) + kwargs.putAll(getAdditionalProperties()); cfg.setKwargs(kwargs); return cfg; } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java index e0d666f..a0a428f 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java @@ -23,14 +23,9 @@ public abstract class BasePromptScorer extends APIScorer { protected String judgmentApiKey; protected String organizationId; - protected BasePromptScorer( - APIScorerType scoreType, - String name, - String prompt, + protected BasePromptScorer(APIScorerType scoreType, String name, String prompt, double threshold, - Map options, - String judgmentApiKey, - String organizationId) { + Map options, String judgmentApiKey, String organizationId) { super(scoreType); this.prompt = prompt; this.options = options; @@ -50,16 +45,14 @@ public static boolean scorerExists(String name, String judgmentApiKey, String or return Boolean.TRUE.equals(response.getExists()); } catch (JudgmentAPIError e) { if (e.getStatusCode() == 500) { - throw new JudgmentAPIError( - e.getStatusCode(), + throw new JudgmentAPIError(e.getStatusCode(), "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " + e.getMessage()); } - throw new JudgmentAPIError( - e.getStatusCode(), "Failed to check if scorer exists: " + e.getMessage()); + throw new JudgmentAPIError(e.getStatusCode(), + "Failed to check if scorer exists: " + e.getMessage()); } catch (IOException | InterruptedException e) { - throw new JudgmentAPIError( - 500, + throw new JudgmentAPIError(500, "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " + e.getMessage()); } @@ -76,36 +69,28 @@ public static com.judgmentlabs.judgeval.internal.api.models.PromptScorer fetchPr FetchPromptScorersResponse response = client.fetchScorers(request); if (response.getScorers() == null || response.getScorers().isEmpty()) { - throw new JudgmentAPIError( - 404, "Failed to fetch prompt scorer '" + name + "': not found"); + throw new JudgmentAPIError(404, + "Failed to fetch prompt scorer '" + name + "': not found"); } return response.getScorers().get(0); } catch (JudgmentAPIError e) { if (e.getStatusCode() == 500) { - throw new JudgmentAPIError( - e.getStatusCode(), + throw new JudgmentAPIError(e.getStatusCode(), "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " + e.getMessage()); } - throw new JudgmentAPIError( - e.getStatusCode(), + throw new JudgmentAPIError(e.getStatusCode(), "Failed to fetch prompt scorer '" + name + "': " + e.getMessage()); } catch (IOException | InterruptedException e) { - throw new JudgmentAPIError( - 500, + throw new JudgmentAPIError(500, "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " + e.getMessage()); } } - public static String pushPromptScorer( - String name, - String prompt, - double threshold, - Map options, - String judgmentApiKey, - String organizationId, + public static String pushPromptScorer(String name, String prompt, double threshold, + Map options, String judgmentApiKey, String organizationId, Boolean isTrace) { try { JudgmentSyncClient client = @@ -125,19 +110,17 @@ public static String pushPromptScorer( request.setIsTrace(isTrace); SavePromptScorerResponse response = client.saveScorer(request); - return response != null ? response.getName() : null; + return response != null ? response.getScorerResponse().getName() : null; } catch (JudgmentAPIError e) { if (e.getStatusCode() == 500) { - throw new JudgmentAPIError( - e.getStatusCode(), + throw new JudgmentAPIError(e.getStatusCode(), "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " + e.getMessage()); } - throw new JudgmentAPIError( - e.getStatusCode(), "Failed to save prompt scorer: " + e.getMessage()); + throw new JudgmentAPIError(e.getStatusCode(), + "Failed to save prompt scorer: " + e.getMessage()); } catch (IOException | InterruptedException e) { - throw new JudgmentAPIError( - 500, + throw new JudgmentAPIError(500, "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " + e.getMessage()); } @@ -192,13 +175,7 @@ public Map getConfig() { } protected void pushPromptScorer() { - pushPromptScorer( - getName(), - prompt, - getThreshold(), - options, - judgmentApiKey, - organizationId, + pushPromptScorer(getName(), prompt, getThreshold(), options, judgmentApiKey, organizationId, isTrace()); } @@ -206,14 +183,8 @@ protected void pushPromptScorer() { @Override public String toString() { - return "PromptScorer(name=" - + getName() - + ", prompt=" - + prompt - + ", threshold=" + return "PromptScorer(name=" + getName() + ", prompt=" + prompt + ", threshold=" + getThreshold() - + ", options=" - + options - + ")"; + + ", options=" + options + ")"; } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java index d01cc96..ea4194b 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java @@ -11,30 +11,13 @@ public class PromptScorer extends BasePromptScorer { public PromptScorer(String name, String prompt, double threshold, Map options) { - super( - APIScorerType.PROMPT_SCORER, - name, - prompt, - threshold, - options, - Env.JUDGMENT_API_KEY, + super(APIScorerType.PROMPT_SCORER, name, prompt, threshold, options, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); } - public PromptScorer( - String name, - String prompt, - double threshold, - Map options, - String judgmentApiKey, - String organizationId) { - super( - APIScorerType.PROMPT_SCORER, - name, - prompt, - threshold, - options, - judgmentApiKey, + public PromptScorer(String name, String prompt, double threshold, Map options, + String judgmentApiKey, String organizationId) { + super(APIScorerType.PROMPT_SCORER, name, prompt, threshold, options, judgmentApiKey, organizationId); } @@ -64,13 +47,9 @@ public static PromptScorer get(String name, String judgmentApiKey, String organi } } - return new PromptScorer( - name, - scorerConfig.getPrompt(), - scorerConfig.getThreshold() != null ? scorerConfig.getThreshold() : 0.5, - options, - judgmentApiKey, - organizationId); + return new PromptScorer(name, scorerConfig.getPrompt(), + scorerConfig.getThreshold() != null ? scorerConfig.getThreshold() : 0.5, options, + judgmentApiKey, organizationId); } public static PromptScorer create(String name, String prompt) { @@ -81,29 +60,21 @@ public static PromptScorer create(String name, String prompt, double threshold) return create(name, prompt, threshold, null); } - public static PromptScorer create( - String name, String prompt, double threshold, Map options) { + public static PromptScorer create(String name, String prompt, double threshold, + Map options) { return create(name, prompt, threshold, options, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); } - public static PromptScorer create( - String name, - String prompt, - double threshold, - Map options, - String judgmentApiKey, - String organizationId) { + public static PromptScorer create(String name, String prompt, double threshold, + Map options, String judgmentApiKey, String organizationId) { if (!scorerExists(name, judgmentApiKey, organizationId)) { - pushPromptScorer( - name, prompt, threshold, options, judgmentApiKey, organizationId, false); - return new PromptScorer( - name, prompt, threshold, options, judgmentApiKey, organizationId); + pushPromptScorer(name, prompt, threshold, options, judgmentApiKey, organizationId, + false); + return new PromptScorer(name, prompt, threshold, options, judgmentApiKey, + organizationId); } else { - throw new JudgmentAPIError( - 400, - "Scorer with name " - + name - + " already exists. Either use the existing scorer with the get() method or use a new name."); + throw new JudgmentAPIError(400, "Scorer with name " + name + + " already exists. Either use the existing scorer with the get() method or use a new name."); } } @@ -197,8 +168,8 @@ public PromptScorer build() { if (name == null || prompt == null) { throw new IllegalArgumentException("Name and prompt are required"); } - return new PromptScorer( - name, prompt, threshold, options, judgmentApiKey, organizationId); + return new PromptScorer(name, prompt, threshold, options, judgmentApiKey, + organizationId); } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java index 94d0b03..2275d3b 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java @@ -10,32 +10,16 @@ public class TracePromptScorer extends BasePromptScorer { - public TracePromptScorer( - String name, String prompt, double threshold, Map options) { - super( - APIScorerType.TRACE_PROMPT_SCORER, - name, - prompt, - threshold, - options, + public TracePromptScorer(String name, String prompt, double threshold, + Map options) { + super(APIScorerType.TRACE_PROMPT_SCORER, name, prompt, threshold, options, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); } - public TracePromptScorer( - String name, - String prompt, - double threshold, - Map options, - String judgmentApiKey, - String organizationId) { - super( - APIScorerType.TRACE_PROMPT_SCORER, - name, - prompt, - threshold, - options, - judgmentApiKey, + public TracePromptScorer(String name, String prompt, double threshold, + Map options, String judgmentApiKey, String organizationId) { + super(APIScorerType.TRACE_PROMPT_SCORER, name, prompt, threshold, options, judgmentApiKey, organizationId); } @@ -48,8 +32,8 @@ public static TracePromptScorer get(String name, String judgmentApiKey, String o fetchPromptScorer(name, judgmentApiKey, organizationId); if (!Boolean.TRUE.equals(scorerConfig.getIsTrace())) { - throw new JudgmentAPIError( - 400, "Scorer with name " + name + " is not a TracePromptScorer"); + throw new JudgmentAPIError(400, + "Scorer with name " + name + " is not a TracePromptScorer"); } Map options = null; @@ -66,13 +50,9 @@ public static TracePromptScorer get(String name, String judgmentApiKey, String o } } - return new TracePromptScorer( - name, - scorerConfig.getPrompt(), - scorerConfig.getThreshold() != null ? scorerConfig.getThreshold() : 0.5, - options, - judgmentApiKey, - organizationId); + return new TracePromptScorer(name, scorerConfig.getPrompt(), + scorerConfig.getThreshold() != null ? scorerConfig.getThreshold() : 0.5, options, + judgmentApiKey, organizationId); } public static TracePromptScorer create(String name, String prompt) { @@ -83,29 +63,21 @@ public static TracePromptScorer create(String name, String prompt, double thresh return create(name, prompt, threshold, null); } - public static TracePromptScorer create( - String name, String prompt, double threshold, Map options) { + public static TracePromptScorer create(String name, String prompt, double threshold, + Map options) { return create(name, prompt, threshold, options, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); } - public static TracePromptScorer create( - String name, - String prompt, - double threshold, - Map options, - String judgmentApiKey, - String organizationId) { + public static TracePromptScorer create(String name, String prompt, double threshold, + Map options, String judgmentApiKey, String organizationId) { if (!scorerExists(name, judgmentApiKey, organizationId)) { - pushPromptScorer( - name, prompt, threshold, options, judgmentApiKey, organizationId, true); - return new TracePromptScorer( - name, prompt, threshold, options, judgmentApiKey, organizationId); + pushPromptScorer(name, prompt, threshold, options, judgmentApiKey, organizationId, + true); + return new TracePromptScorer(name, prompt, threshold, options, judgmentApiKey, + organizationId); } else { - throw new JudgmentAPIError( - 400, - "Scorer with name " - + name - + " already exists. Either use the existing scorer with the get() method or use a new name."); + throw new JudgmentAPIError(400, "Scorer with name " + name + + " already exists. Either use the existing scorer with the get() method or use a new name."); } } @@ -199,8 +171,8 @@ public TracePromptScorer build() { if (name == null || prompt == null) { throw new IllegalArgumentException("Name and prompt are required"); } - return new TracePromptScorer( - name, prompt, threshold, options, judgmentApiKey, organizationId); + return new TracePromptScorer(name, prompt, threshold, options, judgmentApiKey, + organizationId); } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java index abf454b..f57f3a2 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java @@ -4,6 +4,9 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; + +import org.jetbrains.annotations.NotNull; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; @@ -33,220 +36,306 @@ public abstract class BaseTracer { protected final JudgmentSyncClient apiClient; protected final ISerializer serializer; protected final ObjectMapper jacksonMapper; - protected final String projectId; + protected final Optional projectId; - protected BaseTracer( - TracerConfiguration configuration, ISerializer serializer, boolean initialize) { + protected BaseTracer(@NotNull TracerConfiguration configuration, + @NotNull ISerializer serializer, boolean initialize) { this.configuration = Objects.requireNonNull(configuration, "Configuration cannot be null"); - this.apiClient = - new JudgmentSyncClient( - configuration.apiUrl(), - configuration.apiKey(), - configuration.organizationId()); + this.apiClient = new JudgmentSyncClient(configuration.apiUrl(), configuration.apiKey(), + configuration.organizationId()); this.serializer = Objects.requireNonNull(serializer, "Serializer cannot be null"); this.jacksonMapper = new ObjectMapper(); this.projectId = resolveProjectId(configuration.projectName()); - if (this.projectId == null) { - Logger.error( - "Failed to resolve project " - + configuration.projectName() - + ", please create it first at https://app.judgmentlabs.ai/org/" - + configuration.organizationId() - + "/projects. Skipping Judgment export."); - } + this.projectId.ifPresentOrElse(id -> { + }, () -> Logger.error("Failed to resolve project " + configuration.projectName() + + ", please create it first at https://app.judgmentlabs.ai/org/" + + configuration.organizationId() + "/projects. Skipping Judgment export.")); if (initialize) { initialize(); } } + /** + * Initializes the tracer with OpenTelemetry SDK configuration and span exporters. Must be + * implemented by subclasses. + */ public abstract void initialize(); /** - * Gets the SpanExporter for OpenTelemetry integration. Returns NoOpSpanExporter if project ID - * cannot be resolved. + * Gets the span exporter for sending traces to the Judgment Labs backend. Returns a + * NoOpSpanExporter if the project ID is not resolved. + * + * @return the configured SpanExporter instance */ public SpanExporter getSpanExporter() { - if (projectId == null) { + return projectId.map(this::createJudgmentSpanExporter).orElseGet(() -> { Logger.error( "Project not resolved; cannot create exporter, returning NoOpSpanExporter"); return new NoOpSpanExporter(); - } - return createJudgmentSpanExporter(projectId); + }); } + /** + * Sets the kind of the current span (e.g., "llm", "tool", "span"). + * + * @param kind the span kind to set, ignored if null + */ public void setSpanKind(String kind) { - if (kind == null) { - return; - } - withCurrentSpan( - span -> - span.setAttribute( - OpenTelemetryKeys.AttributeKeys.JUDGMENT_SPAN_KIND, kind)); + Optional.ofNullable(kind).ifPresent(k -> withCurrentSpan( + span -> span.setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_SPAN_KIND, k))); } private static void withCurrentSpan(java.util.function.Consumer action) { - Span span = Span.current(); - if (span != null) { - action.accept(span); - } + Optional.ofNullable(Span.current()).ifPresent(action); } + /** + * Sets an attribute on the current span by serializing the object value. + * + * @param key the attribute key + * @param value the attribute value, ignored if null + */ public void setAttribute(String key, Object value) { - if (value == null) { - return; - } - setAttribute(key, value, value.getClass()); + Optional.ofNullable(value).ifPresent(v -> setAttribute(key, v, v.getClass())); } + /** + * Sets an attribute on the current span by serializing the object value with the specified + * type. + * + * @param key the attribute key + * @param value the attribute value, ignored if null + * @param type the type to use for serialization + */ public void setAttribute(String key, Object value, Type type) { - if (value == null) { - return; - } - withCurrentSpan(span -> span.setAttribute(key, serializer.serialize(value, type))); + Optional.ofNullable(value).ifPresent(v -> withCurrentSpan( + span -> span.setAttribute(key, serializer.serialize(v, type)))); } + /** + * Sets a string attribute on the current span. + * + * @param key the attribute key + * @param value the string value, ignored if null + */ public void setAttribute(String key, String value) { - if (value == null) { - return; - } - withCurrentSpan(span -> span.setAttribute(key, value)); + Optional.ofNullable(value) + .ifPresent(v -> withCurrentSpan(span -> span.setAttribute(key, v))); } + /** + * Sets a long attribute on the current span. + * + * @param key the attribute key + * @param value the long value + */ public void setAttribute(String key, long value) { withCurrentSpan(span -> span.setAttribute(key, value)); } + /** + * Sets a double attribute on the current span. + * + * @param key the attribute key + * @param value the double value + */ public void setAttribute(String key, double value) { withCurrentSpan(span -> span.setAttribute(key, value)); } + /** + * Sets a boolean attribute on the current span. + * + * @param key the attribute key + * @param value the boolean value + */ public void setAttribute(String key, boolean value) { withCurrentSpan(span -> span.setAttribute(key, value)); } - /** Asynchronously evaluates a scorer with an example using current trace context. */ - public void asyncEvaluate(BaseScorer scorer, Example example, String model) { + private Optional getSampledSpanContext() { + return Optional.ofNullable(Span.current()).filter(span -> span.getSpanContext().isSampled()) + .map(Span::getSpanContext); + } + + private Optional getSampledSpan() { + return Optional.ofNullable(Span.current()) + .filter(span -> span.getSpanContext().isSampled()); + } + + private boolean isEvaluationEnabled() { + return configuration.enableEvaluation(); + } + + private void logEvaluationInfo(String method, @NotNull String traceId, @NotNull String spanId, + @NotNull String scorerName) { + Logger.info(method + ": project=" + configuration.projectName() + ", traceId=" + traceId + + ", spanId=" + spanId + ", scorer=" + scorerName); + } + + private void safeExecute(String operation, Runnable action) { try { - if (!configuration.enableEvaluation()) { - return; - } + action.run(); + } catch (Exception e) { + Logger.error("Failed to " + operation + ": " + e.getMessage()); + } + } - Span currentSpan = Span.current(); - if (currentSpan == null || !currentSpan.getSpanContext().isSampled()) { + /** + * Asynchronously evaluates a scorer against an example, associating it with the current trace. + * + * @param scorer the scorer to evaluate + * @param example the example to evaluate against + * @param model the model name, or null to use the default + */ + public void asyncEvaluate(@NotNull BaseScorer scorer, @NotNull Example example, String model) { + safeExecute("evaluate scorer", () -> { + if (!isEvaluationEnabled()) { return; } - SpanContext spanContext = currentSpan.getSpanContext(); - String traceId = spanContext.getTraceId(); - String spanId = spanContext.getSpanId(); - - Logger.info( - "asyncEvaluate: project=" - + configuration.projectName() - + ", traceId=" - + traceId - + ", spanId=" - + spanId - + ", scorer=" - + scorer.getName()); - - ExampleEvaluationRun evaluationRun = - createEvaluationRun(scorer, example, model, traceId, spanId); - enqueueEvaluation(evaluationRun); - } catch (Exception e) { - Logger.error("Failed to evaluate scorer: " + e.getMessage()); - } + getSampledSpanContext().ifPresent(spanContext -> { + String traceId = spanContext.getTraceId(); + String spanId = spanContext.getSpanId(); + + logEvaluationInfo("asyncEvaluate", traceId, spanId, scorer.getName()); + + ExampleEvaluationRun evaluationRun = + createEvaluationRun(scorer, example, model, traceId, spanId); + enqueueEvaluation(evaluationRun); + }); + }); } - public void asyncEvaluate(BaseScorer scorer, Example example) { + /** + * Asynchronously evaluates a scorer against an example using the default model. + * + * @param scorer the scorer to evaluate + * @param example the example to evaluate against + */ + public void asyncEvaluate(@NotNull BaseScorer scorer, @NotNull Example example) { asyncEvaluate(scorer, example, null); } - /** Asynchronously evaluates a scorer with trace context. Sets evaluation as span attribute. */ - public void asyncTraceEvaluate(BaseScorer scorer, String model) { - try { - if (!configuration.enableEvaluation()) { + /** + * Asynchronously evaluates a scorer for the current trace, attaching the evaluation as a span + * attribute. + * + * @param scorer the scorer to evaluate + * @param model the model name, or null to use the default + */ + public void asyncTraceEvaluate(@NotNull BaseScorer scorer, String model) { + safeExecute("evaluate trace scorer", () -> { + if (!isEvaluationEnabled()) { return; } - Span currentSpan = Span.current(); - if (currentSpan == null || !currentSpan.getSpanContext().isSampled()) { - return; - } + getSampledSpan().ifPresent(currentSpan -> { + SpanContext spanContext = currentSpan.getSpanContext(); + String traceId = spanContext.getTraceId(); + String spanId = spanContext.getSpanId(); - SpanContext spanContext = currentSpan.getSpanContext(); - String traceId = spanContext.getTraceId(); - String spanId = spanContext.getSpanId(); - - Logger.info( - "asyncTraceEvaluate: project=" - + configuration.projectName() - + ", traceId=" - + traceId - + ", spanId=" - + spanId - + ", scorer=" - + scorer.getName()); - - TraceEvaluationRun evaluationRun = - createTraceEvaluationRun(scorer, model, traceId, spanId); - try { - String traceEvalJson = jacksonMapper.writeValueAsString(evaluationRun); - currentSpan.setAttribute( - OpenTelemetryKeys.AttributeKeys.PENDING_TRACE_EVAL, traceEvalJson); - } catch (Exception e) { - Logger.error("Failed to serialize trace evaluation: " + e.getMessage()); - } - } catch (Exception e) { - Logger.error("Failed to evaluate trace scorer: " + e.getMessage()); - } + logEvaluationInfo("asyncTraceEvaluate", traceId, spanId, scorer.getName()); + + TraceEvaluationRun evaluationRun = + createTraceEvaluationRun(scorer, model, traceId, spanId); + try { + String traceEvalJson = jacksonMapper.writeValueAsString(evaluationRun); + currentSpan.setAttribute(OpenTelemetryKeys.AttributeKeys.PENDING_TRACE_EVAL, + traceEvalJson); + } catch (Exception e) { + Logger.error("Failed to serialize trace evaluation: " + e.getMessage()); + } + }); + }); } - public void asyncTraceEvaluate(BaseScorer scorer) { + /** + * Asynchronously evaluates a scorer for the current trace using the default model. + * + * @param scorer the scorer to evaluate + */ + public void asyncTraceEvaluate(@NotNull BaseScorer scorer) { asyncTraceEvaluate(scorer, null); } + /** + * Sets multiple attributes on the current span from a map. + * + * @param attributes the map of attribute key-value pairs, ignored if null + */ public void setAttributes(Map attributes) { - if (attributes == null) { - return; - } - withCurrentSpan(span -> attributes.forEach(this::setAttribute)); + Optional.ofNullable(attributes) + .ifPresent(attrs -> withCurrentSpan(span -> attrs.forEach(this::setAttribute))); } + /** + * Sets the current span kind to "llm". + */ public void setLLMSpan() { setSpanKind("llm"); } + /** + * Sets the current span kind to "tool". + */ public void setToolSpan() { setSpanKind("tool"); } + /** + * Sets the current span kind to "span". + */ public void setGeneralSpan() { setSpanKind("span"); } + /** + * Sets the input attribute on the current span by serializing the object. + * + * @param input the input object to set, ignored if null + */ public void setInput(Object input) { setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_INPUT, input); } + /** + * Sets the output attribute on the current span by serializing the object. + * + * @param output the output object to set, ignored if null + */ public void setOutput(Object output) { setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_OUTPUT, output); } + /** + * Sets the input attribute on the current span with a specific type for serialization. + * + * @param input the input object to set, ignored if null + * @param type the type to use for serialization + */ public void setInput(Object input, Type type) { setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_INPUT, input, type); } + /** + * Sets the output attribute on the current span with a specific type for serialization. + * + * @param output the output object to set, ignored if null + * @param type the type to use for serialization + */ public void setOutput(Object output, Type type) { setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_OUTPUT, output, type); } /** - * Creates a new span with the given name and executes the provided runnable within its scope. - * The span is automatically ended when the runnable completes. + * Creates a new span, executes the runnable within its context, and ends the span. + * + * @param spanName the name of the span + * @param runnable the code to execute within the span context */ - public void span(String spanName, Runnable runnable) { + public void span(@NotNull String spanName, @NotNull Runnable runnable) { Span span = getTracer().spanBuilder(spanName).startSpan(); try (Scope scope = span.makeCurrent()) { runnable.run(); @@ -256,11 +345,16 @@ public void span(String spanName, Runnable runnable) { } /** - * Creates a new span with the given name and executes the provided callable within its scope. - * The span is automatically ended when the callable completes. Returns the result of the - * callable. + * Creates a new span, executes the callable within its context, and ends the span. + * + * @param the return type of the callable + * @param spanName the name of the span + * @param callable the code to execute within the span context + * @return the result of the callable + * @throws Exception if the callable throws an exception */ - public T span(String spanName, java.util.concurrent.Callable callable) throws Exception { + public T span(@NotNull String spanName, @NotNull java.util.concurrent.Callable callable) + throws Exception { Span span = getTracer().spanBuilder(spanName).startSpan(); try (Scope scope = span.makeCurrent()) { return callable.call(); @@ -269,79 +363,81 @@ public T span(String spanName, java.util.concurrent.Callable callable) th } } - /** Gets the OpenTelemetry tracer instance. */ + /** + * Gets the OpenTelemetry tracer instance. + * + * @return the configured Tracer instance + */ public Tracer getTracer() { return GlobalOpenTelemetry.get().getTracer(TRACER_NAME); } /** - * Creates a new span with the given name. The caller is responsible for ending the span - * manually. + * Creates and returns a new span with the given name. The span must be ended manually by + * calling {@link Span#end()}. + * + * @param spanName the name of the span + * @return the newly created span */ - public static Span span(String spanName) { + public static Span span(@NotNull String spanName) { return GlobalOpenTelemetry.get().getTracer(TRACER_NAME).spanBuilder(spanName).startSpan(); } - protected String resolveProjectId(String name) { + protected Optional resolveProjectId(@NotNull String name) { try { ResolveProjectNameRequest request = new ResolveProjectNameRequest(); request.setProjectName(name); ResolveProjectNameResponse response = apiClient.projectsResolve(request); - Object projectId = response.getProjectId(); - return projectId != null ? projectId.toString() : null; + return Optional.ofNullable(response.getProjectId()).map(Object::toString); } catch (Exception e) { - return null; + return Optional.empty(); } } - private JudgmentSpanExporter createJudgmentSpanExporter(String projectId) { - String endpoint = - configuration.apiUrl().endsWith("/") - ? configuration.apiUrl() + "otel/v1/traces" - : configuration.apiUrl() + "/otel/v1/traces"; - return JudgmentSpanExporter.builder() - .endpoint(endpoint) - .apiKey(configuration.apiKey()) - .organizationId(configuration.organizationId()) - .projectId(projectId) - .build(); + private String buildEndpoint(@NotNull String baseUrl) { + return baseUrl.endsWith("/") ? baseUrl + "otel/v1/traces" : baseUrl + "/otel/v1/traces"; + } + + private JudgmentSpanExporter createJudgmentSpanExporter(@NotNull String projectId) { + return JudgmentSpanExporter.builder().endpoint(buildEndpoint(configuration.apiUrl())) + .apiKey(configuration.apiKey()).organizationId(configuration.organizationId()) + .projectId(projectId).build(); + } + + private String generateRunId(String prefix, String spanId) { + return prefix + Optional.ofNullable(spanId) + .orElseGet(() -> String.valueOf(System.currentTimeMillis())); + } + + private String getModelName(String model) { + return Optional.ofNullable(model).orElse(Env.JUDGMENT_DEFAULT_GPT_MODEL); } - private ExampleEvaluationRun createEvaluationRun( - BaseScorer scorer, Example example, String model, String traceId, String spanId) { - String runId = "async_evaluate_" + (spanId != null ? spanId : System.currentTimeMillis()); - String modelName = model != null ? model : Env.JUDGMENT_DEFAULT_GPT_MODEL; + private ExampleEvaluationRun createEvaluationRun(@NotNull BaseScorer scorer, + @NotNull Example example, String model, @NotNull String traceId, String spanId) { + String runId = generateRunId("async_evaluate_", spanId); + String modelName = getModelName(model); - ExampleEvaluationRun evaluationRun = - new ExampleEvaluationRun( - configuration.projectName(), - runId, - List.of(example), - List.of(scorer.getScorerConfig()), - modelName, - configuration.organizationId()); + ExampleEvaluationRun evaluationRun = new ExampleEvaluationRun(configuration.projectName(), + runId, List.of(example), List.of(scorer.getScorerConfig()), modelName, + configuration.organizationId()); evaluationRun.setTraceId(traceId); evaluationRun.setTraceSpanId(spanId); return evaluationRun; } - private TraceEvaluationRun createTraceEvaluationRun( - BaseScorer scorer, String model, String traceId, String spanId) { - String evalName = - "async_trace_evaluate_" + (spanId != null ? spanId : System.currentTimeMillis()); - String modelName = model != null ? model : Env.JUDGMENT_DEFAULT_GPT_MODEL; + private TraceEvaluationRun createTraceEvaluationRun(@NotNull BaseScorer scorer, String model, + @NotNull String traceId, String spanId) { + String evalName = generateRunId("async_trace_evaluate_", spanId); + String modelName = getModelName(model); - return TraceEvaluationRun.builder() - .projectName(configuration.projectName()) - .evalName(evalName) - .scorer(scorer.getScorerConfig()) - .model(modelName) - .organizationId(configuration.organizationId()) - .traceAndSpanId(traceId, spanId) + return TraceEvaluationRun.builder().projectName(configuration.projectName()) + .evalName(evalName).scorer(scorer.getScorerConfig()).model(modelName) + .organizationId(configuration.organizationId()).traceAndSpanId(traceId, spanId) .build(); } - private void enqueueEvaluation(ExampleEvaluationRun evaluationRun) { + private void enqueueEvaluation(@NotNull ExampleEvaluationRun evaluationRun) { try { apiClient.addToRunEvalQueue(evaluationRun); } catch (Exception e) { @@ -354,7 +450,7 @@ protected static class GsonSerializer implements ISerializer { @Override public String serialize(Object obj) { - return serialize(obj, obj.getClass()); + return Optional.ofNullable(obj).map(o -> serialize(o, o.getClass())).orElse(null); } @Override @@ -363,7 +459,7 @@ public String serialize(Object obj, Type type) { return gson.toJson(obj, type); } catch (Exception e) { Logger.error("Failed to serialize object: " + e.getMessage()); - return obj != null ? obj.toString() : null; + return Optional.ofNullable(obj).map(Object::toString).orElse(null); } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java index d5d394a..06a8887 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java @@ -1,5 +1,9 @@ package com.judgmentlabs.judgeval.tracer; +import java.util.Optional; + +import org.jetbrains.annotations.NotNull; + import com.judgmentlabs.judgeval.Version; import io.opentelemetry.api.GlobalOpenTelemetry; @@ -21,43 +25,55 @@ */ public final class Tracer extends BaseTracer { - private Tracer( - TracerConfiguration configuration, ISerializer serializer, boolean shouldInitialize) { + private Tracer(TracerConfiguration configuration, ISerializer serializer, + boolean shouldInitialize) { super(configuration, serializer, shouldInitialize); } + /** + * Creates a new TracerBuilder for constructing a Tracer instance. + * + * @return a new TracerBuilder + */ public static TracerBuilder builder() { return new TracerBuilder(); } + /** + * Creates a Tracer with default configuration for the given project name. + * + * @param projectName the name of the project + * @return a new Tracer instance with default configuration + */ public static Tracer createDefault(String projectName) { return builder().configuration(TracerConfiguration.createDefault(projectName)).build(); } + /** + * Creates a Tracer with the provided configuration. + * + * @param configuration the tracer configuration + * @return a new Tracer instance with the given configuration + */ public static Tracer createWithConfiguration(TracerConfiguration configuration) { return builder().configuration(configuration).build(); } - /** Initializes the OpenTelemetry SDK with batch span processor and registers it globally. */ + /** + * Initializes the OpenTelemetry SDK with batch span processor and registers it globally. + */ @Override public void initialize() { SpanExporter spanExporter = getSpanExporter(); - var resource = - Resource.getDefault() - .merge( - Resource.create( - Attributes.builder() - .put("service.name", configuration.projectName()) - .put("telemetry.sdk.name", TRACER_NAME) - .put("telemetry.sdk.version", Version.getVersion()) - .build())); - - SdkTracerProvider tracerProvider = - SdkTracerProvider.builder() - .setResource(resource) - .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build()) - .build(); + var resource = Resource.getDefault() + .merge(Resource.create( + Attributes.builder().put("service.name", configuration.projectName()) + .put("telemetry.sdk.name", TRACER_NAME) + .put("telemetry.sdk.version", Version.getVersion()).build())); + + SdkTracerProvider tracerProvider = SdkTracerProvider.builder().setResource(resource) + .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build()).build(); OpenTelemetry openTelemetry = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build(); @@ -65,33 +81,54 @@ public void initialize() { GlobalOpenTelemetry.set(openTelemetry); } - /** Builder for creating Tracer instances. */ public static final class TracerBuilder { private TracerConfiguration configuration; private ISerializer serializer = new GsonSerializer(); private boolean initialize = false; - public TracerBuilder configuration(TracerConfiguration configuration) { + /** + * Sets the tracer configuration. + * + * @param configuration the configuration to use + * @return this builder for method chaining + */ + public TracerBuilder configuration(@NotNull TracerConfiguration configuration) { this.configuration = configuration; return this; } - public TracerBuilder serializer(ISerializer serializer) { + /** + * Sets the serializer to use for attribute serialization. + * + * @param serializer the serializer to use + * @return this builder for method chaining + */ + public TracerBuilder serializer(@NotNull ISerializer serializer) { this.serializer = serializer; return this; } + /** + * Sets whether to automatically initialize the tracer after construction. + * + * @param initialize true to initialize automatically, false otherwise + * @return this builder for method chaining + */ public TracerBuilder initialize(boolean initialize) { this.initialize = initialize; return this; } + /** + * Builds a new Tracer instance with the configured settings. + * + * @return a new Tracer instance + * @throws IllegalArgumentException if configuration is not set + */ public Tracer build() { - if (configuration == null) { - throw new IllegalArgumentException("Configuration is required"); - } - - return new Tracer(configuration, serializer, initialize); + return Optional.ofNullable(configuration) + .map(config -> new Tracer(config, serializer, initialize)) + .orElseThrow(() -> new IllegalArgumentException("Configuration is required")); } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java b/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java index 14b49d3..43cb7c4 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java @@ -1,21 +1,19 @@ package com.judgmentlabs.judgeval.tracer; -import java.util.Objects; +import java.util.Optional; + +import org.jetbrains.annotations.NotNull; /** * Configuration for the Judgment Tracer that controls how tracing and evaluation behave. - * - *

This class encapsulates all configuration parameters needed to initialize a {@link Tracer}. - * - *

Example usage: + *

+ * This class encapsulates all configuration parameters needed to initialize a {@link Tracer}. + *

+ * Example usage: * * 

{@code
- * TracerConfiguration config = TracerConfiguration.builder()
- *         .projectName("my-project")
- *         .apiKey("your-api-key")
- *         .organizationId("your-org-id")
- *         .enableEvaluation(true)
- *         .build();
+ * TracerConfiguration config = TracerConfiguration.builder().projectName("my-project")
+ *         .apiKey("your-api-key").organizationId("your-org-id").enableEvaluation(true).build();
  *
  * Tracer tracer = Tracer.createWithConfiguration(config);
  * }
@@ -29,18 +27,18 @@ public final class TracerConfiguration { private final String apiUrl; private final boolean enableEvaluation; - private TracerConfiguration(Builder builder) { - this.projectName = - Objects.requireNonNull(builder.projectName, "Project name cannot be null").trim(); - this.apiKey = Objects.requireNonNull(builder.apiKey, "API key cannot be null"); - this.organizationId = - Objects.requireNonNull(builder.organizationId, "Organization ID cannot be null"); - this.apiUrl = Objects.requireNonNull(builder.apiUrl, "API URL cannot be null"); + private TracerConfiguration(@NotNull Builder builder) { + this.projectName = Optional.ofNullable(builder.projectName).map(String::trim) + .filter(name -> !name.isEmpty()) + .orElseThrow( + () -> new IllegalArgumentException("Project name cannot be null or empty")); + this.apiKey = Optional.ofNullable(builder.apiKey) + .orElseThrow(() -> new IllegalArgumentException("API key cannot be null")); + this.organizationId = Optional.ofNullable(builder.organizationId) + .orElseThrow(() -> new IllegalArgumentException("Organization ID cannot be null")); + this.apiUrl = Optional.ofNullable(builder.apiUrl) + .orElseThrow(() -> new IllegalArgumentException("API URL cannot be null")); this.enableEvaluation = builder.enableEvaluation; - - if (this.projectName.isEmpty()) { - throw new IllegalArgumentException("Project name cannot be empty"); - } } public String projectName() { @@ -69,14 +67,13 @@ public static Builder builder() { /** * Creates a default configuration with the given project name. - * - *

This method uses default values from environment variables: - * + *

+ * This method uses default values from environment variables: *

    - *
  • API Key: {@code Env.JUDGMENT_API_KEY} - *
  • Organization ID: {@code Env.JUDGMENT_ORG_ID} - *
  • API URL: {@code Env.JUDGMENT_API_URL} - *
  • Evaluation: enabled + *
  • API Key: {@code Env.JUDGMENT_API_KEY} + *
  • Organization ID: {@code Env.JUDGMENT_ORG_ID} + *
  • API URL: {@code Env.JUDGMENT_API_URL} + *
  • Evaluation: enabled *
* * @param projectName the name of the project @@ -89,17 +86,13 @@ public static TracerConfiguration createDefault(String projectName) { /** * Builder for creating TracerConfiguration instances. - * - *

Example usage: + *

+ * Example usage: * * 

{@code
-     * TracerConfiguration config = TracerConfiguration.builder()
-     *         .projectName("my-project")
-     *         .apiKey("custom-api-key")
-     *         .organizationId("custom-org-id")
-     *         .apiUrl("https://custom-api.judgmentlabs.ai")
-     *         .enableEvaluation(false)
-     *         .build();
+     * TracerConfiguration config = TracerConfiguration.builder().projectName("my-project")
+     *         .apiKey("custom-api-key").organizationId("custom-org-id")
+     *         .apiUrl("https://custom-api.judgmentlabs.ai").enableEvaluation(false).build();
      * }
*/ public static final class Builder { @@ -109,22 +102,22 @@ public static final class Builder { private String apiUrl = com.judgmentlabs.judgeval.Env.JUDGMENT_API_URL; private boolean enableEvaluation = true; - public Builder projectName(String projectName) { + public Builder projectName(@NotNull String projectName) { this.projectName = projectName; return this; } - public Builder apiKey(String apiKey) { + public Builder apiKey(@NotNull String apiKey) { this.apiKey = apiKey; return this; } - public Builder organizationId(String organizationId) { + public Builder organizationId(@NotNull String organizationId) { this.organizationId = organizationId; return this; } - public Builder apiUrl(String apiUrl) { + public Builder apiUrl(@NotNull String apiUrl) { this.apiUrl = apiUrl; return this; } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java index 8de675a..5abfbe6 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java @@ -1,6 +1,9 @@ package com.judgmentlabs.judgeval.tracer.exporters; import java.util.Collection; +import java.util.Optional; + +import org.jetbrains.annotations.NotNull; import com.judgmentlabs.judgeval.utils.Logger; @@ -11,8 +14,8 @@ /** * SpanExporter implementation that sends spans to Judgment Labs with project identification. - * - *

This exporter wraps the OTLP HTTP exporter and adds Judgment Labs specific headers and project + *

+ * This exporter wraps the OTLP HTTP exporter and adds Judgment Labs specific headers and project * identification to all exported spans. */ public class JudgmentSpanExporter implements SpanExporter { @@ -27,18 +30,15 @@ public class JudgmentSpanExporter implements SpanExporter { * @param projectId the project ID (must not be null or empty) * @throws IllegalArgumentException if projectId is null or empty */ - public JudgmentSpanExporter( - String endpoint, String apiKey, String organizationId, String projectId) { - if (projectId == null || projectId.isEmpty()) { + protected JudgmentSpanExporter(@NotNull String endpoint, @NotNull String apiKey, + @NotNull String organizationId, @NotNull String projectId) { + if (projectId.isEmpty()) { throw new IllegalArgumentException("projectId is required for JudgmentSpanExporter"); } - this.delegate = - OtlpHttpSpanExporter.builder() - .setEndpoint(endpoint) - .addHeader("Authorization", "Bearer " + apiKey) - .addHeader("X-Organization-Id", organizationId) - .addHeader("X-Project-Id", projectId) - .build(); + this.delegate = OtlpHttpSpanExporter.builder().setEndpoint(endpoint) + .addHeader("Authorization", "Bearer " + apiKey) + .addHeader("X-Organization-Id", organizationId).addHeader("X-Project-Id", projectId) + .build(); } /** @@ -50,23 +50,41 @@ public static Builder builder() { return new Builder(); } + /** + * Exports the collection of spans to the Judgment Labs backend. + * + * @param spans the collection of spans to export + * @return a CompletableResultCode representing the export operation status + */ @Override public CompletableResultCode export(Collection spans) { Logger.info("Exported " + spans.size() + " spans"); return delegate.export(spans); } + /** + * Flushes any pending span exports. + * + * @return a CompletableResultCode representing the flush operation status + */ @Override public CompletableResultCode flush() { return delegate.flush(); } + /** + * Shuts down this exporter and releases any resources. + * + * @return a CompletableResultCode representing the shutdown operation status + */ @Override public CompletableResultCode shutdown() { return delegate.shutdown(); } - /** Builder for creating JudgmentSpanExporter instances. */ + /** + * Builder for creating JudgmentSpanExporter instances. + */ public static final class Builder { private String endpoint; private String apiKey; @@ -81,7 +99,7 @@ private Builder() {} * @param endpoint the endpoint URL * @return this builder for method chaining */ - public Builder endpoint(String endpoint) { + public Builder endpoint(@NotNull String endpoint) { this.endpoint = endpoint; return this; } @@ -92,7 +110,7 @@ public Builder endpoint(String endpoint) { * @param apiKey the API key * @return this builder for method chaining */ - public Builder apiKey(String apiKey) { + public Builder apiKey(@NotNull String apiKey) { this.apiKey = apiKey; return this; } @@ -103,7 +121,7 @@ public Builder apiKey(String apiKey) { * @param organizationId the organization ID * @return this builder for method chaining */ - public Builder organizationId(String organizationId) { + public Builder organizationId(@NotNull String organizationId) { this.organizationId = organizationId; return this; } @@ -114,32 +132,37 @@ public Builder organizationId(String organizationId) { * @param projectId the project ID * @return this builder for method chaining */ - public Builder projectId(String projectId) { + public Builder projectId(@NotNull String projectId) { this.projectId = projectId; return this; } /** - * Builds a new JudgmentSpanExporter with the current configuration. + * Builds a new JudgmentSpanExporter instance with the configured settings. * * @return a new JudgmentSpanExporter instance - * @throws IllegalArgumentException if required fields are missing + * @throws IllegalArgumentException if any required field is null or empty */ public JudgmentSpanExporter build() { - if (endpoint == null || endpoint.trim().isEmpty()) { - throw new IllegalArgumentException("Endpoint is required"); - } - if (apiKey == null || apiKey.trim().isEmpty()) { - throw new IllegalArgumentException("API key is required"); - } - if (organizationId == null || organizationId.trim().isEmpty()) { - throw new IllegalArgumentException("Organization ID is required"); - } - if (projectId == null || projectId.trim().isEmpty()) { - throw new IllegalArgumentException("Project ID is required"); - } - - return new JudgmentSpanExporter(endpoint, apiKey, organizationId, projectId); + String validEndpoint = + Optional.ofNullable(endpoint).map(String::trim).filter(e -> !e.isEmpty()) + .orElseThrow( + () -> new IllegalArgumentException("Endpoint is required")); + String validApiKey = + Optional.ofNullable(apiKey).map(String::trim).filter(key -> !key.isEmpty()) + .orElseThrow(() -> new IllegalArgumentException("API key is required")); + String validOrganizationId = + Optional.ofNullable(organizationId).map(String::trim) + .filter(id -> !id.isEmpty()) + .orElseThrow(() -> new IllegalArgumentException( + "Organization ID is required")); + String validProjectId = + Optional.ofNullable(projectId).map(String::trim).filter(id -> !id.isEmpty()) + .orElseThrow( + () -> new IllegalArgumentException("Project ID is required")); + + return new JudgmentSpanExporter(validEndpoint, validApiKey, validOrganizationId, + validProjectId); } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/NoOpSpanExporter.java b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/NoOpSpanExporter.java index 01509e8..ad5757f 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/NoOpSpanExporter.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/NoOpSpanExporter.java @@ -6,17 +6,37 @@ import io.opentelemetry.sdk.trace.data.SpanData; import io.opentelemetry.sdk.trace.export.SpanExporter; +/** + * A no-op implementation of SpanExporter that discards all spans. Used as a fallback when project + * resolution fails or when spans should not be exported. + */ public class NoOpSpanExporter implements SpanExporter { + /** + * Discards the collection of spans without exporting. + * + * @param spans the collection of spans (ignored) + * @return a successful CompletableResultCode + */ @Override public CompletableResultCode export(Collection spans) { return CompletableResultCode.ofSuccess(); } + /** + * Performs a no-op flush operation. + * + * @return a successful CompletableResultCode + */ @Override public CompletableResultCode flush() { return CompletableResultCode.ofSuccess(); } + /** + * Performs a no-op shutdown operation. + * + * @return a successful CompletableResultCode + */ @Override public CompletableResultCode shutdown() { return CompletableResultCode.ofSuccess(); diff --git a/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java b/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java index b711dd1..22a4a5b 100644 --- a/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java +++ b/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java @@ -17,11 +17,7 @@ public class Logger { DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); public enum Level { - DEBUG(0, GRAY), - INFO(1, GRAY), - WARNING(2, YELLOW), - ERROR(3, RED), - CRITICAL(4, RED); + DEBUG(0, GRAY), INFO(1, GRAY), WARNING(2, YELLOW), ERROR(3, RED), CRITICAL(4, RED); private final int value; private final String color; From 302f608699ad997b5c0d92b8d0c83fd5cd98197f Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 29 Oct 2025 14:56:30 -0700 Subject: [PATCH 03/11] significantly simplify prompt scorers --- .../judgeval/scorers/APIScorer.java | 10 +- .../prompt_scorer/BasePromptScorer.java | 113 ++---------------- .../prompt_scorer/PromptScorer.java | 97 +-------------- .../prompt_scorer/TracePromptScorer.java | 99 +-------------- 4 files changed, 21 insertions(+), 298 deletions(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java index 952d614..2b71ea0 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java @@ -3,6 +3,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import com.fasterxml.jackson.annotation.JsonIgnore; import com.judgmentlabs.judgeval.data.APIScorerType; @@ -48,20 +49,17 @@ public void setRequiredParams(List requiredParams) { @Override public Double getThreshold() { - Double threshold = super.getThreshold(); - return threshold != null ? threshold : 0.5; + return Optional.ofNullable(super.getThreshold()).orElse(0.5); } @Override public String getName() { - Object name = super.getName(); - return name != null ? name.toString() : null; + return Optional.ofNullable(super.getName()).map(Object::toString).orElse(null); } @Override public Boolean getStrictMode() { - Boolean strictMode = super.getStrictMode(); - return strictMode != null ? strictMode : false; + return Optional.ofNullable(super.getStrictMode()).orElse(false); } @Override diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java index a0a428f..2e90502 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java @@ -1,8 +1,8 @@ package com.judgmentlabs.judgeval.scorers.api_scorers.prompt_scorer; -import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import com.judgmentlabs.judgeval.Env; import com.judgmentlabs.judgeval.data.APIScorerType; @@ -10,12 +10,9 @@ import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersRequest; import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersResponse; -import com.judgmentlabs.judgeval.internal.api.models.SavePromptScorerRequest; -import com.judgmentlabs.judgeval.internal.api.models.SavePromptScorerResponse; import com.judgmentlabs.judgeval.internal.api.models.ScorerExistsRequest; import com.judgmentlabs.judgeval.internal.api.models.ScorerExistsResponse; import com.judgmentlabs.judgeval.scorers.APIScorer; -import com.judgmentlabs.judgeval.utils.Logger; public abstract class BasePromptScorer extends APIScorer { protected String prompt; @@ -32,7 +29,7 @@ protected BasePromptScorer(APIScorerType scoreType, String name, String prompt, this.judgmentApiKey = judgmentApiKey; this.organizationId = organizationId; setName(name); - setThreshold(threshold); + super.setThreshold(threshold); } public static boolean scorerExists(String name, String judgmentApiKey, String organizationId) { @@ -43,18 +40,8 @@ public static boolean scorerExists(String name, String judgmentApiKey, String or request.setName(name); ScorerExistsResponse response = client.scorerExists(request); return Boolean.TRUE.equals(response.getExists()); - } catch (JudgmentAPIError e) { - if (e.getStatusCode() == 500) { - throw new JudgmentAPIError(e.getStatusCode(), - "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " - + e.getMessage()); - } - throw new JudgmentAPIError(e.getStatusCode(), - "Failed to check if scorer exists: " + e.getMessage()); - } catch (IOException | InterruptedException e) { - throw new JudgmentAPIError(500, - "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " - + e.getMessage()); + } catch (Exception e) { + throw new JudgmentAPIError(500, "Failed to check if scorer exists: " + e.getMessage()); } } @@ -68,87 +55,18 @@ public static com.judgmentlabs.judgeval.internal.api.models.PromptScorer fetchPr FetchPromptScorersResponse response = client.fetchScorers(request); - if (response.getScorers() == null || response.getScorers().isEmpty()) { - throw new JudgmentAPIError(404, - "Failed to fetch prompt scorer '" + name + "': not found"); - } - - return response.getScorers().get(0); - } catch (JudgmentAPIError e) { - if (e.getStatusCode() == 500) { - throw new JudgmentAPIError(e.getStatusCode(), - "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " - + e.getMessage()); - } - throw new JudgmentAPIError(e.getStatusCode(), - "Failed to fetch prompt scorer '" + name + "': " + e.getMessage()); - } catch (IOException | InterruptedException e) { - throw new JudgmentAPIError(500, - "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " - + e.getMessage()); - } - } - - public static String pushPromptScorer(String name, String prompt, double threshold, - Map options, String judgmentApiKey, String organizationId, - Boolean isTrace) { - try { - JudgmentSyncClient client = - new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); - SavePromptScorerRequest request = new SavePromptScorerRequest(); - request.setName(name); - request.setPrompt(prompt); - request.setThreshold(threshold); - Map apiOptions = null; - if (options != null) { - apiOptions = new HashMap<>(); - for (Map.Entry entry : options.entrySet()) { - apiOptions.put(entry.getKey(), entry.getValue()); - } - } - request.setOptions(apiOptions); - request.setIsTrace(isTrace); - - SavePromptScorerResponse response = client.saveScorer(request); - return response != null ? response.getScorerResponse().getName() : null; - } catch (JudgmentAPIError e) { - if (e.getStatusCode() == 500) { - throw new JudgmentAPIError(e.getStatusCode(), - "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " - + e.getMessage()); - } - throw new JudgmentAPIError(e.getStatusCode(), - "Failed to save prompt scorer: " + e.getMessage()); - } catch (IOException | InterruptedException e) { + return Optional.ofNullable(response) + .map(FetchPromptScorersResponse::getScorers) + .filter(scorers -> scorers != null && !scorers.isEmpty()) + .map(scorers -> scorers.get(0)) + .orElseThrow(() -> new JudgmentAPIError(404, + "Failed to fetch prompt scorer '" + name + "': not found")); + } catch (Exception e) { throw new JudgmentAPIError(500, - "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " - + e.getMessage()); + "Failed to fetch prompt scorer '" + name + "': " + e.getMessage()); } } - public void setThreshold(double threshold) { - super.setThreshold(threshold); - pushPromptScorer(); - } - - public void setPrompt(String prompt) { - this.prompt = prompt; - pushPromptScorer(); - Logger.info("Successfully updated prompt for " + getName()); - } - - public void setOptions(Map options) { - this.options = options; - pushPromptScorer(); - Logger.info("Successfully updated options for " + getName()); - } - - public void appendToPrompt(String promptAddition) { - this.prompt += promptAddition; - pushPromptScorer(); - Logger.info("Successfully appended to prompt for " + getName()); - } - public Double getThreshold() { return super.getThreshold(); } @@ -158,7 +76,7 @@ public String getPrompt() { } public Map getOptions() { - return options != null ? new HashMap<>(options) : null; + return Optional.ofNullable(options).map(HashMap::new).orElse(null); } public String getScorerName() { @@ -174,11 +92,6 @@ public Map getConfig() { return config; } - protected void pushPromptScorer() { - pushPromptScorer(getName(), prompt, getThreshold(), options, judgmentApiKey, organizationId, - isTrace()); - } - protected abstract boolean isTrace(); @Override diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java index ea4194b..c0922c3 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java @@ -2,6 +2,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.Optional; import com.judgmentlabs.judgeval.Env; import com.judgmentlabs.judgeval.data.APIScorerType; @@ -48,35 +49,10 @@ public static PromptScorer get(String name, String judgmentApiKey, String organi } return new PromptScorer(name, scorerConfig.getPrompt(), - scorerConfig.getThreshold() != null ? scorerConfig.getThreshold() : 0.5, options, + Optional.ofNullable(scorerConfig.getThreshold()).orElse(0.5), options, judgmentApiKey, organizationId); } - public static PromptScorer create(String name, String prompt) { - return create(name, prompt, 0.5, null); - } - - public static PromptScorer create(String name, String prompt, double threshold) { - return create(name, prompt, threshold, null); - } - - public static PromptScorer create(String name, String prompt, double threshold, - Map options) { - return create(name, prompt, threshold, options, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); - } - - public static PromptScorer create(String name, String prompt, double threshold, - Map options, String judgmentApiKey, String organizationId) { - if (!scorerExists(name, judgmentApiKey, organizationId)) { - pushPromptScorer(name, prompt, threshold, options, judgmentApiKey, organizationId, - false); - return new PromptScorer(name, prompt, threshold, options, judgmentApiKey, - organizationId); - } else { - throw new JudgmentAPIError(400, "Scorer with name " + name - + " already exists. Either use the existing scorer with the get() method or use a new name."); - } - } @Override public ScorerConfig getScorerConfig() { @@ -103,73 +79,4 @@ protected boolean isTrace() { return false; } - public static Builder builder() { - return new Builder(); - } - - public static Builder builder(String name, String prompt) { - return new Builder(name, prompt); - } - - public static final class Builder { - private String name; - private String prompt; - private double threshold = 0.5; - private Map options; - private String judgmentApiKey = Env.JUDGMENT_API_KEY; - private String organizationId = Env.JUDGMENT_ORG_ID; - - private Builder() {} - - private Builder(String name, String prompt) { - this.name = name; - this.prompt = prompt; - } - - public Builder name(String name) { - this.name = name; - return this; - } - - public Builder prompt(String prompt) { - this.prompt = prompt; - return this; - } - - public Builder threshold(double threshold) { - this.threshold = threshold; - return this; - } - - public Builder options(Map options) { - this.options = options; - return this; - } - - public Builder option(String key, Double value) { - if (this.options == null) { - this.options = new HashMap<>(); - } - this.options.put(key, value); - return this; - } - - public Builder judgmentApiKey(String judgmentApiKey) { - this.judgmentApiKey = judgmentApiKey; - return this; - } - - public Builder organizationId(String organizationId) { - this.organizationId = organizationId; - return this; - } - - public PromptScorer build() { - if (name == null || prompt == null) { - throw new IllegalArgumentException("Name and prompt are required"); - } - return new PromptScorer(name, prompt, threshold, options, judgmentApiKey, - organizationId); - } - } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java index 2275d3b..3ac7707 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java @@ -2,6 +2,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.Optional; import com.judgmentlabs.judgeval.Env; import com.judgmentlabs.judgeval.data.APIScorerType; @@ -51,36 +52,10 @@ public static TracePromptScorer get(String name, String judgmentApiKey, String o } return new TracePromptScorer(name, scorerConfig.getPrompt(), - scorerConfig.getThreshold() != null ? scorerConfig.getThreshold() : 0.5, options, + Optional.ofNullable(scorerConfig.getThreshold()).orElse(0.5), options, judgmentApiKey, organizationId); } - public static TracePromptScorer create(String name, String prompt) { - return create(name, prompt, 0.5, null); - } - - public static TracePromptScorer create(String name, String prompt, double threshold) { - return create(name, prompt, threshold, null); - } - - public static TracePromptScorer create(String name, String prompt, double threshold, - Map options) { - return create(name, prompt, threshold, options, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); - } - - public static TracePromptScorer create(String name, String prompt, double threshold, - Map options, String judgmentApiKey, String organizationId) { - if (!scorerExists(name, judgmentApiKey, organizationId)) { - pushPromptScorer(name, prompt, threshold, options, judgmentApiKey, organizationId, - true); - return new TracePromptScorer(name, prompt, threshold, options, judgmentApiKey, - organizationId); - } else { - throw new JudgmentAPIError(400, "Scorer with name " + name - + " already exists. Either use the existing scorer with the get() method or use a new name."); - } - } - @Override public ScorerConfig getScorerConfig() { ScorerConfig cfg = new ScorerConfig(); @@ -105,74 +80,4 @@ public ScorerConfig getScorerConfig() { protected boolean isTrace() { return true; } - - public static Builder builder() { - return new Builder(); - } - - public static Builder builder(String name, String prompt) { - return new Builder(name, prompt); - } - - public static final class Builder { - private String name; - private String prompt; - private double threshold = 0.5; - private Map options; - private String judgmentApiKey = Env.JUDGMENT_API_KEY; - private String organizationId = Env.JUDGMENT_ORG_ID; - - private Builder() {} - - private Builder(String name, String prompt) { - this.name = name; - this.prompt = prompt; - } - - public Builder name(String name) { - this.name = name; - return this; - } - - public Builder prompt(String prompt) { - this.prompt = prompt; - return this; - } - - public Builder threshold(double threshold) { - this.threshold = threshold; - return this; - } - - public Builder options(Map options) { - this.options = options; - return this; - } - - public Builder option(String key, Double value) { - if (this.options == null) { - this.options = new HashMap<>(); - } - this.options.put(key, value); - return this; - } - - public Builder judgmentApiKey(String judgmentApiKey) { - this.judgmentApiKey = judgmentApiKey; - return this; - } - - public Builder organizationId(String organizationId) { - this.organizationId = organizationId; - return this; - } - - public TracePromptScorer build() { - if (name == null || prompt == null) { - throw new IllegalArgumentException("Name and prompt are required"); - } - return new TracePromptScorer(name, prompt, threshold, options, judgmentApiKey, - organizationId); - } - } } From 0925f2b7f7372690d47a0710110ab887fe7468de Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 29 Oct 2025 16:03:53 -0700 Subject: [PATCH 04/11] better lint --- .vscode/java-formatter.xml | 590 +++++++++--------- .../java/com/judgmentlabs/judgeval/Env.java | 18 +- .../com/judgmentlabs/judgeval/Version.java | 7 +- .../judgeval/data/APIScorerType.java | 20 +- .../judgmentlabs/judgeval/data/Example.java | 6 +- .../judgeval/data/ExampleEvaluationRun.java | 35 +- .../judgeval/data/ScoringResult.java | 6 +- .../judgeval/data/TraceEvaluationRun.java | 43 +- .../judgeval/exceptions/JudgmentAPIError.java | 2 +- .../internal/api/JudgmentAsyncClient.java | 105 ++-- .../internal/api/JudgmentSyncClient.java | 102 +-- .../internal/api/models/BaseScorer.java | 57 +- .../internal/api/models/EvalResults.java | 2 +- .../internal/api/models/EvalResultsFetch.java | 4 +- .../judgeval/internal/api/models/Example.java | 12 +- .../api/models/ExampleEvaluationRun.java | 36 +- .../api/models/FetchPromptScorersRequest.java | 4 +- .../models/FetchPromptScorersResponse.java | 2 +- .../internal/api/models/OtelTraceSpan.java | 66 +- .../internal/api/models/PromptScorer.java | 43 +- .../api/models/ResolveProjectNameRequest.java | 2 +- .../models/ResolveProjectNameResponse.java | 2 +- .../api/models/SavePromptScorerRequest.java | 14 +- .../api/models/SavePromptScorerResponse.java | 2 +- .../internal/api/models/ScorerConfig.java | 17 +- .../internal/api/models/ScorerData.java | 31 +- .../api/models/ScorerExistsRequest.java | 5 +- .../api/models/ScorerExistsResponse.java | 2 +- .../internal/api/models/ScoringResult.java | 24 +- .../api/models/TraceEvaluationRun.java | 34 +- .../judgeval/scorers/APIScorer.java | 21 +- .../judgeval/scorers/BaseScorer.java | 4 +- .../api_scorers/AnswerCorrectnessScorer.java | 3 +- .../api_scorers/AnswerRelevancyScorer.java | 3 +- .../scorers/api_scorers/DerailmentScorer.java | 3 +- .../api_scorers/FaithfulnessScorer.java | 3 +- .../InstructionAdherenceScorer.java | 3 +- .../prompt_scorer/BasePromptScorer.java | 33 +- .../prompt_scorer/PromptScorer.java | 13 +- .../prompt_scorer/TracePromptScorer.java | 25 +- .../judgeval/tracer/BaseTracer.java | 276 ++++---- .../judgeval/tracer/JudgevalTraceKeys.java | 51 ++ .../judgeval/tracer/OpenTelemetryKeys.java | 51 -- .../judgmentlabs/judgeval/tracer/Tracer.java | 59 +- .../judgeval/tracer/TracerConfiguration.java | 55 +- .../exporters/JudgmentSpanExporter.java | 90 +-- .../tracer/exporters/NoOpSpanExporter.java | 7 +- .../judgmentlabs/judgeval/utils/Logger.java | 33 +- 48 files changed, 1050 insertions(+), 976 deletions(-) create mode 100644 src/main/java/com/judgmentlabs/judgeval/tracer/JudgevalTraceKeys.java delete mode 100644 src/main/java/com/judgmentlabs/judgeval/tracer/OpenTelemetryKeys.java diff --git a/.vscode/java-formatter.xml b/.vscode/java-formatter.xml index a14f80e..ac58365 100644 --- a/.vscode/java-formatter.xml +++ b/.vscode/java-formatter.xml @@ -1,337 +1,315 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + + + + + - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - + + + + + + + + - + + + + + + + + + - - - - - - - - - - - + + + + + - - - - + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + - - - - - - - + + - - + + + + - - - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - \ No newline at end of file + diff --git a/src/main/java/com/judgmentlabs/judgeval/Env.java b/src/main/java/com/judgmentlabs/judgeval/Env.java index e2a07e8..e3b7440 100644 --- a/src/main/java/com/judgmentlabs/judgeval/Env.java +++ b/src/main/java/com/judgmentlabs/judgeval/Env.java @@ -1,16 +1,16 @@ package com.judgmentlabs.judgeval; public final class Env { - private Env() {} + private Env() { + } - public static final String JUDGMENT_API_KEY = getEnvVar("JUDGMENT_API_KEY"); - public static final String JUDGMENT_ORG_ID = getEnvVar("JUDGMENT_ORG_ID"); - public static final String JUDGMENT_API_URL = - getEnvVar("JUDGMENT_API_URL", "https://api.judgmentlabs.ai"); - public static final String JUDGMENT_DEFAULT_GPT_MODEL = - getEnvVar("JUDGMENT_DEFAULT_GPT_MODEL", "gpt-4.1"); - public static final String JUDGMENT_NO_COLOR = getEnvVar("JUDGMENT_NO_COLOR"); - public static final String JUDGMENT_LOG_LEVEL = getEnvVar("JUDGMENT_LOG_LEVEL", "warn"); + public static final String JUDGMENT_API_KEY = getEnvVar("JUDGMENT_API_KEY"); + public static final String JUDGMENT_ORG_ID = getEnvVar("JUDGMENT_ORG_ID"); + public static final String JUDGMENT_API_URL = getEnvVar("JUDGMENT_API_URL", + "https://api.judgmentlabs.ai"); + public static final String JUDGMENT_DEFAULT_GPT_MODEL = getEnvVar("JUDGMENT_DEFAULT_GPT_MODEL", "gpt-4.1"); + public static final String JUDGMENT_NO_COLOR = getEnvVar("JUDGMENT_NO_COLOR"); + public static final String JUDGMENT_LOG_LEVEL = getEnvVar("JUDGMENT_LOG_LEVEL", "warn"); private static String getEnvVar(String varName) { return getEnvVar(varName, null); diff --git a/src/main/java/com/judgmentlabs/judgeval/Version.java b/src/main/java/com/judgmentlabs/judgeval/Version.java index f8d252e..d97531b 100644 --- a/src/main/java/com/judgmentlabs/judgeval/Version.java +++ b/src/main/java/com/judgmentlabs/judgeval/Version.java @@ -2,7 +2,8 @@ public final class Version { - private Version() {} + private Version() { + } /** * Gets the current SDK version. @@ -11,8 +12,6 @@ private Version() {} */ public static String getVersion() { Package pkg = Version.class.getPackage(); - return (pkg != null && pkg.getImplementationVersion() != null) - ? pkg.getImplementationVersion() - : "unknown"; + return (pkg != null && pkg.getImplementationVersion() != null) ? pkg.getImplementationVersion() : "unknown"; } } diff --git a/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java b/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java index 0f11b45..49e9af2 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java @@ -1,14 +1,18 @@ package com.judgmentlabs.judgeval.data; public enum APIScorerType { - PROMPT_SCORER("Prompt Scorer"), TRACE_PROMPT_SCORER("Trace Prompt Scorer"), FAITHFULNESS( - "Faithfulness"), ANSWER_RELEVANCY("Answer Relevancy"), ANSWER_CORRECTNESS( - "Answer Correctness"), INSTRUCTION_ADHERENCE( - "Instruction Adherence"), EXECUTION_ORDER( - "Execution Order"), DERAILMENT("Derailment"), TOOL_ORDER( - "Tool Order"), CLASSIFIER( - "Classifier"), TOOL_DEPENDENCY( - "Tool Dependency"), CUSTOM("Custom"); + PROMPT_SCORER("Prompt Scorer"), + TRACE_PROMPT_SCORER("Trace Prompt Scorer"), + FAITHFULNESS("Faithfulness"), + ANSWER_RELEVANCY("Answer Relevancy"), + ANSWER_CORRECTNESS("Answer Correctness"), + INSTRUCTION_ADHERENCE("Instruction Adherence"), + EXECUTION_ORDER("Execution Order"), + DERAILMENT("Derailment"), + TOOL_ORDER("Tool Order"), + CLASSIFIER("Classifier"), + TOOL_DEPENDENCY("Tool Dependency"), + CUSTOM("Custom"); private final String value; diff --git a/src/main/java/com/judgmentlabs/judgeval/data/Example.java b/src/main/java/com/judgmentlabs/judgeval/data/Example.java index ebfc049..a09ec10 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/Example.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/Example.java @@ -7,8 +7,10 @@ public class Example extends com.judgmentlabs.judgeval.internal.api.models.Examp public Example() { super(); - setExampleId(UUID.randomUUID().toString()); - setCreatedAt(Instant.now().toString()); + setExampleId(UUID.randomUUID() + .toString()); + setCreatedAt(Instant.now() + .toString()); setName(null); } diff --git a/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java index ae4b31f..435dd89 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java @@ -9,15 +9,17 @@ import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; -public class ExampleEvaluationRun - extends com.judgmentlabs.judgeval.internal.api.models.ExampleEvaluationRun { +public class ExampleEvaluationRun extends com.judgmentlabs.judgeval.internal.api.models.ExampleEvaluationRun { private String organizationId; public ExampleEvaluationRun() { super(); - setId(UUID.randomUUID().toString()); - setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); + setId(UUID.randomUUID() + .toString()); + setCreatedAt(Instant.now() + .atOffset(ZoneOffset.UTC) + .format(DateTimeFormatter.ISO_INSTANT)); } public ExampleEvaluationRun(String projectName, String evalName, List examples, @@ -26,8 +28,7 @@ public ExampleEvaluationRun(String projectName, String evalName, List e setProjectName(projectName); setEvalName(evalName); @SuppressWarnings("unchecked") - List internalExamples = - (List) (List) examples; + List internalExamples = (List) (List) examples; setExamples(internalExamples); setModel(model); setOrganizationId(organizationId); @@ -69,14 +70,15 @@ public static Builder builder(String projectName, String evalName) { } public static final class Builder { - private String projectName; - private String evalName; - private List examples; + private String projectName; + private String evalName; + private List examples; private List scorers; - private String model; - private String organizationId; + private String model; + private String organizationId; - private Builder() {} + private Builder() { + } private Builder(String projectName, String evalName) { this.projectName = projectName; @@ -130,10 +132,12 @@ public Builder organizationId(String organizationId) { } public ExampleEvaluationRun build() { - if (projectName == null || projectName.trim().isEmpty()) { + if (projectName == null || projectName.trim() + .isEmpty()) { throw new IllegalArgumentException("Project name is required"); } - if (evalName == null || evalName.trim().isEmpty()) { + if (evalName == null || evalName.trim() + .isEmpty()) { throw new IllegalArgumentException("Evaluation name is required"); } if (examples == null || examples.isEmpty()) { @@ -143,8 +147,7 @@ public ExampleEvaluationRun build() { throw new IllegalArgumentException("At least one scorer is required"); } - return new ExampleEvaluationRun(projectName, evalName, examples, scorers, model, - organizationId); + return new ExampleEvaluationRun(projectName, evalName, examples, scorers, model, organizationId); } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java b/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java index 096a141..0f262c7 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java @@ -22,8 +22,7 @@ public Builder success(Boolean success) { public Builder scorersData(List scorersData) { @SuppressWarnings("unchecked") - List internalList = - (List) (List) scorersData; + List internalList = (List) (List) scorersData; result.setScorersData(internalList); return this; } @@ -32,7 +31,8 @@ public Builder scorerData(ScorerData scorerData) { if (result.getScorersData() == null) { result.setScorersData(new java.util.ArrayList<>()); } - result.getScorersData().add(scorerData); + result.getScorersData() + .add(scorerData); return this; } diff --git a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java index 526e640..2687700 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java @@ -9,20 +9,22 @@ import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; -public class TraceEvaluationRun - extends com.judgmentlabs.judgeval.internal.api.models.TraceEvaluationRun { +public class TraceEvaluationRun extends com.judgmentlabs.judgeval.internal.api.models.TraceEvaluationRun { private String organizationId; public TraceEvaluationRun() { super(); - setId(UUID.randomUUID().toString()); - setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); + setId(UUID.randomUUID() + .toString()); + setCreatedAt(Instant.now() + .atOffset(ZoneOffset.UTC) + .format(DateTimeFormatter.ISO_INSTANT)); setIsOffline(false); } - public TraceEvaluationRun(String projectName, String evalName, List scorers, - String model, String organizationId, List> traceAndSpanIds) { + public TraceEvaluationRun(String projectName, String evalName, List scorers, String model, + String organizationId, List> traceAndSpanIds) { this(); setProjectName(projectName); setEvalName(evalName); @@ -35,15 +37,13 @@ public TraceEvaluationRun(String projectName, String evalName, List> convertTraceAndSpanIds(List> traceAndSpanIds) { if (traceAndSpanIds == null || traceAndSpanIds.isEmpty()) { - throw new IllegalArgumentException( - "Trace and span IDs are required for trace evaluations."); + throw new IllegalArgumentException("Trace and span IDs are required for trace evaluations."); } List> converted = new java.util.ArrayList<>(); for (List pair : traceAndSpanIds) { if (pair == null || pair.size() != 2) { - throw new IllegalArgumentException( - "Each trace and span ID pair must contain exactly 2 elements."); + throw new IllegalArgumentException("Each trace and span ID pair must contain exactly 2 elements."); } converted.add(List.of(pair.get(0), pair.get(1))); } @@ -84,14 +84,15 @@ public static Builder builder(String projectName, String evalName) { } public static final class Builder { - private String projectName; - private String evalName; + private String projectName; + private String evalName; private List scorers; - private String model; - private String organizationId; + private String model; + private String organizationId; private List> traceAndSpanIds; - private Builder() {} + private Builder() { + } private Builder(String projectName, String evalName) { this.projectName = projectName; @@ -145,22 +146,22 @@ public Builder traceAndSpanId(String traceId, String spanId) { } public TraceEvaluationRun build() { - if (projectName == null || projectName.trim().isEmpty()) { + if (projectName == null || projectName.trim() + .isEmpty()) { throw new IllegalArgumentException("Project name is required"); } - if (evalName == null || evalName.trim().isEmpty()) { + if (evalName == null || evalName.trim() + .isEmpty()) { throw new IllegalArgumentException("Evaluation name is required"); } if (scorers == null || scorers.isEmpty()) { throw new IllegalArgumentException("At least one scorer is required"); } if (traceAndSpanIds == null || traceAndSpanIds.isEmpty()) { - throw new IllegalArgumentException( - "At least one trace and span ID pair is required"); + throw new IllegalArgumentException("At least one trace and span ID pair is required"); } - return new TraceEvaluationRun(projectName, evalName, scorers, model, organizationId, - traceAndSpanIds); + return new TraceEvaluationRun(projectName, evalName, scorers, model, organizationId, traceAndSpanIds); } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentAPIError.java b/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentAPIError.java index b200c0c..8126891 100644 --- a/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentAPIError.java +++ b/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentAPIError.java @@ -1,7 +1,7 @@ package com.judgmentlabs.judgeval.exceptions; public class JudgmentAPIError extends RuntimeException { - private final int statusCode; + private final int statusCode; private final String detail; public JudgmentAPIError(int statusCode, String detail) { diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java index 1749154..f7c4b1c 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java @@ -16,19 +16,19 @@ import com.judgmentlabs.judgeval.internal.api.models.*; public class JudgmentAsyncClient { - private final HttpClient client; + private final HttpClient client; private final ObjectMapper mapper; - private final String baseUrl; - private final String apiKey; - private final String organizationId; + private final String baseUrl; + private final String apiKey; + private final String organizationId; - public JudgmentAsyncClient(@NotNull String baseUrl, @NotNull String apiKey, - @NotNull String organizationId) { + public JudgmentAsyncClient(@NotNull String baseUrl, @NotNull String apiKey, @NotNull String organizationId) { this.baseUrl = Objects.requireNonNull(baseUrl, "Base URL cannot be null"); this.apiKey = Objects.requireNonNull(apiKey, "API key cannot be null"); - this.organizationId = - Objects.requireNonNull(organizationId, "Organization ID cannot be null"); - this.client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build(); + this.organizationId = Objects.requireNonNull(organizationId, "Organization ID cannot be null"); + this.client = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_1_1) + .build(); this.mapper = new ObjectMapper(); } @@ -36,10 +36,10 @@ private String buildUrl(String path, Map queryParams) { StringBuilder url = new StringBuilder(baseUrl).append(path); if (!queryParams.isEmpty()) { url.append("?"); - String queryString = - queryParams.entrySet().stream() - .map(entry -> entry.getKey() + "=" + entry.getValue()) - .reduce("", (a, b) -> a.isEmpty() ? b : a + "&" + b); + String queryString = queryParams.entrySet() + .stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .reduce("", (a, b) -> a.isEmpty() ? b : a + "&" + b); url.append(queryString); } return url.toString(); @@ -50,18 +50,17 @@ private String buildUrl(String path) { } private String[] buildHeaders() { - return new String[] {"Content-Type", "application/json", "Authorization", - "Bearer " + apiKey, - "X-Organization-Id", organizationId}; + return new String[] { "Content-Type", "application/json", "Authorization", "Bearer " + apiKey, + "X-Organization-Id", organizationId }; } private T handleResponse(HttpResponse response) { if (response.statusCode() >= 400) { - throw new RuntimeException( - "HTTP Error: " + response.statusCode() + " - " + response.body()); + throw new RuntimeException("HTTP Error: " + response.statusCode() + " - " + response.body()); } try { - return mapper.readValue(response.body(), new TypeReference() {}); + return mapper.readValue(response.body(), new TypeReference() { + }); } catch (Exception e) { throw new RuntimeException("Failed to parse response", e); } @@ -75,9 +74,11 @@ public CompletableFuture addToRunEvalQueue(@NotNull ExampleEvaluationRun } catch (Exception e) { throw new RuntimeException("Failed to serialize payload", e); } - HttpRequest request = - HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } @@ -90,9 +91,11 @@ public CompletableFuture logEvalResults(@NotNull EvalResults payload) { } catch (Exception e) { throw new RuntimeException("Failed to serialize payload", e); } - HttpRequest request = - HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } @@ -105,15 +108,16 @@ public CompletableFuture fetchExperimentRun(@NotNull EvalResultsFetch pa } catch (Exception e) { throw new RuntimeException("Failed to serialize payload", e); } - HttpRequest request = - HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } - public CompletableFuture scorerExists( - @NotNull ScorerExistsRequest payload) { + public CompletableFuture scorerExists(@NotNull ScorerExistsRequest payload) { String url = buildUrl("/scorer_exists/"); String jsonPayload; try { @@ -121,15 +125,16 @@ public CompletableFuture scorerExists( } catch (Exception e) { throw new RuntimeException("Failed to serialize payload", e); } - HttpRequest request = - HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } - public CompletableFuture saveScorer( - @NotNull SavePromptScorerRequest payload) { + public CompletableFuture saveScorer(@NotNull SavePromptScorerRequest payload) { String url = buildUrl("/save_scorer/"); String jsonPayload; try { @@ -137,15 +142,16 @@ public CompletableFuture saveScorer( } catch (Exception e) { throw new RuntimeException("Failed to serialize payload", e); } - HttpRequest request = - HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } - public CompletableFuture fetchScorers( - @NotNull FetchPromptScorersRequest payload) { + public CompletableFuture fetchScorers(@NotNull FetchPromptScorersRequest payload) { String url = buildUrl("/fetch_scorers/"); String jsonPayload; try { @@ -153,15 +159,16 @@ public CompletableFuture fetchScorers( } catch (Exception e) { throw new RuntimeException("Failed to serialize payload", e); } - HttpRequest request = - HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } - public CompletableFuture projectsResolve( - @NotNull ResolveProjectNameRequest payload) { + public CompletableFuture projectsResolve(@NotNull ResolveProjectNameRequest payload) { String url = buildUrl("/projects/resolve/"); String jsonPayload; try { @@ -169,9 +176,11 @@ public CompletableFuture projectsResolve( } catch (Exception e) { throw new RuntimeException("Failed to serialize payload", e); } - HttpRequest request = - HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java index 995bd68..0b33b8b 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java @@ -26,19 +26,19 @@ import com.judgmentlabs.judgeval.internal.api.models.ScorerExistsResponse; public class JudgmentSyncClient { - private final HttpClient client; + private final HttpClient client; private final ObjectMapper mapper; - private final String baseUrl; - private final String apiKey; - private final String organizationId; + private final String baseUrl; + private final String apiKey; + private final String organizationId; - public JudgmentSyncClient(@NotNull String baseUrl, @NotNull String apiKey, - @NotNull String organizationId) { + public JudgmentSyncClient(@NotNull String baseUrl, @NotNull String apiKey, @NotNull String organizationId) { this.baseUrl = Objects.requireNonNull(baseUrl, "Base URL cannot be null"); this.apiKey = Objects.requireNonNull(apiKey, "API key cannot be null"); - this.organizationId = - Objects.requireNonNull(organizationId, "Organization ID cannot be null"); - this.client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build(); + this.organizationId = Objects.requireNonNull(organizationId, "Organization ID cannot be null"); + this.client = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_1_1) + .build(); this.mapper = new ObjectMapper(); } @@ -46,10 +46,10 @@ private String buildUrl(String path, Map queryParams) { StringBuilder url = new StringBuilder(baseUrl).append(path); if (!queryParams.isEmpty()) { url.append("?"); - String queryString = - queryParams.entrySet().stream() - .map(entry -> entry.getKey() + "=" + entry.getValue()) - .reduce("", (a, b) -> a.isEmpty() ? b : a + "&" + b); + String queryString = queryParams.entrySet() + .stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .reduce("", (a, b) -> a.isEmpty() ? b : a + "&" + b); url.append(queryString); } return url.toString(); @@ -60,52 +60,54 @@ private String buildUrl(String path) { } private String[] buildHeaders() { - return new String[] {"Content-Type", "application/json", "Authorization", - "Bearer " + apiKey, - "X-Organization-Id", organizationId}; + return new String[] { "Content-Type", "application/json", "Authorization", "Bearer " + apiKey, + "X-Organization-Id", organizationId }; } private T handleResponse(HttpResponse response) throws IOException { if (response.statusCode() >= 400) { - throw new RuntimeException( - "HTTP Error: " + response.statusCode() + " - " + response.body()); + throw new RuntimeException("HTTP Error: " + response.statusCode() + " - " + response.body()); } try { - return mapper.readValue(response.body(), new TypeReference() {}); + return mapper.readValue(response.body(), new TypeReference() { + }); } catch (Exception e) { throw new RuntimeException("Failed to parse response", e); } } - public Object addToRunEvalQueue(@NotNull ExampleEvaluationRun payload) - throws IOException, InterruptedException { + public Object addToRunEvalQueue(@NotNull ExampleEvaluationRun payload) throws IOException, InterruptedException { String url = buildUrl("/add_to_run_eval_queue/"); String jsonPayload = mapper.writeValueAsString(payload); - HttpRequest request = - HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return handleResponse(response); } - public Object logEvalResults(@NotNull EvalResults payload) - throws IOException, InterruptedException { + public Object logEvalResults(@NotNull EvalResults payload) throws IOException, InterruptedException { String url = buildUrl("/log_eval_results/"); String jsonPayload = mapper.writeValueAsString(payload); - HttpRequest request = - HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return handleResponse(response); } - public Object fetchExperimentRun(@NotNull EvalResultsFetch payload) - throws IOException, InterruptedException { + public Object fetchExperimentRun(@NotNull EvalResultsFetch payload) throws IOException, InterruptedException { String url = buildUrl("/fetch_experiment_run/"); String jsonPayload = mapper.writeValueAsString(payload); - HttpRequest request = - HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return handleResponse(response); } @@ -114,9 +116,11 @@ public ScorerExistsResponse scorerExists(@NotNull ScorerExistsRequest payload) throws IOException, InterruptedException { String url = buildUrl("/scorer_exists/"); String jsonPayload = mapper.writeValueAsString(payload); - HttpRequest request = - HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return mapper.readValue(response.body(), ScorerExistsResponse.class); } @@ -125,9 +129,11 @@ public SavePromptScorerResponse saveScorer(@NotNull SavePromptScorerRequest payl throws IOException, InterruptedException { String url = buildUrl("/save_scorer/"); String jsonPayload = mapper.writeValueAsString(payload); - HttpRequest request = - HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return mapper.readValue(response.body(), SavePromptScorerResponse.class); } @@ -136,9 +142,11 @@ public FetchPromptScorersResponse fetchScorers(@NotNull FetchPromptScorersReques throws IOException, InterruptedException { String url = buildUrl("/fetch_scorers/"); String jsonPayload = mapper.writeValueAsString(payload); - HttpRequest request = - HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return mapper.readValue(response.body(), FetchPromptScorersResponse.class); } @@ -147,9 +155,11 @@ public ResolveProjectNameResponse projectsResolve(@NotNull ResolveProjectNameReq throws IOException, InterruptedException { String url = buildUrl("/projects/resolve/"); String jsonPayload = mapper.writeValueAsString(payload); - HttpRequest request = - HttpRequest.newBuilder().POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return mapper.readValue(response.body(), ResolveProjectNameResponse.class); } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java index 5b42a44..688c427 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java @@ -13,37 +13,37 @@ public class BaseScorer { @JsonProperty("score_type") @NotNull - private String scoreType; + private String scoreType; @JsonProperty("threshold") - private Double threshold; + private Double threshold; @JsonProperty("name") - private String name; + private String name; @JsonProperty("class_name") - private String className; + private String className; @JsonProperty("score") - private Double score; + private Double score; @JsonProperty("score_breakdown") - private Object scoreBreakdown; + private Object scoreBreakdown; @JsonProperty("reason") - private String reason; + private String reason; @JsonProperty("using_native_model") - private Boolean usingNativeModel; + private Boolean usingNativeModel; @JsonProperty("success") - private Boolean success; + private Boolean success; @JsonProperty("model") - private String model; + private String model; @JsonProperty("model_client") - private Object modelClient; + private Object modelClient; @JsonProperty("strict_mode") - private Boolean strictMode; + private Boolean strictMode; @JsonProperty("error") - private String error; + private String error; @JsonProperty("additional_metadata") - private Object additionalMetadata; + private Object additionalMetadata; @JsonProperty("user") - private String user; + private String user; @JsonProperty("server_hosted") - private Boolean serverHosted; + private Boolean serverHosted; private Map additionalProperties = new HashMap<>(); @@ -192,28 +192,21 @@ public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) return false; BaseScorer other = (BaseScorer) obj; - return Objects.equals(scoreType, other.scoreType) - && Objects.equals(threshold, other.threshold) + return Objects.equals(scoreType, other.scoreType) && Objects.equals(threshold, other.threshold) && Objects.equals(name, other.name) && Objects.equals(className, other.className) - && Objects.equals(score, other.score) - && Objects.equals(scoreBreakdown, other.scoreBreakdown) - && Objects.equals(reason, other.reason) - && Objects.equals(usingNativeModel, other.usingNativeModel) + && Objects.equals(score, other.score) && Objects.equals(scoreBreakdown, other.scoreBreakdown) + && Objects.equals(reason, other.reason) && Objects.equals(usingNativeModel, other.usingNativeModel) && Objects.equals(success, other.success) && Objects.equals(model, other.model) - && Objects.equals(modelClient, other.modelClient) - && Objects.equals(strictMode, other.strictMode) - && Objects.equals(error, other.error) - && Objects.equals(additionalMetadata, other.additionalMetadata) - && Objects.equals(user, other.user) - && Objects.equals(serverHosted, other.serverHosted) + && Objects.equals(modelClient, other.modelClient) && Objects.equals(strictMode, other.strictMode) + && Objects.equals(error, other.error) && Objects.equals(additionalMetadata, other.additionalMetadata) + && Objects.equals(user, other.user) && Objects.equals(serverHosted, other.serverHosted) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash(scoreType, threshold, name, className, score, scoreBreakdown, reason, - usingNativeModel, success, model, modelClient, strictMode, error, - additionalMetadata, user, - serverHosted, Objects.hashCode(additionalProperties)); + return Objects.hash(scoreType, threshold, name, className, score, scoreBreakdown, reason, usingNativeModel, + success, model, modelClient, strictMode, error, additionalMetadata, user, serverHosted, + Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java index 25de32f..79116c5 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java @@ -17,7 +17,7 @@ public class EvalResults { private List results; @JsonProperty("run") @NotNull - private Object run; + private Object run; private Map additionalProperties = new HashMap<>(); diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java index b6a0b35..f49f04b 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java @@ -13,10 +13,10 @@ public class EvalResultsFetch { @JsonProperty("experiment_run_id") @NotNull - private String experimentRunId; + private String experimentRunId; @JsonProperty("project_name") @NotNull - private String projectName; + private String projectName; private Map additionalProperties = new HashMap<>(); diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java index e160933..c9e82c9 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java @@ -10,11 +10,11 @@ public class Example { @JsonProperty("example_id") - private String exampleId; + private String exampleId; @JsonProperty("created_at") - private String createdAt; + private String createdAt; @JsonProperty("name") - private String name; + private String name; private Map additionalProperties = new HashMap<>(); @@ -59,10 +59,8 @@ public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) return false; Example other = (Example) obj; - return Objects.equals(exampleId, other.exampleId) - && Objects.equals(createdAt, other.createdAt) - && Objects.equals(name, other.name) - && Objects.equals(additionalProperties, other.additionalProperties); + return Objects.equals(exampleId, other.exampleId) && Objects.equals(createdAt, other.createdAt) + && Objects.equals(name, other.name) && Objects.equals(additionalProperties, other.additionalProperties); } @Override diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java index 3cc0edd..19be0d9 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java @@ -13,28 +13,28 @@ public class ExampleEvaluationRun { @JsonProperty("id") - private String id; + private String id; @JsonProperty("project_name") @NotNull - private String projectName; + private String projectName; @JsonProperty("eval_name") @NotNull - private String evalName; + private String evalName; @JsonProperty("custom_scorers") - private List customScorers; + private List customScorers; @JsonProperty("judgment_scorers") - private List judgmentScorers; + private List judgmentScorers; @JsonProperty("model") - private String model; + private String model; @JsonProperty("created_at") - private String createdAt; + private String createdAt; @JsonProperty("examples") @NotNull - private List examples; + private List examples; @JsonProperty("trace_span_id") - private String traceSpanId; + private String traceSpanId; @JsonProperty("trace_id") - private String traceId; + private String traceId; private Map additionalProperties = new HashMap<>(); @@ -136,20 +136,16 @@ public boolean equals(Object obj) { return false; ExampleEvaluationRun other = (ExampleEvaluationRun) obj; return Objects.equals(id, other.id) && Objects.equals(projectName, other.projectName) - && Objects.equals(evalName, other.evalName) - && Objects.equals(customScorers, other.customScorers) - && Objects.equals(judgmentScorers, other.judgmentScorers) - && Objects.equals(model, other.model) && Objects.equals(createdAt, other.createdAt) - && Objects.equals(examples, other.examples) - && Objects.equals(traceSpanId, other.traceSpanId) - && Objects.equals(traceId, other.traceId) + && Objects.equals(evalName, other.evalName) && Objects.equals(customScorers, other.customScorers) + && Objects.equals(judgmentScorers, other.judgmentScorers) && Objects.equals(model, other.model) + && Objects.equals(createdAt, other.createdAt) && Objects.equals(examples, other.examples) + && Objects.equals(traceSpanId, other.traceSpanId) && Objects.equals(traceId, other.traceId) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash(id, projectName, evalName, customScorers, judgmentScorers, model, - createdAt, - examples, traceSpanId, traceId, Objects.hashCode(additionalProperties)); + return Objects.hash(id, projectName, evalName, customScorers, judgmentScorers, model, createdAt, examples, + traceSpanId, traceId, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java index 79f6be3..c650a72 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java @@ -11,9 +11,9 @@ public class FetchPromptScorersRequest { @JsonProperty("names") - private List names; + private List names; @JsonProperty("is_trace") - private Boolean isTrace; + private Boolean isTrace; private Map additionalProperties = new HashMap<>(); diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java index 478b9ef..f4f4061 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java @@ -14,7 +14,7 @@ public class FetchPromptScorersResponse { @JsonProperty("scorers") @NotNull - private List scorers; + private List scorers; private Map additionalProperties = new HashMap<>(); diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java index 076c555..a4c0075 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java @@ -14,45 +14,45 @@ public class OtelTraceSpan { @JsonProperty("organization_id") @NotNull - private String organizationId; + private String organizationId; @JsonProperty("project_id") - private String projectId; + private String projectId; @JsonProperty("user_id") @NotNull - private String userId; + private String userId; @JsonProperty("timestamp") @NotNull - private String timestamp; + private String timestamp; @JsonProperty("trace_id") @NotNull - private String traceId; + private String traceId; @JsonProperty("span_id") @NotNull - private String spanId; + private String spanId; @JsonProperty("parent_span_id") - private String parentSpanId; + private String parentSpanId; @JsonProperty("trace_state") - private String traceState; + private String traceState; @JsonProperty("span_name") - private String spanName; + private String spanName; @JsonProperty("span_kind") - private String spanKind; + private String spanKind; @JsonProperty("service_name") - private String serviceName; + private String serviceName; @JsonProperty("resource_attributes") - private Object resourceAttributes; + private Object resourceAttributes; @JsonProperty("span_attributes") - private Object spanAttributes; + private Object spanAttributes; @JsonProperty("duration") - private Integer duration; + private Integer duration; @JsonProperty("status_code") - private Integer statusCode; + private Integer statusCode; @JsonProperty("status_message") - private String statusMessage; + private String statusMessage; @JsonProperty("events") - private List events; + private List events; @JsonProperty("links") - private List links; + private List links; private Map additionalProperties = new HashMap<>(); @@ -217,32 +217,24 @@ public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) return false; OtelTraceSpan other = (OtelTraceSpan) obj; - return Objects.equals(organizationId, other.organizationId) - && Objects.equals(projectId, other.projectId) - && Objects.equals(userId, other.userId) - && Objects.equals(timestamp, other.timestamp) - && Objects.equals(traceId, other.traceId) - && Objects.equals(spanId, other.spanId) - && Objects.equals(parentSpanId, other.parentSpanId) - && Objects.equals(traceState, other.traceState) - && Objects.equals(spanName, other.spanName) - && Objects.equals(spanKind, other.spanKind) + return Objects.equals(organizationId, other.organizationId) && Objects.equals(projectId, other.projectId) + && Objects.equals(userId, other.userId) && Objects.equals(timestamp, other.timestamp) + && Objects.equals(traceId, other.traceId) && Objects.equals(spanId, other.spanId) + && Objects.equals(parentSpanId, other.parentSpanId) && Objects.equals(traceState, other.traceState) + && Objects.equals(spanName, other.spanName) && Objects.equals(spanKind, other.spanKind) && Objects.equals(serviceName, other.serviceName) && Objects.equals(resourceAttributes, other.resourceAttributes) - && Objects.equals(spanAttributes, other.spanAttributes) - && Objects.equals(duration, other.duration) - && Objects.equals(statusCode, other.statusCode) - && Objects.equals(statusMessage, other.statusMessage) + && Objects.equals(spanAttributes, other.spanAttributes) && Objects.equals(duration, other.duration) + && Objects.equals(statusCode, other.statusCode) && Objects.equals(statusMessage, other.statusMessage) && Objects.equals(events, other.events) && Objects.equals(links, other.links) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash(organizationId, projectId, userId, timestamp, traceId, spanId, - parentSpanId, - traceState, spanName, spanKind, serviceName, resourceAttributes, spanAttributes, - duration, - statusCode, statusMessage, events, links, Objects.hashCode(additionalProperties)); + return Objects.hash(organizationId, projectId, userId, timestamp, traceId, spanId, parentSpanId, traceState, + spanName, spanKind, serviceName, resourceAttributes, spanAttributes, duration, statusCode, + statusMessage, + events, links, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java index c2d8ba2..874f779 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java @@ -13,36 +13,36 @@ public class PromptScorer { @JsonProperty("id") @NotNull - private String id; + private String id; @JsonProperty("user_id") @NotNull - private String userId; + private String userId; @JsonProperty("organization_id") @NotNull - private String organizationId; + private String organizationId; @JsonProperty("name") @NotNull - private String name; + private String name; @JsonProperty("prompt") @NotNull - private String prompt; + private String prompt; @JsonProperty("threshold") @NotNull - private Double threshold; + private Double threshold; @JsonProperty("model") - private String model; + private String model; @JsonProperty("options") - private Object options; + private Object options; @JsonProperty("description") - private String description; + private String description; @JsonProperty("created_at") - private String createdAt; + private String createdAt; @JsonProperty("updated_at") - private String updatedAt; + private String updatedAt; @JsonProperty("is_trace") - private Boolean isTrace; + private Boolean isTrace; @JsonProperty("is_bucket_rubric") - private Boolean isBucketRubric; + private Boolean isBucketRubric; private Map additionalProperties = new HashMap<>(); @@ -168,23 +168,18 @@ public boolean equals(Object obj) { return false; PromptScorer other = (PromptScorer) obj; return Objects.equals(id, other.id) && Objects.equals(userId, other.userId) - && Objects.equals(organizationId, other.organizationId) - && Objects.equals(name, other.name) - && Objects.equals(prompt, other.prompt) - && Objects.equals(threshold, other.threshold) + && Objects.equals(organizationId, other.organizationId) && Objects.equals(name, other.name) + && Objects.equals(prompt, other.prompt) && Objects.equals(threshold, other.threshold) && Objects.equals(model, other.model) && Objects.equals(options, other.options) - && Objects.equals(description, other.description) - && Objects.equals(createdAt, other.createdAt) - && Objects.equals(updatedAt, other.updatedAt) - && Objects.equals(isTrace, other.isTrace) + && Objects.equals(description, other.description) && Objects.equals(createdAt, other.createdAt) + && Objects.equals(updatedAt, other.updatedAt) && Objects.equals(isTrace, other.isTrace) && Objects.equals(isBucketRubric, other.isBucketRubric) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash(id, userId, organizationId, name, prompt, threshold, model, options, - description, createdAt, updatedAt, isTrace, isBucketRubric, - Objects.hashCode(additionalProperties)); + return Objects.hash(id, userId, organizationId, name, prompt, threshold, model, options, description, + createdAt, updatedAt, isTrace, isBucketRubric, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java index 5398b07..631ba16 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java @@ -13,7 +13,7 @@ public class ResolveProjectNameRequest { @JsonProperty("project_name") @NotNull - private String projectName; + private String projectName; private Map additionalProperties = new HashMap<>(); diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java index 94cc197..a3da9dc 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java @@ -13,7 +13,7 @@ public class ResolveProjectNameResponse { @JsonProperty("project_id") @NotNull - private String projectId; + private String projectId; private Map additionalProperties = new HashMap<>(); diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java index dc19cc3..c6d48b0 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java @@ -13,21 +13,21 @@ public class SavePromptScorerRequest { @JsonProperty("name") @NotNull - private String name; + private String name; @JsonProperty("prompt") @NotNull - private String prompt; + private String prompt; @JsonProperty("threshold") @NotNull - private Double threshold; + private Double threshold; @JsonProperty("model") - private String model; + private String model; @JsonProperty("is_trace") - private Boolean isTrace; + private Boolean isTrace; @JsonProperty("options") - private Object options; + private Object options; @JsonProperty("description") - private String description; + private String description; private Map additionalProperties = new HashMap<>(); diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java index 8748242..3db78c7 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java @@ -13,7 +13,7 @@ public class SavePromptScorerResponse { @JsonProperty("scorer_response") @NotNull - private PromptScorer scorerResponse; + private PromptScorer scorerResponse; private Map additionalProperties = new HashMap<>(); diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java index da2cd2c..12c4cb6 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java @@ -14,19 +14,19 @@ public class ScorerConfig { @JsonProperty("score_type") @NotNull - private String scoreType; + private String scoreType; @JsonProperty("name") - private String name; + private String name; @JsonProperty("threshold") - private Double threshold; + private Double threshold; @JsonProperty("model") - private String model; + private String model; @JsonProperty("strict_mode") - private Boolean strictMode; + private Boolean strictMode; @JsonProperty("required_params") - private List requiredParams; + private List requiredParams; @JsonProperty("kwargs") - private Object kwargs; + private Object kwargs; private Map additionalProperties = new HashMap<>(); @@ -105,8 +105,7 @@ public boolean equals(Object obj) { ScorerConfig other = (ScorerConfig) obj; return Objects.equals(scoreType, other.scoreType) && Objects.equals(name, other.name) && Objects.equals(threshold, other.threshold) && Objects.equals(model, other.model) - && Objects.equals(strictMode, other.strictMode) - && Objects.equals(requiredParams, other.requiredParams) + && Objects.equals(strictMode, other.strictMode) && Objects.equals(requiredParams, other.requiredParams) && Objects.equals(kwargs, other.kwargs) && Objects.equals(additionalProperties, other.additionalProperties); } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java index a72c72b..b28d8c6 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java @@ -12,28 +12,28 @@ public class ScorerData { @JsonProperty("id") - private String id; + private String id; @JsonProperty("name") @NotNull - private String name; + private String name; @JsonProperty("threshold") @NotNull - private Double threshold; + private Double threshold; @JsonProperty("success") @NotNull - private Boolean success; + private Boolean success; @JsonProperty("score") - private Double score; + private Double score; @JsonProperty("reason") - private String reason; + private String reason; @JsonProperty("strict_mode") - private Boolean strictMode; + private Boolean strictMode; @JsonProperty("evaluation_model") - private String evaluationModel; + private String evaluationModel; @JsonProperty("error") - private String error; + private String error; @JsonProperty("additional_metadata") - private Object additionalMetadata; + private Object additionalMetadata; private Map additionalProperties = new HashMap<>(); @@ -135,20 +135,17 @@ public boolean equals(Object obj) { return false; ScorerData other = (ScorerData) obj; return Objects.equals(id, other.id) && Objects.equals(name, other.name) - && Objects.equals(threshold, other.threshold) - && Objects.equals(success, other.success) + && Objects.equals(threshold, other.threshold) && Objects.equals(success, other.success) && Objects.equals(score, other.score) && Objects.equals(reason, other.reason) && Objects.equals(strictMode, other.strictMode) && Objects.equals(evaluationModel, other.evaluationModel) - && Objects.equals(error, other.error) - && Objects.equals(additionalMetadata, other.additionalMetadata) + && Objects.equals(error, other.error) && Objects.equals(additionalMetadata, other.additionalMetadata) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash(id, name, threshold, success, score, reason, strictMode, - evaluationModel, - error, additionalMetadata, Objects.hashCode(additionalProperties)); + return Objects.hash(id, name, threshold, success, score, reason, strictMode, evaluationModel, error, + additionalMetadata, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java index 540b646..9d21493 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java @@ -13,7 +13,7 @@ public class ScorerExistsRequest { @JsonProperty("name") @NotNull - private String name; + private String name; private Map additionalProperties = new HashMap<>(); @@ -42,8 +42,7 @@ public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) return false; ScorerExistsRequest other = (ScorerExistsRequest) obj; - return Objects.equals(name, other.name) - && Objects.equals(additionalProperties, other.additionalProperties); + return Objects.equals(name, other.name) && Objects.equals(additionalProperties, other.additionalProperties); } @Override diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java index 8bd564d..5a528fb 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java @@ -13,7 +13,7 @@ public class ScorerExistsResponse { @JsonProperty("exists") @NotNull - private Boolean exists; + private Boolean exists; private Map additionalProperties = new HashMap<>(); diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java index 27bb357..97d5a46 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java @@ -14,20 +14,20 @@ public class ScoringResult { @JsonProperty("success") @NotNull - private Boolean success; + private Boolean success; @JsonProperty("scorers_data") @NotNull - private List scorersData; + private List scorersData; @JsonProperty("name") - private String name; + private String name; @JsonProperty("data_object") - private Object dataObject; + private Object dataObject; @JsonProperty("trace_id") - private String traceId; + private String traceId; @JsonProperty("run_duration") - private Double runDuration; + private Double runDuration; @JsonProperty("evaluation_cost") - private Double evaluationCost; + private Double evaluationCost; private Map additionalProperties = new HashMap<>(); @@ -104,18 +104,16 @@ public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) return false; ScoringResult other = (ScoringResult) obj; - return Objects.equals(success, other.success) - && Objects.equals(scorersData, other.scorersData) + return Objects.equals(success, other.success) && Objects.equals(scorersData, other.scorersData) && Objects.equals(name, other.name) && Objects.equals(dataObject, other.dataObject) - && Objects.equals(traceId, other.traceId) - && Objects.equals(runDuration, other.runDuration) + && Objects.equals(traceId, other.traceId) && Objects.equals(runDuration, other.runDuration) && Objects.equals(evaluationCost, other.evaluationCost) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash(success, scorersData, name, dataObject, traceId, runDuration, - evaluationCost, Objects.hashCode(additionalProperties)); + return Objects.hash(success, scorersData, name, dataObject, traceId, runDuration, evaluationCost, + Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java index af949aa..7fcd56c 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java @@ -13,28 +13,28 @@ public class TraceEvaluationRun { @JsonProperty("id") - private String id; + private String id; @JsonProperty("project_name") @NotNull - private String projectName; + private String projectName; @JsonProperty("eval_name") @NotNull - private String evalName; + private String evalName; @JsonProperty("custom_scorers") - private List customScorers; + private List customScorers; @JsonProperty("judgment_scorers") - private List judgmentScorers; + private List judgmentScorers; @JsonProperty("model") - private String model; + private String model; @JsonProperty("created_at") - private String createdAt; + private String createdAt; @JsonProperty("trace_and_span_ids") @NotNull - private List> traceAndSpanIds; + private List> traceAndSpanIds; @JsonProperty("is_offline") - private Boolean isOffline; + private Boolean isOffline; @JsonProperty("is_bucket_run") - private Boolean isBucketRun; + private Boolean isBucketRun; private Map additionalProperties = new HashMap<>(); @@ -136,20 +136,16 @@ public boolean equals(Object obj) { return false; TraceEvaluationRun other = (TraceEvaluationRun) obj; return Objects.equals(id, other.id) && Objects.equals(projectName, other.projectName) - && Objects.equals(evalName, other.evalName) - && Objects.equals(customScorers, other.customScorers) - && Objects.equals(judgmentScorers, other.judgmentScorers) - && Objects.equals(model, other.model) && Objects.equals(createdAt, other.createdAt) - && Objects.equals(traceAndSpanIds, other.traceAndSpanIds) - && Objects.equals(isOffline, other.isOffline) - && Objects.equals(isBucketRun, other.isBucketRun) + && Objects.equals(evalName, other.evalName) && Objects.equals(customScorers, other.customScorers) + && Objects.equals(judgmentScorers, other.judgmentScorers) && Objects.equals(model, other.model) + && Objects.equals(createdAt, other.createdAt) && Objects.equals(traceAndSpanIds, other.traceAndSpanIds) + && Objects.equals(isOffline, other.isOffline) && Objects.equals(isBucketRun, other.isBucketRun) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash(id, projectName, evalName, customScorers, judgmentScorers, model, - createdAt, + return Objects.hash(id, projectName, evalName, customScorers, judgmentScorers, model, createdAt, traceAndSpanIds, isOffline, isBucketRun, Objects.hashCode(additionalProperties)); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java index 2b71ea0..6ad60e3 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java @@ -9,12 +9,11 @@ import com.judgmentlabs.judgeval.data.APIScorerType; import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; -public class APIScorer extends com.judgmentlabs.judgeval.internal.api.models.BaseScorer - implements BaseScorer { +public class APIScorer extends com.judgmentlabs.judgeval.internal.api.models.BaseScorer implements BaseScorer { private APIScorerType scoreType; @JsonIgnore - private List requiredParams; + private List requiredParams; public APIScorer(APIScorerType scoreType) { super(); @@ -29,8 +28,7 @@ public APIScorer(APIScorerType scoreType) { public void setThreshold(double threshold) { if (threshold < 0 || threshold > 1) { - throw new IllegalArgumentException( - "Threshold must be between 0 and 1, got: " + threshold); + throw new IllegalArgumentException("Threshold must be between 0 and 1, got: " + threshold); } super.setThreshold(threshold); } @@ -49,17 +47,21 @@ public void setRequiredParams(List requiredParams) { @Override public Double getThreshold() { - return Optional.ofNullable(super.getThreshold()).orElse(0.5); + return Optional.ofNullable(super.getThreshold()) + .orElse(0.5); } @Override public String getName() { - return Optional.ofNullable(super.getName()).map(Object::toString).orElse(null); + return Optional.ofNullable(super.getName()) + .map(Object::toString) + .orElse(null); } @Override public Boolean getStrictMode() { - return Optional.ofNullable(super.getStrictMode()).orElse(false); + return Optional.ofNullable(super.getStrictMode()) + .orElse(false); } @Override @@ -86,7 +88,8 @@ public static final class Builder { private Builder(Class scorerClass) { try { - this.scorer = scorerClass.getDeclaredConstructor().newInstance(); + this.scorer = scorerClass.getDeclaredConstructor() + .newInstance(); } catch (Exception e) { throw new RuntimeException("Failed to create scorer instance", e); } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java index ce73512..3a5b231 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java @@ -3,8 +3,8 @@ import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; /** - * Minimal interface for scorers used by BaseTracer. Only requires the essential methods needed for - * evaluation. + * Minimal interface for scorers used by BaseTracer. Only requires the essential + * methods needed for evaluation. */ public interface BaseScorer { /** diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java index 483d4e9..e0ec743 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java @@ -20,6 +20,7 @@ public static AnswerCorrectnessScorer create() { } public static AnswerCorrectnessScorer create(double threshold) { - return builder().threshold(threshold).build(); + return builder().threshold(threshold) + .build(); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java index ece195b..7434399 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java @@ -20,6 +20,7 @@ public static AnswerRelevancyScorer create() { } public static AnswerRelevancyScorer create(double threshold) { - return builder().threshold(threshold).build(); + return builder().threshold(threshold) + .build(); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/DerailmentScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/DerailmentScorer.java index b949f4f..431728d 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/DerailmentScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/DerailmentScorer.java @@ -17,6 +17,7 @@ public static DerailmentScorer create() { } public static DerailmentScorer create(double threshold) { - return builder().threshold(threshold).build(); + return builder().threshold(threshold) + .build(); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java index dd4ca09..bb53d23 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java @@ -20,6 +20,7 @@ public static FaithfulnessScorer create() { } public static FaithfulnessScorer create(double threshold) { - return builder().threshold(threshold).build(); + return builder().threshold(threshold) + .build(); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java index 96416b8..da71351 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java @@ -21,6 +21,7 @@ public static InstructionAdherenceScorer create() { } public static InstructionAdherenceScorer create(double threshold) { - return builder().threshold(threshold).build(); + return builder().threshold(threshold) + .build(); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java index 2e90502..f44c7f9 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java @@ -15,13 +15,12 @@ import com.judgmentlabs.judgeval.scorers.APIScorer; public abstract class BasePromptScorer extends APIScorer { - protected String prompt; + protected String prompt; protected Map options; - protected String judgmentApiKey; - protected String organizationId; + protected String judgmentApiKey; + protected String organizationId; - protected BasePromptScorer(APIScorerType scoreType, String name, String prompt, - double threshold, + protected BasePromptScorer(APIScorerType scoreType, String name, String prompt, double threshold, Map options, String judgmentApiKey, String organizationId) { super(scoreType); this.prompt = prompt; @@ -34,8 +33,7 @@ protected BasePromptScorer(APIScorerType scoreType, String name, String prompt, public static boolean scorerExists(String name, String judgmentApiKey, String organizationId) { try { - JudgmentSyncClient client = - new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); + JudgmentSyncClient client = new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); ScorerExistsRequest request = new ScorerExistsRequest(); request.setName(name); ScorerExistsResponse response = client.scorerExists(request); @@ -45,11 +43,10 @@ public static boolean scorerExists(String name, String judgmentApiKey, String or } } - public static com.judgmentlabs.judgeval.internal.api.models.PromptScorer fetchPromptScorer( - String name, String judgmentApiKey, String organizationId) { + public static com.judgmentlabs.judgeval.internal.api.models.PromptScorer fetchPromptScorer(String name, + String judgmentApiKey, String organizationId) { try { - JudgmentSyncClient client = - new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); + JudgmentSyncClient client = new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); FetchPromptScorersRequest request = new FetchPromptScorersRequest(); request.setNames(java.util.Collections.singletonList(name)); @@ -59,11 +56,10 @@ public static com.judgmentlabs.judgeval.internal.api.models.PromptScorer fetchPr .map(FetchPromptScorersResponse::getScorers) .filter(scorers -> scorers != null && !scorers.isEmpty()) .map(scorers -> scorers.get(0)) - .orElseThrow(() -> new JudgmentAPIError(404, - "Failed to fetch prompt scorer '" + name + "': not found")); + .orElseThrow( + () -> new JudgmentAPIError(404, "Failed to fetch prompt scorer '" + name + "': not found")); } catch (Exception e) { - throw new JudgmentAPIError(500, - "Failed to fetch prompt scorer '" + name + "': " + e.getMessage()); + throw new JudgmentAPIError(500, "Failed to fetch prompt scorer '" + name + "': " + e.getMessage()); } } @@ -76,7 +72,9 @@ public String getPrompt() { } public Map getOptions() { - return Optional.ofNullable(options).map(HashMap::new).orElse(null); + return Optional.ofNullable(options) + .map(HashMap::new) + .orElse(null); } public String getScorerName() { @@ -96,8 +94,7 @@ public Map getConfig() { @Override public String toString() { - return "PromptScorer(name=" + getName() + ", prompt=" + prompt + ", threshold=" - + getThreshold() + return "PromptScorer(name=" + getName() + ", prompt=" + prompt + ", threshold=" + getThreshold() + ", options=" + options + ")"; } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java index c0922c3..7ebd225 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java @@ -18,8 +18,7 @@ public PromptScorer(String name, String prompt, double threshold, Map options, String judgmentApiKey, String organizationId) { - super(APIScorerType.PROMPT_SCORER, name, prompt, threshold, options, judgmentApiKey, - organizationId); + super(APIScorerType.PROMPT_SCORER, name, prompt, threshold, options, judgmentApiKey, organizationId); } public static PromptScorer get(String name) { @@ -27,8 +26,8 @@ public static PromptScorer get(String name) { } public static PromptScorer get(String name, String judgmentApiKey, String organizationId) { - com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorerConfig = - fetchPromptScorer(name, judgmentApiKey, organizationId); + com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorerConfig = fetchPromptScorer(name, + judgmentApiKey, organizationId); if (Boolean.TRUE.equals(scorerConfig.getIsTrace())) { throw new JudgmentAPIError(400, "Scorer with name " + name + " is not a PromptScorer"); @@ -48,12 +47,10 @@ public static PromptScorer get(String name, String judgmentApiKey, String organi } } - return new PromptScorer(name, scorerConfig.getPrompt(), - Optional.ofNullable(scorerConfig.getThreshold()).orElse(0.5), options, - judgmentApiKey, organizationId); + return new PromptScorer(name, scorerConfig.getPrompt(), Optional.ofNullable(scorerConfig.getThreshold()) + .orElse(0.5), options, judgmentApiKey, organizationId); } - @Override public ScorerConfig getScorerConfig() { ScorerConfig cfg = new ScorerConfig(); diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java index 3ac7707..fe813bb 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java @@ -11,17 +11,14 @@ public class TracePromptScorer extends BasePromptScorer { - public TracePromptScorer(String name, String prompt, double threshold, - Map options) { - super(APIScorerType.TRACE_PROMPT_SCORER, name, prompt, threshold, options, - Env.JUDGMENT_API_KEY, + public TracePromptScorer(String name, String prompt, double threshold, Map options) { + super(APIScorerType.TRACE_PROMPT_SCORER, name, prompt, threshold, options, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); } - public TracePromptScorer(String name, String prompt, double threshold, - Map options, String judgmentApiKey, String organizationId) { - super(APIScorerType.TRACE_PROMPT_SCORER, name, prompt, threshold, options, judgmentApiKey, - organizationId); + public TracePromptScorer(String name, String prompt, double threshold, Map options, + String judgmentApiKey, String organizationId) { + super(APIScorerType.TRACE_PROMPT_SCORER, name, prompt, threshold, options, judgmentApiKey, organizationId); } public static TracePromptScorer get(String name) { @@ -29,12 +26,11 @@ public static TracePromptScorer get(String name) { } public static TracePromptScorer get(String name, String judgmentApiKey, String organizationId) { - com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorerConfig = - fetchPromptScorer(name, judgmentApiKey, organizationId); + com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorerConfig = fetchPromptScorer(name, + judgmentApiKey, organizationId); if (!Boolean.TRUE.equals(scorerConfig.getIsTrace())) { - throw new JudgmentAPIError(400, - "Scorer with name " + name + " is not a TracePromptScorer"); + throw new JudgmentAPIError(400, "Scorer with name " + name + " is not a TracePromptScorer"); } Map options = null; @@ -52,8 +48,9 @@ public static TracePromptScorer get(String name, String judgmentApiKey, String o } return new TracePromptScorer(name, scorerConfig.getPrompt(), - Optional.ofNullable(scorerConfig.getThreshold()).orElse(0.5), options, - judgmentApiKey, organizationId); + Optional.ofNullable(scorerConfig.getThreshold()) + .orElse(0.5), + options, judgmentApiKey, organizationId); } @Override diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java index f57f3a2..d894268 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java @@ -30,16 +30,16 @@ import io.opentelemetry.sdk.trace.export.SpanExporter; public abstract class BaseTracer { - public static final String TRACER_NAME = "judgeval"; + public static final String TRACER_NAME = "judgeval"; protected final TracerConfiguration configuration; - protected final JudgmentSyncClient apiClient; - protected final ISerializer serializer; - protected final ObjectMapper jacksonMapper; - protected final Optional projectId; + protected final JudgmentSyncClient apiClient; + protected final ISerializer serializer; + protected final ObjectMapper jacksonMapper; + protected final Optional projectId; - protected BaseTracer(@NotNull TracerConfiguration configuration, - @NotNull ISerializer serializer, boolean initialize) { + protected BaseTracer(@NotNull TracerConfiguration configuration, @NotNull ISerializer serializer, + boolean initialize) { this.configuration = Objects.requireNonNull(configuration, "Configuration cannot be null"); this.apiClient = new JudgmentSyncClient(configuration.apiUrl(), configuration.apiKey(), configuration.organizationId()); @@ -48,8 +48,8 @@ protected BaseTracer(@NotNull TracerConfiguration configuration, this.projectId = resolveProjectId(configuration.projectName()); this.projectId.ifPresentOrElse(id -> { }, () -> Logger.error("Failed to resolve project " + configuration.projectName() - + ", please create it first at https://app.judgmentlabs.ai/org/" - + configuration.organizationId() + "/projects. Skipping Judgment export.")); + + ", please create it first at https://app.judgmentlabs.ai/org/" + configuration.organizationId() + + "/projects. Skipping Judgment export.")); if (initialize) { initialize(); @@ -57,67 +57,78 @@ protected BaseTracer(@NotNull TracerConfiguration configuration, } /** - * Initializes the tracer with OpenTelemetry SDK configuration and span exporters. Must be - * implemented by subclasses. + * Initializes the tracer with OpenTelemetry SDK configuration and span + * exporters. Must be implemented by subclasses. */ public abstract void initialize(); /** - * Gets the span exporter for sending traces to the Judgment Labs backend. Returns a - * NoOpSpanExporter if the project ID is not resolved. + * Gets the span exporter for sending traces to the Judgment Labs backend. + * Returns a NoOpSpanExporter if the project ID is not resolved. * * @return the configured SpanExporter instance */ public SpanExporter getSpanExporter() { - return projectId.map(this::createJudgmentSpanExporter).orElseGet(() -> { - Logger.error( - "Project not resolved; cannot create exporter, returning NoOpSpanExporter"); - return new NoOpSpanExporter(); - }); + return projectId.map(this::createJudgmentSpanExporter) + .orElseGet(() -> { + Logger.error("Project not resolved; cannot create exporter, returning NoOpSpanExporter"); + return new NoOpSpanExporter(); + }); } /** * Sets the kind of the current span (e.g., "llm", "tool", "span"). * - * @param kind the span kind to set, ignored if null + * @param kind + * the span kind to set, ignored if null */ public void setSpanKind(String kind) { - Optional.ofNullable(kind).ifPresent(k -> withCurrentSpan( - span -> span.setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_SPAN_KIND, k))); + Optional.ofNullable(kind) + .ifPresent(k -> withCurrentSpan( + span -> span.setAttribute(JudgevalTraceKeys.AttributeKeys.JUDGMENT_SPAN_KIND, k))); } private static void withCurrentSpan(java.util.function.Consumer action) { - Optional.ofNullable(Span.current()).ifPresent(action); + Optional.ofNullable(Span.current()) + .ifPresent(action); } /** * Sets an attribute on the current span by serializing the object value. * - * @param key the attribute key - * @param value the attribute value, ignored if null + * @param key + * the attribute key + * @param value + * the attribute value, ignored if null */ public void setAttribute(String key, Object value) { - Optional.ofNullable(value).ifPresent(v -> setAttribute(key, v, v.getClass())); + Optional.ofNullable(value) + .ifPresent(v -> setAttribute(key, v, v.getClass())); } /** - * Sets an attribute on the current span by serializing the object value with the specified - * type. + * Sets an attribute on the current span by serializing the object value with + * the specified type. * - * @param key the attribute key - * @param value the attribute value, ignored if null - * @param type the type to use for serialization + * @param key + * the attribute key + * @param value + * the attribute value, ignored if null + * @param type + * the type to use for serialization */ public void setAttribute(String key, Object value, Type type) { - Optional.ofNullable(value).ifPresent(v -> withCurrentSpan( - span -> span.setAttribute(key, serializer.serialize(v, type)))); + Optional.ofNullable(value) + .ifPresent(v -> withCurrentSpan(span -> span.setAttribute(key, serializer.serialize(v, type)))); } /** * Sets a string attribute on the current span. * - * @param key the attribute key - * @param value the string value, ignored if null + * @param key + * the attribute key + * @param value + * the string value, ignored if null */ public void setAttribute(String key, String value) { Optional.ofNullable(value) @@ -127,8 +138,10 @@ public void setAttribute(String key, String value) { /** * Sets a long attribute on the current span. * - * @param key the attribute key - * @param value the long value + * @param key + * the attribute key + * @param value + * the long value */ public void setAttribute(String key, long value) { withCurrentSpan(span -> span.setAttribute(key, value)); @@ -137,8 +150,10 @@ public void setAttribute(String key, long value) { /** * Sets a double attribute on the current span. * - * @param key the attribute key - * @param value the double value + * @param key + * the attribute key + * @param value + * the double value */ public void setAttribute(String key, double value) { withCurrentSpan(span -> span.setAttribute(key, value)); @@ -147,21 +162,26 @@ public void setAttribute(String key, double value) { /** * Sets a boolean attribute on the current span. * - * @param key the attribute key - * @param value the boolean value + * @param key + * the attribute key + * @param value + * the boolean value */ public void setAttribute(String key, boolean value) { withCurrentSpan(span -> span.setAttribute(key, value)); } private Optional getSampledSpanContext() { - return Optional.ofNullable(Span.current()).filter(span -> span.getSpanContext().isSampled()) + return Optional.ofNullable(Span.current()) + .filter(span -> span.getSpanContext() + .isSampled()) .map(Span::getSpanContext); } private Optional getSampledSpan() { return Optional.ofNullable(Span.current()) - .filter(span -> span.getSpanContext().isSampled()); + .filter(span -> span.getSpanContext() + .isSampled()); } private boolean isEvaluationEnabled() { @@ -170,8 +190,8 @@ private boolean isEvaluationEnabled() { private void logEvaluationInfo(String method, @NotNull String traceId, @NotNull String spanId, @NotNull String scorerName) { - Logger.info(method + ": project=" + configuration.projectName() + ", traceId=" + traceId - + ", spanId=" + spanId + ", scorer=" + scorerName); + Logger.info(method + ": project=" + configuration.projectName() + ", traceId=" + traceId + ", spanId=" + + spanId + ", scorer=" + scorerName); } private void safeExecute(String operation, Runnable action) { @@ -183,11 +203,15 @@ private void safeExecute(String operation, Runnable action) { } /** - * Asynchronously evaluates a scorer against an example, associating it with the current trace. + * Asynchronously evaluates a scorer against an example, associating it with the + * current trace. * - * @param scorer the scorer to evaluate - * @param example the example to evaluate against - * @param model the model name, or null to use the default + * @param scorer + * the scorer to evaluate + * @param example + * the example to evaluate against + * @param model + * the model name, or null to use the default */ public void asyncEvaluate(@NotNull BaseScorer scorer, @NotNull Example example, String model) { safeExecute("evaluate scorer", () -> { @@ -201,8 +225,7 @@ public void asyncEvaluate(@NotNull BaseScorer scorer, @NotNull Example example, logEvaluationInfo("asyncEvaluate", traceId, spanId, scorer.getName()); - ExampleEvaluationRun evaluationRun = - createEvaluationRun(scorer, example, model, traceId, spanId); + ExampleEvaluationRun evaluationRun = createEvaluationRun(scorer, example, model, traceId, spanId); enqueueEvaluation(evaluationRun); }); }); @@ -211,19 +234,23 @@ public void asyncEvaluate(@NotNull BaseScorer scorer, @NotNull Example example, /** * Asynchronously evaluates a scorer against an example using the default model. * - * @param scorer the scorer to evaluate - * @param example the example to evaluate against + * @param scorer + * the scorer to evaluate + * @param example + * the example to evaluate against */ public void asyncEvaluate(@NotNull BaseScorer scorer, @NotNull Example example) { asyncEvaluate(scorer, example, null); } /** - * Asynchronously evaluates a scorer for the current trace, attaching the evaluation as a span - * attribute. + * Asynchronously evaluates a scorer for the current trace, attaching the + * evaluation as a span attribute. * - * @param scorer the scorer to evaluate - * @param model the model name, or null to use the default + * @param scorer + * the scorer to evaluate + * @param model + * the model name, or null to use the default */ public void asyncTraceEvaluate(@NotNull BaseScorer scorer, String model) { safeExecute("evaluate trace scorer", () -> { @@ -238,12 +265,10 @@ public void asyncTraceEvaluate(@NotNull BaseScorer scorer, String model) { logEvaluationInfo("asyncTraceEvaluate", traceId, spanId, scorer.getName()); - TraceEvaluationRun evaluationRun = - createTraceEvaluationRun(scorer, model, traceId, spanId); + TraceEvaluationRun evaluationRun = createTraceEvaluationRun(scorer, model, traceId, spanId); try { String traceEvalJson = jacksonMapper.writeValueAsString(evaluationRun); - currentSpan.setAttribute(OpenTelemetryKeys.AttributeKeys.PENDING_TRACE_EVAL, - traceEvalJson); + currentSpan.setAttribute(JudgevalTraceKeys.AttributeKeys.PENDING_TRACE_EVAL, traceEvalJson); } catch (Exception e) { Logger.error("Failed to serialize trace evaluation: " + e.getMessage()); } @@ -252,9 +277,11 @@ public void asyncTraceEvaluate(@NotNull BaseScorer scorer, String model) { } /** - * Asynchronously evaluates a scorer for the current trace using the default model. + * Asynchronously evaluates a scorer for the current trace using the default + * model. * - * @param scorer the scorer to evaluate + * @param scorer + * the scorer to evaluate */ public void asyncTraceEvaluate(@NotNull BaseScorer scorer) { asyncTraceEvaluate(scorer, null); @@ -263,7 +290,8 @@ public void asyncTraceEvaluate(@NotNull BaseScorer scorer) { /** * Sets multiple attributes on the current span from a map. * - * @param attributes the map of attribute key-value pairs, ignored if null + * @param attributes + * the map of attribute key-value pairs, ignored if null */ public void setAttributes(Map attributes) { Optional.ofNullable(attributes) @@ -294,49 +322,61 @@ public void setGeneralSpan() { /** * Sets the input attribute on the current span by serializing the object. * - * @param input the input object to set, ignored if null + * @param input + * the input object to set, ignored if null */ public void setInput(Object input) { - setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_INPUT, input); + setAttribute(JudgevalTraceKeys.AttributeKeys.JUDGMENT_INPUT, input); } /** * Sets the output attribute on the current span by serializing the object. * - * @param output the output object to set, ignored if null + * @param output + * the output object to set, ignored if null */ public void setOutput(Object output) { - setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_OUTPUT, output); + setAttribute(JudgevalTraceKeys.AttributeKeys.JUDGMENT_OUTPUT, output); } /** - * Sets the input attribute on the current span with a specific type for serialization. + * Sets the input attribute on the current span with a specific type for + * serialization. * - * @param input the input object to set, ignored if null - * @param type the type to use for serialization + * @param input + * the input object to set, ignored if null + * @param type + * the type to use for serialization */ public void setInput(Object input, Type type) { - setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_INPUT, input, type); + setAttribute(JudgevalTraceKeys.AttributeKeys.JUDGMENT_INPUT, input, type); } /** - * Sets the output attribute on the current span with a specific type for serialization. + * Sets the output attribute on the current span with a specific type for + * serialization. * - * @param output the output object to set, ignored if null - * @param type the type to use for serialization + * @param output + * the output object to set, ignored if null + * @param type + * the type to use for serialization */ public void setOutput(Object output, Type type) { - setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_OUTPUT, output, type); + setAttribute(JudgevalTraceKeys.AttributeKeys.JUDGMENT_OUTPUT, output, type); } /** - * Creates a new span, executes the runnable within its context, and ends the span. + * Creates a new span, executes the runnable within its context, and ends the + * span. * - * @param spanName the name of the span - * @param runnable the code to execute within the span context + * @param spanName + * the name of the span + * @param runnable + * the code to execute within the span context */ public void span(@NotNull String spanName, @NotNull Runnable runnable) { - Span span = getTracer().spanBuilder(spanName).startSpan(); + Span span = getTracer().spanBuilder(spanName) + .startSpan(); try (Scope scope = span.makeCurrent()) { runnable.run(); } finally { @@ -345,17 +385,22 @@ public void span(@NotNull String spanName, @NotNull Runnable runnable) { } /** - * Creates a new span, executes the callable within its context, and ends the span. + * Creates a new span, executes the callable within its context, and ends the + * span. * - * @param the return type of the callable - * @param spanName the name of the span - * @param callable the code to execute within the span context + * @param + * the return type of the callable + * @param spanName + * the name of the span + * @param callable + * the code to execute within the span context * @return the result of the callable - * @throws Exception if the callable throws an exception + * @throws Exception + * if the callable throws an exception */ - public T span(@NotNull String spanName, @NotNull java.util.concurrent.Callable callable) - throws Exception { - Span span = getTracer().spanBuilder(spanName).startSpan(); + public T span(@NotNull String spanName, @NotNull java.util.concurrent.Callable callable) throws Exception { + Span span = getTracer().spanBuilder(spanName) + .startSpan(); try (Scope scope = span.makeCurrent()) { return callable.call(); } finally { @@ -369,18 +414,23 @@ public T span(@NotNull String spanName, @NotNull java.util.concurrent.Callab * @return the configured Tracer instance */ public Tracer getTracer() { - return GlobalOpenTelemetry.get().getTracer(TRACER_NAME); + return GlobalOpenTelemetry.get() + .getTracer(TRACER_NAME); } /** - * Creates and returns a new span with the given name. The span must be ended manually by - * calling {@link Span#end()}. + * Creates and returns a new span with the given name. The span must be ended + * manually by calling {@link Span#end()}. * - * @param spanName the name of the span + * @param spanName + * the name of the span * @return the newly created span */ public static Span span(@NotNull String spanName) { - return GlobalOpenTelemetry.get().getTracer(TRACER_NAME).spanBuilder(spanName).startSpan(); + return GlobalOpenTelemetry.get() + .getTracer(TRACER_NAME) + .spanBuilder(spanName) + .startSpan(); } protected Optional resolveProjectId(@NotNull String name) { @@ -388,7 +438,8 @@ protected Optional resolveProjectId(@NotNull String name) { ResolveProjectNameRequest request = new ResolveProjectNameRequest(); request.setProjectName(name); ResolveProjectNameResponse response = apiClient.projectsResolve(request); - return Optional.ofNullable(response.getProjectId()).map(Object::toString); + return Optional.ofNullable(response.getProjectId()) + .map(Object::toString); } catch (Exception e) { return Optional.empty(); } @@ -399,9 +450,12 @@ private String buildEndpoint(@NotNull String baseUrl) { } private JudgmentSpanExporter createJudgmentSpanExporter(@NotNull String projectId) { - return JudgmentSpanExporter.builder().endpoint(buildEndpoint(configuration.apiUrl())) - .apiKey(configuration.apiKey()).organizationId(configuration.organizationId()) - .projectId(projectId).build(); + return JudgmentSpanExporter.builder() + .endpoint(buildEndpoint(configuration.apiUrl())) + .apiKey(configuration.apiKey()) + .organizationId(configuration.organizationId()) + .projectId(projectId) + .build(); } private String generateRunId(String prefix, String spanId) { @@ -410,17 +464,17 @@ private String generateRunId(String prefix, String spanId) { } private String getModelName(String model) { - return Optional.ofNullable(model).orElse(Env.JUDGMENT_DEFAULT_GPT_MODEL); + return Optional.ofNullable(model) + .orElse(Env.JUDGMENT_DEFAULT_GPT_MODEL); } - private ExampleEvaluationRun createEvaluationRun(@NotNull BaseScorer scorer, - @NotNull Example example, String model, @NotNull String traceId, String spanId) { + private ExampleEvaluationRun createEvaluationRun(@NotNull BaseScorer scorer, @NotNull Example example, + String model, @NotNull String traceId, String spanId) { String runId = generateRunId("async_evaluate_", spanId); String modelName = getModelName(model); - ExampleEvaluationRun evaluationRun = new ExampleEvaluationRun(configuration.projectName(), - runId, List.of(example), List.of(scorer.getScorerConfig()), modelName, - configuration.organizationId()); + ExampleEvaluationRun evaluationRun = new ExampleEvaluationRun(configuration.projectName(), runId, + List.of(example), List.of(scorer.getScorerConfig()), modelName, configuration.organizationId()); evaluationRun.setTraceId(traceId); evaluationRun.setTraceSpanId(spanId); return evaluationRun; @@ -431,9 +485,13 @@ private TraceEvaluationRun createTraceEvaluationRun(@NotNull BaseScorer scorer, String evalName = generateRunId("async_trace_evaluate_", spanId); String modelName = getModelName(model); - return TraceEvaluationRun.builder().projectName(configuration.projectName()) - .evalName(evalName).scorer(scorer.getScorerConfig()).model(modelName) - .organizationId(configuration.organizationId()).traceAndSpanId(traceId, spanId) + return TraceEvaluationRun.builder() + .projectName(configuration.projectName()) + .evalName(evalName) + .scorer(scorer.getScorerConfig()) + .model(modelName) + .organizationId(configuration.organizationId()) + .traceAndSpanId(traceId, spanId) .build(); } @@ -450,7 +508,9 @@ protected static class GsonSerializer implements ISerializer { @Override public String serialize(Object obj) { - return Optional.ofNullable(obj).map(o -> serialize(o, o.getClass())).orElse(null); + return Optional.ofNullable(obj) + .map(o -> serialize(o, o.getClass())) + .orElse(null); } @Override @@ -459,7 +519,9 @@ public String serialize(Object obj, Type type) { return gson.toJson(obj, type); } catch (Exception e) { Logger.error("Failed to serialize object: " + e.getMessage()); - return Optional.ofNullable(obj).map(Object::toString).orElse(null); + return Optional.ofNullable(obj) + .map(Object::toString) + .orElse(null); } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/JudgevalTraceKeys.java b/src/main/java/com/judgmentlabs/judgeval/tracer/JudgevalTraceKeys.java new file mode 100644 index 0000000..65e2fd4 --- /dev/null +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/JudgevalTraceKeys.java @@ -0,0 +1,51 @@ +package com.judgmentlabs.judgeval.tracer; + +public final class JudgevalTraceKeys { + public static final class AttributeKeys { + public static final String JUDGMENT_SPAN_KIND = "judgment.span_kind"; + public static final String JUDGMENT_INPUT = "judgment.input"; + public static final String JUDGMENT_OUTPUT = "judgment.output"; + public static final String JUDGMENT_OFFLINE_MODE = "judgment.offline_mode"; + public static final String JUDGMENT_UPDATE_ID = "judgment.update_id"; + public static final String JUDGMENT_CUSTOMER_ID = "judgment.customer_id"; + public static final String JUDGMENT_AGENT_ID = "judgment.agent_id"; + public static final String JUDGMENT_PARENT_AGENT_ID = "judgment.parent_agent_id"; + public static final String JUDGMENT_AGENT_CLASS_NAME = "judgment.agent_class_name"; + public static final String JUDGMENT_AGENT_INSTANCE_NAME = "judgment.agent_instance_name"; + public static final String JUDGMENT_IS_AGENT_ENTRY_POINT = "judgment.is_agent_entry_point"; + public static final String JUDGMENT_CUMULATIVE_LLM_COST = "judgment.cumulative_llm_cost"; + public static final String JUDGMENT_STATE_BEFORE = "judgment.state_before"; + public static final String JUDGMENT_STATE_AFTER = "judgment.state_after"; + public static final String PENDING_TRACE_EVAL = "judgment.pending_trace_eval"; + + public static final String GEN_AI_PROMPT = "gen_ai.prompt"; + public static final String GEN_AI_COMPLETION = "gen_ai.completion"; + public static final String GEN_AI_REQUEST_MODEL = "gen_ai.request.model"; + public static final String GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"; + public static final String GEN_AI_SYSTEM = "gen_ai.system"; + public static final String GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"; + public static final String GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"; + public static final String GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS = "gen_ai.usage.cache_creation_input_tokens"; + public static final String GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS = "gen_ai.usage.cache_read_input_tokens"; + public static final String GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"; + public static final String GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"; + public static final String GEN_AI_RESPONSE_FINISH_REASONS = "gen_ai.response.finish_reasons"; + + private AttributeKeys() { + } + } + + public static final class ResourceKeys { + public static final String SERVICE_NAME = "service.name"; + public static final String TELEMETRY_SDK_LANGUAGE = "telemetry.sdk.language"; + public static final String TELEMETRY_SDK_NAME = "telemetry.sdk.name"; + public static final String TELEMETRY_SDK_VERSION = "telemetry.sdk.version"; + public static final String JUDGMENT_PROJECT_ID = "judgment.project_id"; + + private ResourceKeys() { + } + } + + private JudgevalTraceKeys() { + } +} diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/OpenTelemetryKeys.java b/src/main/java/com/judgmentlabs/judgeval/tracer/OpenTelemetryKeys.java deleted file mode 100644 index aade76f..0000000 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/OpenTelemetryKeys.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.judgmentlabs.judgeval.tracer; - -public final class OpenTelemetryKeys { - public static final class AttributeKeys { - public static final String JUDGMENT_SPAN_KIND = "judgment.span_kind"; - public static final String JUDGMENT_INPUT = "judgment.input"; - public static final String JUDGMENT_OUTPUT = "judgment.output"; - public static final String JUDGMENT_OFFLINE_MODE = "judgment.offline_mode"; - public static final String JUDGMENT_UPDATE_ID = "judgment.update_id"; - public static final String JUDGMENT_CUSTOMER_ID = "judgment.customer_id"; - public static final String JUDGMENT_AGENT_ID = "judgment.agent_id"; - public static final String JUDGMENT_PARENT_AGENT_ID = "judgment.parent_agent_id"; - public static final String JUDGMENT_AGENT_CLASS_NAME = "judgment.agent_class_name"; - public static final String JUDGMENT_AGENT_INSTANCE_NAME = "judgment.agent_instance_name"; - public static final String JUDGMENT_IS_AGENT_ENTRY_POINT = "judgment.is_agent_entry_point"; - public static final String JUDGMENT_CUMULATIVE_LLM_COST = "judgment.cumulative_llm_cost"; - public static final String JUDGMENT_STATE_BEFORE = "judgment.state_before"; - public static final String JUDGMENT_STATE_AFTER = "judgment.state_after"; - public static final String PENDING_TRACE_EVAL = "judgment.pending_trace_eval"; - - public static final String GEN_AI_PROMPT = "gen_ai.prompt"; - public static final String GEN_AI_COMPLETION = "gen_ai.completion"; - public static final String GEN_AI_REQUEST_MODEL = "gen_ai.request.model"; - public static final String GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"; - public static final String GEN_AI_SYSTEM = "gen_ai.system"; - public static final String GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"; - public static final String GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"; - public static final String GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS = - "gen_ai.usage.cache_creation_input_tokens"; - public static final String GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS = - "gen_ai.usage.cache_read_input_tokens"; - public static final String GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"; - public static final String GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"; - public static final String GEN_AI_RESPONSE_FINISH_REASONS = - "gen_ai.response.finish_reasons"; - - private AttributeKeys() {} - } - - public static final class ResourceKeys { - public static final String SERVICE_NAME = "service.name"; - public static final String TELEMETRY_SDK_LANGUAGE = "telemetry.sdk.language"; - public static final String TELEMETRY_SDK_NAME = "telemetry.sdk.name"; - public static final String TELEMETRY_SDK_VERSION = "telemetry.sdk.version"; - public static final String JUDGMENT_PROJECT_ID = "judgment.project_id"; - - private ResourceKeys() {} - } - - private OpenTelemetryKeys() {} -} diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java index 06a8887..36da85a 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java @@ -25,8 +25,7 @@ */ public final class Tracer extends BaseTracer { - private Tracer(TracerConfiguration configuration, ISerializer serializer, - boolean shouldInitialize) { + private Tracer(TracerConfiguration configuration, ISerializer serializer, boolean shouldInitialize) { super(configuration, serializer, shouldInitialize); } @@ -42,54 +41,65 @@ public static TracerBuilder builder() { /** * Creates a Tracer with default configuration for the given project name. * - * @param projectName the name of the project + * @param projectName + * the name of the project * @return a new Tracer instance with default configuration */ public static Tracer createDefault(String projectName) { - return builder().configuration(TracerConfiguration.createDefault(projectName)).build(); + return builder().configuration(TracerConfiguration.createDefault(projectName)) + .build(); } /** * Creates a Tracer with the provided configuration. * - * @param configuration the tracer configuration + * @param configuration + * the tracer configuration * @return a new Tracer instance with the given configuration */ public static Tracer createWithConfiguration(TracerConfiguration configuration) { - return builder().configuration(configuration).build(); + return builder().configuration(configuration) + .build(); } /** - * Initializes the OpenTelemetry SDK with batch span processor and registers it globally. + * Initializes the OpenTelemetry SDK with batch span processor and registers it + * globally. */ @Override public void initialize() { SpanExporter spanExporter = getSpanExporter(); var resource = Resource.getDefault() - .merge(Resource.create( - Attributes.builder().put("service.name", configuration.projectName()) - .put("telemetry.sdk.name", TRACER_NAME) - .put("telemetry.sdk.version", Version.getVersion()).build())); - - SdkTracerProvider tracerProvider = SdkTracerProvider.builder().setResource(resource) - .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build()).build(); - - OpenTelemetry openTelemetry = - OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build(); + .merge(Resource.create(Attributes.builder() + .put("service.name", configuration.projectName()) + .put("telemetry.sdk.name", TRACER_NAME) + .put("telemetry.sdk.version", Version.getVersion()) + .build())); + + SdkTracerProvider tracerProvider = SdkTracerProvider.builder() + .setResource(resource) + .addSpanProcessor(BatchSpanProcessor.builder(spanExporter) + .build()) + .build(); + + OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .build(); GlobalOpenTelemetry.set(openTelemetry); } public static final class TracerBuilder { private TracerConfiguration configuration; - private ISerializer serializer = new GsonSerializer(); - private boolean initialize = false; + private ISerializer serializer = new GsonSerializer(); + private boolean initialize = false; /** * Sets the tracer configuration. * - * @param configuration the configuration to use + * @param configuration + * the configuration to use * @return this builder for method chaining */ public TracerBuilder configuration(@NotNull TracerConfiguration configuration) { @@ -100,7 +110,8 @@ public TracerBuilder configuration(@NotNull TracerConfiguration configuration) { /** * Sets the serializer to use for attribute serialization. * - * @param serializer the serializer to use + * @param serializer + * the serializer to use * @return this builder for method chaining */ public TracerBuilder serializer(@NotNull ISerializer serializer) { @@ -111,7 +122,8 @@ public TracerBuilder serializer(@NotNull ISerializer serializer) { /** * Sets whether to automatically initialize the tracer after construction. * - * @param initialize true to initialize automatically, false otherwise + * @param initialize + * true to initialize automatically, false otherwise * @return this builder for method chaining */ public TracerBuilder initialize(boolean initialize) { @@ -123,7 +135,8 @@ public TracerBuilder initialize(boolean initialize) { * Builds a new Tracer instance with the configured settings. * * @return a new Tracer instance - * @throws IllegalArgumentException if configuration is not set + * @throws IllegalArgumentException + * if configuration is not set */ public Tracer build() { return Optional.ofNullable(configuration) diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java b/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java index 43cb7c4..31b3bbf 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java @@ -5,15 +5,21 @@ import org.jetbrains.annotations.NotNull; /** - * Configuration for the Judgment Tracer that controls how tracing and evaluation behave. + * Configuration for the Judgment Tracer that controls how tracing and + * evaluation behave. *

- * This class encapsulates all configuration parameters needed to initialize a {@link Tracer}. + * This class encapsulates all configuration parameters needed to initialize a + * {@link Tracer}. *

* Example usage: * * 

{@code
- * TracerConfiguration config = TracerConfiguration.builder().projectName("my-project")
- *         .apiKey("your-api-key").organizationId("your-org-id").enableEvaluation(true).build();
+ * TracerConfiguration config = TracerConfiguration.builder()
+ *         .projectName("my-project")
+ *         .apiKey("your-api-key")
+ *         .organizationId("your-org-id")
+ *         .enableEvaluation(true)
+ *         .build();
  *
  * Tracer tracer = Tracer.createWithConfiguration(config);
  * }
@@ -21,17 +27,17 @@ * @see Tracer */ public final class TracerConfiguration { - private final String projectName; - private final String apiKey; - private final String organizationId; - private final String apiUrl; + private final String projectName; + private final String apiKey; + private final String organizationId; + private final String apiUrl; private final boolean enableEvaluation; private TracerConfiguration(@NotNull Builder builder) { - this.projectName = Optional.ofNullable(builder.projectName).map(String::trim) + this.projectName = Optional.ofNullable(builder.projectName) + .map(String::trim) .filter(name -> !name.isEmpty()) - .orElseThrow( - () -> new IllegalArgumentException("Project name cannot be null or empty")); + .orElseThrow(() -> new IllegalArgumentException("Project name cannot be null or empty")); this.apiKey = Optional.ofNullable(builder.apiKey) .orElseThrow(() -> new IllegalArgumentException("API key cannot be null")); this.organizationId = Optional.ofNullable(builder.organizationId) @@ -76,12 +82,15 @@ public static Builder builder() { *
  • Evaluation: enabled * * - * @param projectName the name of the project + * @param projectName + * the name of the project * @return a new TracerConfiguration with default values - * @throws IllegalArgumentException if project name is null or empty + * @throws IllegalArgumentException + * if project name is null or empty */ public static TracerConfiguration createDefault(String projectName) { - return builder().projectName(projectName).build(); + return builder().projectName(projectName) + .build(); } /** @@ -90,16 +99,20 @@ public static TracerConfiguration createDefault(String projectName) { * Example usage: * * 
    {@code
-     * TracerConfiguration config = TracerConfiguration.builder().projectName("my-project")
-     *         .apiKey("custom-api-key").organizationId("custom-org-id")
-     *         .apiUrl("https://custom-api.judgmentlabs.ai").enableEvaluation(false).build();
+     * TracerConfiguration config = TracerConfiguration.builder()
+     *         .projectName("my-project")
+     *         .apiKey("custom-api-key")
+     *         .organizationId("custom-org-id")
+     *         .apiUrl("https://custom-api.judgmentlabs.ai")
+     *         .enableEvaluation(false)
+     *         .build();
      * }
    */ public static final class Builder { - private String projectName; - private String apiKey = com.judgmentlabs.judgeval.Env.JUDGMENT_API_KEY; - private String organizationId = com.judgmentlabs.judgeval.Env.JUDGMENT_ORG_ID; - private String apiUrl = com.judgmentlabs.judgeval.Env.JUDGMENT_API_URL; + private String projectName; + private String apiKey = com.judgmentlabs.judgeval.Env.JUDGMENT_API_KEY; + private String organizationId = com.judgmentlabs.judgeval.Env.JUDGMENT_ORG_ID; + private String apiUrl = com.judgmentlabs.judgeval.Env.JUDGMENT_API_URL; private boolean enableEvaluation = true; public Builder projectName(@NotNull String projectName) { diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java index 5abfbe6..8cf7a10 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java @@ -13,10 +13,11 @@ import io.opentelemetry.sdk.trace.export.SpanExporter; /** - * SpanExporter implementation that sends spans to Judgment Labs with project identification. + * SpanExporter implementation that sends spans to Judgment Labs with project + * identification. *

    - * This exporter wraps the OTLP HTTP exporter and adds Judgment Labs specific headers and project - * identification to all exported spans. + * This exporter wraps the OTLP HTTP exporter and adds Judgment Labs specific + * headers and project identification to all exported spans. */ public class JudgmentSpanExporter implements SpanExporter { private final SpanExporter delegate; @@ -24,20 +25,27 @@ public class JudgmentSpanExporter implements SpanExporter { /** * Creates a new JudgmentSpanExporter with the specified configuration. * - * @param endpoint the OTLP endpoint URL - * @param apiKey the API key for authentication - * @param organizationId the organization ID - * @param projectId the project ID (must not be null or empty) - * @throws IllegalArgumentException if projectId is null or empty + * @param endpoint + * the OTLP endpoint URL + * @param apiKey + * the API key for authentication + * @param organizationId + * the organization ID + * @param projectId + * the project ID (must not be null or empty) + * @throws IllegalArgumentException + * if projectId is null or empty */ - protected JudgmentSpanExporter(@NotNull String endpoint, @NotNull String apiKey, - @NotNull String organizationId, @NotNull String projectId) { + protected JudgmentSpanExporter(@NotNull String endpoint, @NotNull String apiKey, @NotNull String organizationId, + @NotNull String projectId) { if (projectId.isEmpty()) { throw new IllegalArgumentException("projectId is required for JudgmentSpanExporter"); } - this.delegate = OtlpHttpSpanExporter.builder().setEndpoint(endpoint) + this.delegate = OtlpHttpSpanExporter.builder() + .setEndpoint(endpoint) .addHeader("Authorization", "Bearer " + apiKey) - .addHeader("X-Organization-Id", organizationId).addHeader("X-Project-Id", projectId) + .addHeader("X-Organization-Id", organizationId) + .addHeader("X-Project-Id", projectId) .build(); } @@ -53,7 +61,8 @@ public static Builder builder() { /** * Exports the collection of spans to the Judgment Labs backend. * - * @param spans the collection of spans to export + * @param spans + * the collection of spans to export * @return a CompletableResultCode representing the export operation status */ @Override @@ -91,12 +100,14 @@ public static final class Builder { private String organizationId; private String projectId; - private Builder() {} + private Builder() { + } /** * Sets the OTLP endpoint URL. * - * @param endpoint the endpoint URL + * @param endpoint + * the endpoint URL * @return this builder for method chaining */ public Builder endpoint(@NotNull String endpoint) { @@ -107,7 +118,8 @@ public Builder endpoint(@NotNull String endpoint) { /** * Sets the API key for authentication. * - * @param apiKey the API key + * @param apiKey + * the API key * @return this builder for method chaining */ public Builder apiKey(@NotNull String apiKey) { @@ -118,7 +130,8 @@ public Builder apiKey(@NotNull String apiKey) { /** * Sets the organization ID. * - * @param organizationId the organization ID + * @param organizationId + * the organization ID * @return this builder for method chaining */ public Builder organizationId(@NotNull String organizationId) { @@ -129,7 +142,8 @@ public Builder organizationId(@NotNull String organizationId) { /** * Sets the project ID. * - * @param projectId the project ID + * @param projectId + * the project ID * @return this builder for method chaining */ public Builder projectId(@NotNull String projectId) { @@ -141,28 +155,28 @@ public Builder projectId(@NotNull String projectId) { * Builds a new JudgmentSpanExporter instance with the configured settings. * * @return a new JudgmentSpanExporter instance - * @throws IllegalArgumentException if any required field is null or empty + * @throws IllegalArgumentException + * if any required field is null or empty */ public JudgmentSpanExporter build() { - String validEndpoint = - Optional.ofNullable(endpoint).map(String::trim).filter(e -> !e.isEmpty()) - .orElseThrow( - () -> new IllegalArgumentException("Endpoint is required")); - String validApiKey = - Optional.ofNullable(apiKey).map(String::trim).filter(key -> !key.isEmpty()) - .orElseThrow(() -> new IllegalArgumentException("API key is required")); - String validOrganizationId = - Optional.ofNullable(organizationId).map(String::trim) - .filter(id -> !id.isEmpty()) - .orElseThrow(() -> new IllegalArgumentException( - "Organization ID is required")); - String validProjectId = - Optional.ofNullable(projectId).map(String::trim).filter(id -> !id.isEmpty()) - .orElseThrow( - () -> new IllegalArgumentException("Project ID is required")); - - return new JudgmentSpanExporter(validEndpoint, validApiKey, validOrganizationId, - validProjectId); + String validEndpoint = Optional.ofNullable(endpoint) + .map(String::trim) + .filter(e -> !e.isEmpty()) + .orElseThrow(() -> new IllegalArgumentException("Endpoint is required")); + String validApiKey = Optional.ofNullable(apiKey) + .map(String::trim) + .filter(key -> !key.isEmpty()) + .orElseThrow(() -> new IllegalArgumentException("API key is required")); + String validOrganizationId = Optional.ofNullable(organizationId) + .map(String::trim) + .filter(id -> !id.isEmpty()) + .orElseThrow(() -> new IllegalArgumentException("Organization ID is required")); + String validProjectId = Optional.ofNullable(projectId) + .map(String::trim) + .filter(id -> !id.isEmpty()) + .orElseThrow(() -> new IllegalArgumentException("Project ID is required")); + + return new JudgmentSpanExporter(validEndpoint, validApiKey, validOrganizationId, validProjectId); } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/NoOpSpanExporter.java b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/NoOpSpanExporter.java index ad5757f..8689d1c 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/NoOpSpanExporter.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/NoOpSpanExporter.java @@ -7,14 +7,15 @@ import io.opentelemetry.sdk.trace.export.SpanExporter; /** - * A no-op implementation of SpanExporter that discards all spans. Used as a fallback when project - * resolution fails or when spans should not be exported. + * A no-op implementation of SpanExporter that discards all spans. Used as a + * fallback when project resolution fails or when spans should not be exported. */ public class NoOpSpanExporter implements SpanExporter { /** * Discards the collection of spans without exporting. * - * @param spans the collection of spans (ignored) + * @param spans + * the collection of spans (ignored) * @return a successful CompletableResultCode */ @Override diff --git a/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java b/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java index 22a4a5b..ff2796d 100644 --- a/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java +++ b/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java @@ -8,18 +8,21 @@ import com.judgmentlabs.judgeval.Env; public class Logger { - private static final String RESET = "\033[0m"; - private static final String RED = "\033[31m"; - private static final String YELLOW = "\033[33m"; - private static final String GRAY = "\033[90m"; + private static final String RESET = "\033[0m"; + private static final String RED = "\033[31m"; + private static final String YELLOW = "\033[33m"; + private static final String GRAY = "\033[90m"; - private static final DateTimeFormatter DATE_FORMATTER = - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); public enum Level { - DEBUG(0, GRAY), INFO(1, GRAY), WARNING(2, YELLOW), ERROR(3, RED), CRITICAL(4, RED); + DEBUG(0, GRAY), + INFO(1, GRAY), + WARNING(2, YELLOW), + ERROR(3, RED), + CRITICAL(4, RED); - private final int value; + private final int value; private final String color; Level(int value, String color) { @@ -36,10 +39,10 @@ public String getColor() { } } - private static final AtomicBoolean initialized = new AtomicBoolean(false); - private static Level currentLevel = Level.WARNING; - private static boolean useColor = true; - private static PrintStream output = System.out; + private static final AtomicBoolean initialized = new AtomicBoolean(false); + private static Level currentLevel = Level.WARNING; + private static boolean useColor = true; + private static PrintStream output = System.out; private static void initialize() { if (initialized.compareAndSet(false, true)) { @@ -96,9 +99,9 @@ private static void log(Level level, String message) { return; } - String timestamp = LocalDateTime.now().format(DATE_FORMATTER); - String formattedMessage = - String.format("%s - judgeval - %s - %s", timestamp, level.name(), message); + String timestamp = LocalDateTime.now() + .format(DATE_FORMATTER); + String formattedMessage = String.format("%s - judgeval - %s - %s", timestamp, level.name(), message); if (useColor) { formattedMessage = level.getColor() + formattedMessage + RESET; From 3ee2c797e326dbecaa32f162dd2d3f0dd75d14e7 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 29 Oct 2025 16:05:37 -0700 Subject: [PATCH 05/11] feat: prompt scorer caching --- .../prompt_scorer/BasePromptScorer.java | 53 +++++++++++++++++-- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java index f44c7f9..01f1860 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java @@ -2,7 +2,9 @@ import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import com.judgmentlabs.judgeval.Env; import com.judgmentlabs.judgeval.data.APIScorerType; @@ -15,10 +17,12 @@ import com.judgmentlabs.judgeval.scorers.APIScorer; public abstract class BasePromptScorer extends APIScorer { - protected String prompt; - protected Map options; - protected String judgmentApiKey; - protected String organizationId; + private static final Map cache = new ConcurrentHashMap<>(); + + protected String prompt; + protected Map options; + protected String judgmentApiKey; + protected String organizationId; protected BasePromptScorer(APIScorerType scoreType, String name, String prompt, double threshold, Map options, String judgmentApiKey, String organizationId) { @@ -45,6 +49,12 @@ public static boolean scorerExists(String name, String judgmentApiKey, String or public static com.judgmentlabs.judgeval.internal.api.models.PromptScorer fetchPromptScorer(String name, String judgmentApiKey, String organizationId) { + CacheKey key = new CacheKey(name, judgmentApiKey, organizationId); + com.judgmentlabs.judgeval.internal.api.models.PromptScorer cached = cache.get(key); + if (cached != null) { + return cached; + } + try { JudgmentSyncClient client = new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); FetchPromptScorersRequest request = new FetchPromptScorersRequest(); @@ -52,12 +62,17 @@ public static com.judgmentlabs.judgeval.internal.api.models.PromptScorer fetchPr FetchPromptScorersResponse response = client.fetchScorers(request); - return Optional.ofNullable(response) + com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorer = Optional.ofNullable(response) .map(FetchPromptScorersResponse::getScorers) .filter(scorers -> scorers != null && !scorers.isEmpty()) .map(scorers -> scorers.get(0)) .orElseThrow( () -> new JudgmentAPIError(404, "Failed to fetch prompt scorer '" + name + "': not found")); + + cache.put(key, scorer); + return scorer; + } catch (JudgmentAPIError e) { + throw e; } catch (Exception e) { throw new JudgmentAPIError(500, "Failed to fetch prompt scorer '" + name + "': " + e.getMessage()); } @@ -92,6 +107,34 @@ public Map getConfig() { protected abstract boolean isTrace(); + private static final class CacheKey { + private final String name; + private final String apiKey; + private final String organizationId; + + CacheKey(String name, String apiKey, String organizationId) { + this.name = name; + this.apiKey = apiKey; + this.organizationId = organizationId; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; + CacheKey that = (CacheKey) obj; + return Objects.equals(name, that.name) && Objects.equals(apiKey, that.apiKey) + && Objects.equals(organizationId, that.organizationId); + } + + @Override + public int hashCode() { + return Objects.hash(name, apiKey, organizationId); + } + } + @Override public String toString() { return "PromptScorer(name=" + getName() + ", prompt=" + prompt + ", threshold=" + getThreshold() From 93b8a0d08c9421a495a380cf9d8193fc93813923 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 29 Oct 2025 16:42:21 -0700 Subject: [PATCH 06/11] remove annotations --- pom.xml | 6 --- scripts/generate_client.py | 20 ++++------ .../internal/api/JudgmentAsyncClient.java | 19 ++++------ .../internal/api/JudgmentSyncClient.java | 19 ++++------ .../internal/api/models/BaseScorer.java | 7 +--- .../internal/api/models/EvalResults.java | 11 ++---- .../internal/api/models/EvalResultsFetch.java | 11 ++---- .../judgeval/internal/api/models/Example.java | 1 - .../api/models/ExampleEvaluationRun.java | 15 +++----- .../api/models/FetchPromptScorersRequest.java | 1 - .../models/FetchPromptScorersResponse.java | 7 +--- .../internal/api/models/OtelTraceSpan.java | 23 +++++------ .../internal/api/models/PromptScorer.java | 27 ++++++------- .../api/models/ResolveProjectNameRequest.java | 7 +--- .../models/ResolveProjectNameResponse.java | 7 +--- .../api/models/SavePromptScorerRequest.java | 15 +++----- .../api/models/SavePromptScorerResponse.java | 7 +--- .../internal/api/models/ScorerConfig.java | 7 +--- .../internal/api/models/ScorerData.java | 15 +++----- .../api/models/ScorerExistsRequest.java | 7 +--- .../api/models/ScorerExistsResponse.java | 7 +--- .../internal/api/models/ScoringResult.java | 11 ++---- .../api/models/TraceEvaluationRun.java | 15 +++----- .../judgeval/tracer/BaseTracer.java | 38 +++++++++---------- .../judgmentlabs/judgeval/tracer/Tracer.java | 6 +-- .../judgeval/tracer/TracerConfiguration.java | 12 +++--- .../exporters/JudgmentSpanExporter.java | 13 +++---- 27 files changed, 127 insertions(+), 207 deletions(-) diff --git a/pom.xml b/pom.xml index 9ae9091..6de0db4 100644 --- a/pom.xml +++ b/pom.xml @@ -111,12 +111,6 @@ gson 2.10.1 - - org.jetbrains - annotations - 24.1.0 - provided - diff --git a/scripts/generate_client.py b/scripts/generate_client.py index 51afec6..54d9bc1 100755 --- a/scripts/generate_client.py +++ b/scripts/generate_client.py @@ -208,9 +208,6 @@ def generate_model_class(className: str, schema: Dict[str, Any]) -> str: "import java.util.Objects;", ] - if has_required: - lines.append("import org.jetbrains.annotations.NotNull;") - lines.extend(["", f"public class {className} {{"]) fields = [] @@ -225,10 +222,10 @@ def generate_model_class(className: str, schema: Dict[str, Any]) -> str: camel_case_name = to_camel_case(field_name) is_required = field_name in required_fields - field_lines = [f' @JsonProperty("{field_name}")'] - if is_required: - field_lines.append(" @NotNull") - field_lines.append(f" private {java_type} {camel_case_name};") + field_lines = [ + f' @JsonProperty("{field_name}")', + f" private {java_type} {camel_case_name};" + ] fields.extend(field_lines) @@ -240,7 +237,7 @@ def generate_model_class(className: str, schema: Dict[str, Any]) -> str: ] ) - setter_param = f"@NotNull {java_type} {camel_case_name}" if is_required else f"{java_type} {camel_case_name}" + setter_param = f"{java_type} {camel_case_name}" setters.extend( [ f" public void set{to_class_name(camel_case_name)}({setter_param}) {{", @@ -317,10 +314,10 @@ def generate_method_signature( for param in query_params: if param["required"]: - params.append(f"@NotNull String {param['name']}") + params.append(f"String {param['name']}") if request_type: - params.append(f"@NotNull {request_type} payload") + params.append(f"{request_type} payload") for param in query_params: if not param["required"]: @@ -438,7 +435,6 @@ def generate_client_class( "import java.util.Map;", "import java.util.Objects;", "import java.util.Optional;", - "import org.jetbrains.annotations.NotNull;", "import com.judgmentlabs.judgeval.internal.api.models.*;", ] @@ -454,7 +450,7 @@ def generate_client_class( " private final String apiKey;", " private final String organizationId;", "", - f" public {className}(@NotNull String baseUrl, @NotNull String apiKey, @NotNull String organizationId) {{", + f" public {className}(String baseUrl, String apiKey, String organizationId) {{", " this.baseUrl = Objects.requireNonNull(baseUrl, \"Base URL cannot be null\");", " this.apiKey = Objects.requireNonNull(apiKey, \"API key cannot be null\");", " this.organizationId = Objects.requireNonNull(organizationId, \"Organization ID cannot be null\");", diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java index f7c4b1c..f6fcd59 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java @@ -9,8 +9,6 @@ import java.util.Objects; import java.util.concurrent.CompletableFuture; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.judgmentlabs.judgeval.internal.api.models.*; @@ -22,7 +20,7 @@ public class JudgmentAsyncClient { private final String apiKey; private final String organizationId; - public JudgmentAsyncClient(@NotNull String baseUrl, @NotNull String apiKey, @NotNull String organizationId) { + public JudgmentAsyncClient(String baseUrl, String apiKey, String organizationId) { this.baseUrl = Objects.requireNonNull(baseUrl, "Base URL cannot be null"); this.apiKey = Objects.requireNonNull(apiKey, "API key cannot be null"); this.organizationId = Objects.requireNonNull(organizationId, "Organization ID cannot be null"); @@ -66,7 +64,7 @@ private T handleResponse(HttpResponse response) { } } - public CompletableFuture addToRunEvalQueue(@NotNull ExampleEvaluationRun payload) { + public CompletableFuture addToRunEvalQueue(ExampleEvaluationRun payload) { String url = buildUrl("/add_to_run_eval_queue/"); String jsonPayload; try { @@ -83,7 +81,7 @@ public CompletableFuture addToRunEvalQueue(@NotNull ExampleEvaluationRun .thenApply(this::handleResponse); } - public CompletableFuture logEvalResults(@NotNull EvalResults payload) { + public CompletableFuture logEvalResults(EvalResults payload) { String url = buildUrl("/log_eval_results/"); String jsonPayload; try { @@ -100,7 +98,7 @@ public CompletableFuture logEvalResults(@NotNull EvalResults payload) { .thenApply(this::handleResponse); } - public CompletableFuture fetchExperimentRun(@NotNull EvalResultsFetch payload) { + public CompletableFuture fetchExperimentRun(EvalResultsFetch payload) { String url = buildUrl("/fetch_experiment_run/"); String jsonPayload; try { @@ -117,7 +115,7 @@ public CompletableFuture fetchExperimentRun(@NotNull EvalResultsFetch pa .thenApply(this::handleResponse); } - public CompletableFuture scorerExists(@NotNull ScorerExistsRequest payload) { + public CompletableFuture scorerExists(ScorerExistsRequest payload) { String url = buildUrl("/scorer_exists/"); String jsonPayload; try { @@ -134,7 +132,7 @@ public CompletableFuture scorerExists(@NotNull ScorerExist .thenApply(this::handleResponse); } - public CompletableFuture saveScorer(@NotNull SavePromptScorerRequest payload) { + public CompletableFuture saveScorer(SavePromptScorerRequest payload) { String url = buildUrl("/save_scorer/"); String jsonPayload; try { @@ -151,7 +149,7 @@ public CompletableFuture saveScorer(@NotNull SavePromp .thenApply(this::handleResponse); } - public CompletableFuture fetchScorers(@NotNull FetchPromptScorersRequest payload) { + public CompletableFuture fetchScorers(FetchPromptScorersRequest payload) { String url = buildUrl("/fetch_scorers/"); String jsonPayload; try { @@ -168,7 +166,7 @@ public CompletableFuture fetchScorers(@NotNull Fetch .thenApply(this::handleResponse); } - public CompletableFuture projectsResolve(@NotNull ResolveProjectNameRequest payload) { + public CompletableFuture projectsResolve(ResolveProjectNameRequest payload) { String url = buildUrl("/projects/resolve/"); String jsonPayload; try { @@ -184,5 +182,4 @@ public CompletableFuture projectsResolve(@NotNull Re return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } - } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java index 0b33b8b..9579f7c 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java @@ -9,8 +9,6 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.judgmentlabs.judgeval.internal.api.models.EvalResults; @@ -32,7 +30,7 @@ public class JudgmentSyncClient { private final String apiKey; private final String organizationId; - public JudgmentSyncClient(@NotNull String baseUrl, @NotNull String apiKey, @NotNull String organizationId) { + public JudgmentSyncClient(String baseUrl, String apiKey, String organizationId) { this.baseUrl = Objects.requireNonNull(baseUrl, "Base URL cannot be null"); this.apiKey = Objects.requireNonNull(apiKey, "API key cannot be null"); this.organizationId = Objects.requireNonNull(organizationId, "Organization ID cannot be null"); @@ -76,7 +74,7 @@ private T handleResponse(HttpResponse response) throws IOException { } } - public Object addToRunEvalQueue(@NotNull ExampleEvaluationRun payload) throws IOException, InterruptedException { + public Object addToRunEvalQueue(ExampleEvaluationRun payload) throws IOException, InterruptedException { String url = buildUrl("/add_to_run_eval_queue/"); String jsonPayload = mapper.writeValueAsString(payload); HttpRequest request = HttpRequest.newBuilder() @@ -88,7 +86,7 @@ public Object addToRunEvalQueue(@NotNull ExampleEvaluationRun payload) throws IO return handleResponse(response); } - public Object logEvalResults(@NotNull EvalResults payload) throws IOException, InterruptedException { + public Object logEvalResults(EvalResults payload) throws IOException, InterruptedException { String url = buildUrl("/log_eval_results/"); String jsonPayload = mapper.writeValueAsString(payload); HttpRequest request = HttpRequest.newBuilder() @@ -100,7 +98,7 @@ public Object logEvalResults(@NotNull EvalResults payload) throws IOException, I return handleResponse(response); } - public Object fetchExperimentRun(@NotNull EvalResultsFetch payload) throws IOException, InterruptedException { + public Object fetchExperimentRun(EvalResultsFetch payload) throws IOException, InterruptedException { String url = buildUrl("/fetch_experiment_run/"); String jsonPayload = mapper.writeValueAsString(payload); HttpRequest request = HttpRequest.newBuilder() @@ -112,7 +110,7 @@ public Object fetchExperimentRun(@NotNull EvalResultsFetch payload) throws IOExc return handleResponse(response); } - public ScorerExistsResponse scorerExists(@NotNull ScorerExistsRequest payload) + public ScorerExistsResponse scorerExists(ScorerExistsRequest payload) throws IOException, InterruptedException { String url = buildUrl("/scorer_exists/"); String jsonPayload = mapper.writeValueAsString(payload); @@ -125,7 +123,7 @@ public ScorerExistsResponse scorerExists(@NotNull ScorerExistsRequest payload) return mapper.readValue(response.body(), ScorerExistsResponse.class); } - public SavePromptScorerResponse saveScorer(@NotNull SavePromptScorerRequest payload) + public SavePromptScorerResponse saveScorer(SavePromptScorerRequest payload) throws IOException, InterruptedException { String url = buildUrl("/save_scorer/"); String jsonPayload = mapper.writeValueAsString(payload); @@ -138,7 +136,7 @@ public SavePromptScorerResponse saveScorer(@NotNull SavePromptScorerRequest payl return mapper.readValue(response.body(), SavePromptScorerResponse.class); } - public FetchPromptScorersResponse fetchScorers(@NotNull FetchPromptScorersRequest payload) + public FetchPromptScorersResponse fetchScorers(FetchPromptScorersRequest payload) throws IOException, InterruptedException { String url = buildUrl("/fetch_scorers/"); String jsonPayload = mapper.writeValueAsString(payload); @@ -151,7 +149,7 @@ public FetchPromptScorersResponse fetchScorers(@NotNull FetchPromptScorersReques return mapper.readValue(response.body(), FetchPromptScorersResponse.class); } - public ResolveProjectNameResponse projectsResolve(@NotNull ResolveProjectNameRequest payload) + public ResolveProjectNameResponse projectsResolve(ResolveProjectNameRequest payload) throws IOException, InterruptedException { String url = buildUrl("/projects/resolve/"); String jsonPayload = mapper.writeValueAsString(payload); @@ -163,5 +161,4 @@ public ResolveProjectNameResponse projectsResolve(@NotNull ResolveProjectNameReq HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return mapper.readValue(response.body(), ResolveProjectNameResponse.class); } - } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java index 688c427..abdb4f1 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java @@ -4,15 +4,13 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class BaseScorer { @JsonProperty("score_type") - @NotNull + private String scoreType; @JsonProperty("threshold") private Double threshold; @@ -44,7 +42,6 @@ public class BaseScorer { private String user; @JsonProperty("server_hosted") private Boolean serverHosted; - private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -121,7 +118,7 @@ public Boolean getServerHosted() { return serverHosted; } - public void setScoreType(@NotNull String scoreType) { + public void setScoreType(String scoreType) { this.scoreType = scoreType; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java index 79116c5..bc7fc54 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java @@ -5,20 +5,17 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class EvalResults { @JsonProperty("results") - @NotNull + private List results; @JsonProperty("run") - @NotNull - private Object run; + private Object run; private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -39,11 +36,11 @@ public Object getRun() { return run; } - public void setResults(@NotNull List results) { + public void setResults(List results) { this.results = results; } - public void setRun(@NotNull Object run) { + public void setRun(Object run) { this.run = run; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java index f49f04b..0390748 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java @@ -4,20 +4,17 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class EvalResultsFetch { @JsonProperty("experiment_run_id") - @NotNull + private String experimentRunId; @JsonProperty("project_name") - @NotNull - private String projectName; + private String projectName; private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -38,11 +35,11 @@ public String getProjectName() { return projectName; } - public void setExperimentRunId(@NotNull String experimentRunId) { + public void setExperimentRunId(String experimentRunId) { this.experimentRunId = experimentRunId; } - public void setProjectName(@NotNull String projectName) { + public void setProjectName(String projectName) { this.projectName = projectName; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java index c9e82c9..4af1f35 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java @@ -15,7 +15,6 @@ public class Example { private String createdAt; @JsonProperty("name") private String name; - private Map additionalProperties = new HashMap<>(); @JsonAnyGetter diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java index 19be0d9..6446d5e 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java @@ -5,8 +5,6 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; @@ -15,10 +13,10 @@ public class ExampleEvaluationRun { @JsonProperty("id") private String id; @JsonProperty("project_name") - @NotNull + private String projectName; @JsonProperty("eval_name") - @NotNull + private String evalName; @JsonProperty("custom_scorers") private List customScorers; @@ -29,13 +27,12 @@ public class ExampleEvaluationRun { @JsonProperty("created_at") private String createdAt; @JsonProperty("examples") - @NotNull + private List examples; @JsonProperty("trace_span_id") private String traceSpanId; @JsonProperty("trace_id") private String traceId; - private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -92,11 +89,11 @@ public void setId(String id) { this.id = id; } - public void setProjectName(@NotNull String projectName) { + public void setProjectName(String projectName) { this.projectName = projectName; } - public void setEvalName(@NotNull String evalName) { + public void setEvalName(String evalName) { this.evalName = evalName; } @@ -116,7 +113,7 @@ public void setCreatedAt(String createdAt) { this.createdAt = createdAt; } - public void setExamples(@NotNull List examples) { + public void setExamples(List examples) { this.examples = examples; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java index c650a72..fc43367 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java @@ -14,7 +14,6 @@ public class FetchPromptScorersRequest { private List names; @JsonProperty("is_trace") private Boolean isTrace; - private Map additionalProperties = new HashMap<>(); @JsonAnyGetter diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java index f4f4061..0950dde 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java @@ -5,17 +5,14 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class FetchPromptScorersResponse { @JsonProperty("scorers") - @NotNull - private List scorers; + private List scorers; private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -32,7 +29,7 @@ public List getScorers() { return scorers; } - public void setScorers(@NotNull List scorers) { + public void setScorers(List scorers) { this.scorers = scorers; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java index a4c0075..142c843 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java @@ -5,29 +5,27 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class OtelTraceSpan { @JsonProperty("organization_id") - @NotNull + private String organizationId; @JsonProperty("project_id") private String projectId; @JsonProperty("user_id") - @NotNull + private String userId; @JsonProperty("timestamp") - @NotNull + private String timestamp; @JsonProperty("trace_id") - @NotNull + private String traceId; @JsonProperty("span_id") - @NotNull + private String spanId; @JsonProperty("parent_span_id") private String parentSpanId; @@ -53,7 +51,6 @@ public class OtelTraceSpan { private List events; @JsonProperty("links") private List links; - private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -138,7 +135,7 @@ public List getLinks() { return links; } - public void setOrganizationId(@NotNull String organizationId) { + public void setOrganizationId(String organizationId) { this.organizationId = organizationId; } @@ -146,19 +143,19 @@ public void setProjectId(String projectId) { this.projectId = projectId; } - public void setUserId(@NotNull String userId) { + public void setUserId(String userId) { this.userId = userId; } - public void setTimestamp(@NotNull String timestamp) { + public void setTimestamp(String timestamp) { this.timestamp = timestamp; } - public void setTraceId(@NotNull String traceId) { + public void setTraceId(String traceId) { this.traceId = traceId; } - public void setSpanId(@NotNull String spanId) { + public void setSpanId(String spanId) { this.spanId = spanId; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java index 874f779..8419bc4 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java @@ -4,30 +4,28 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class PromptScorer { @JsonProperty("id") - @NotNull + private String id; @JsonProperty("user_id") - @NotNull + private String userId; @JsonProperty("organization_id") - @NotNull + private String organizationId; @JsonProperty("name") - @NotNull + private String name; @JsonProperty("prompt") - @NotNull + private String prompt; @JsonProperty("threshold") - @NotNull + private Double threshold; @JsonProperty("model") private String model; @@ -43,7 +41,6 @@ public class PromptScorer { private Boolean isTrace; @JsonProperty("is_bucket_rubric") private Boolean isBucketRubric; - private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -108,27 +105,27 @@ public Boolean getIsBucketRubric() { return isBucketRubric; } - public void setId(@NotNull String id) { + public void setId(String id) { this.id = id; } - public void setUserId(@NotNull String userId) { + public void setUserId(String userId) { this.userId = userId; } - public void setOrganizationId(@NotNull String organizationId) { + public void setOrganizationId(String organizationId) { this.organizationId = organizationId; } - public void setName(@NotNull String name) { + public void setName(String name) { this.name = name; } - public void setPrompt(@NotNull String prompt) { + public void setPrompt(String prompt) { this.prompt = prompt; } - public void setThreshold(@NotNull Double threshold) { + public void setThreshold(Double threshold) { this.threshold = threshold; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java index 631ba16..d95f327 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java @@ -4,17 +4,14 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class ResolveProjectNameRequest { @JsonProperty("project_name") - @NotNull - private String projectName; + private String projectName; private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -31,7 +28,7 @@ public String getProjectName() { return projectName; } - public void setProjectName(@NotNull String projectName) { + public void setProjectName(String projectName) { this.projectName = projectName; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java index a3da9dc..e18ccdd 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java @@ -4,17 +4,14 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class ResolveProjectNameResponse { @JsonProperty("project_id") - @NotNull - private String projectId; + private String projectId; private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -31,7 +28,7 @@ public String getProjectId() { return projectId; } - public void setProjectId(@NotNull String projectId) { + public void setProjectId(String projectId) { this.projectId = projectId; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java index c6d48b0..ecd3df8 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java @@ -4,21 +4,19 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class SavePromptScorerRequest { @JsonProperty("name") - @NotNull + private String name; @JsonProperty("prompt") - @NotNull + private String prompt; @JsonProperty("threshold") - @NotNull + private Double threshold; @JsonProperty("model") private String model; @@ -28,7 +26,6 @@ public class SavePromptScorerRequest { private Object options; @JsonProperty("description") private String description; - private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -69,15 +66,15 @@ public String getDescription() { return description; } - public void setName(@NotNull String name) { + public void setName(String name) { this.name = name; } - public void setPrompt(@NotNull String prompt) { + public void setPrompt(String prompt) { this.prompt = prompt; } - public void setThreshold(@NotNull Double threshold) { + public void setThreshold(Double threshold) { this.threshold = threshold; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java index 3db78c7..4f7ff7c 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java @@ -4,17 +4,14 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class SavePromptScorerResponse { @JsonProperty("scorer_response") - @NotNull - private PromptScorer scorerResponse; + private PromptScorer scorerResponse; private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -31,7 +28,7 @@ public PromptScorer getScorerResponse() { return scorerResponse; } - public void setScorerResponse(@NotNull PromptScorer scorerResponse) { + public void setScorerResponse(PromptScorer scorerResponse) { this.scorerResponse = scorerResponse; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java index 12c4cb6..2c193a9 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java @@ -5,15 +5,13 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class ScorerConfig { @JsonProperty("score_type") - @NotNull + private String scoreType; @JsonProperty("name") private String name; @@ -27,7 +25,6 @@ public class ScorerConfig { private List requiredParams; @JsonProperty("kwargs") private Object kwargs; - private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -68,7 +65,7 @@ public Object getKwargs() { return kwargs; } - public void setScoreType(@NotNull String scoreType) { + public void setScoreType(String scoreType) { this.scoreType = scoreType; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java index b28d8c6..7b3d96d 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java @@ -4,8 +4,6 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; @@ -14,13 +12,13 @@ public class ScorerData { @JsonProperty("id") private String id; @JsonProperty("name") - @NotNull + private String name; @JsonProperty("threshold") - @NotNull + private Double threshold; @JsonProperty("success") - @NotNull + private Boolean success; @JsonProperty("score") private Double score; @@ -34,7 +32,6 @@ public class ScorerData { private String error; @JsonProperty("additional_metadata") private Object additionalMetadata; - private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -91,15 +88,15 @@ public void setId(String id) { this.id = id; } - public void setName(@NotNull String name) { + public void setName(String name) { this.name = name; } - public void setThreshold(@NotNull Double threshold) { + public void setThreshold(Double threshold) { this.threshold = threshold; } - public void setSuccess(@NotNull Boolean success) { + public void setSuccess(Boolean success) { this.success = success; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java index 9d21493..c34fbb6 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java @@ -4,17 +4,14 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class ScorerExistsRequest { @JsonProperty("name") - @NotNull - private String name; + private String name; private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -31,7 +28,7 @@ public String getName() { return name; } - public void setName(@NotNull String name) { + public void setName(String name) { this.name = name; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java index 5a528fb..5716b37 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java @@ -4,17 +4,14 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class ScorerExistsResponse { @JsonProperty("exists") - @NotNull - private Boolean exists; + private Boolean exists; private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -31,7 +28,7 @@ public Boolean getExists() { return exists; } - public void setExists(@NotNull Boolean exists) { + public void setExists(Boolean exists) { this.exists = exists; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java index 97d5a46..b291fe7 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java @@ -5,18 +5,16 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; public class ScoringResult { @JsonProperty("success") - @NotNull + private Boolean success; @JsonProperty("scorers_data") - @NotNull + private List scorersData; @JsonProperty("name") private String name; @@ -28,7 +26,6 @@ public class ScoringResult { private Double runDuration; @JsonProperty("evaluation_cost") private Double evaluationCost; - private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -69,11 +66,11 @@ public Double getEvaluationCost() { return evaluationCost; } - public void setSuccess(@NotNull Boolean success) { + public void setSuccess(Boolean success) { this.success = success; } - public void setScorersData(@NotNull List scorersData) { + public void setScorersData(List scorersData) { this.scorersData = scorersData; } diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java index 7fcd56c..f205bf6 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java @@ -5,8 +5,6 @@ import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonProperty; @@ -15,10 +13,10 @@ public class TraceEvaluationRun { @JsonProperty("id") private String id; @JsonProperty("project_name") - @NotNull + private String projectName; @JsonProperty("eval_name") - @NotNull + private String evalName; @JsonProperty("custom_scorers") private List customScorers; @@ -29,13 +27,12 @@ public class TraceEvaluationRun { @JsonProperty("created_at") private String createdAt; @JsonProperty("trace_and_span_ids") - @NotNull + private List> traceAndSpanIds; @JsonProperty("is_offline") private Boolean isOffline; @JsonProperty("is_bucket_run") private Boolean isBucketRun; - private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -92,11 +89,11 @@ public void setId(String id) { this.id = id; } - public void setProjectName(@NotNull String projectName) { + public void setProjectName(String projectName) { this.projectName = projectName; } - public void setEvalName(@NotNull String evalName) { + public void setEvalName(String evalName) { this.evalName = evalName; } @@ -116,7 +113,7 @@ public void setCreatedAt(String createdAt) { this.createdAt = createdAt; } - public void setTraceAndSpanIds(@NotNull List> traceAndSpanIds) { + public void setTraceAndSpanIds(List> traceAndSpanIds) { this.traceAndSpanIds = traceAndSpanIds; } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java index d894268..60b1231 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java @@ -6,8 +6,6 @@ import java.util.Objects; import java.util.Optional; -import org.jetbrains.annotations.NotNull; - import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.judgmentlabs.judgeval.Env; @@ -38,8 +36,7 @@ public abstract class BaseTracer { protected final ObjectMapper jacksonMapper; protected final Optional projectId; - protected BaseTracer(@NotNull TracerConfiguration configuration, @NotNull ISerializer serializer, - boolean initialize) { + protected BaseTracer(TracerConfiguration configuration, ISerializer serializer, boolean initialize) { this.configuration = Objects.requireNonNull(configuration, "Configuration cannot be null"); this.apiClient = new JudgmentSyncClient(configuration.apiUrl(), configuration.apiKey(), configuration.organizationId()); @@ -188,8 +185,7 @@ private boolean isEvaluationEnabled() { return configuration.enableEvaluation(); } - private void logEvaluationInfo(String method, @NotNull String traceId, @NotNull String spanId, - @NotNull String scorerName) { + private void logEvaluationInfo(String method, String traceId, String spanId, String scorerName) { Logger.info(method + ": project=" + configuration.projectName() + ", traceId=" + traceId + ", spanId=" + spanId + ", scorer=" + scorerName); } @@ -213,7 +209,7 @@ private void safeExecute(String operation, Runnable action) { * @param model * the model name, or null to use the default */ - public void asyncEvaluate(@NotNull BaseScorer scorer, @NotNull Example example, String model) { + public void asyncEvaluate(BaseScorer scorer, Example example, String model) { safeExecute("evaluate scorer", () -> { if (!isEvaluationEnabled()) { return; @@ -239,7 +235,7 @@ public void asyncEvaluate(@NotNull BaseScorer scorer, @NotNull Example example, * @param example * the example to evaluate against */ - public void asyncEvaluate(@NotNull BaseScorer scorer, @NotNull Example example) { + public void asyncEvaluate(BaseScorer scorer, Example example) { asyncEvaluate(scorer, example, null); } @@ -252,7 +248,7 @@ public void asyncEvaluate(@NotNull BaseScorer scorer, @NotNull Example example) * @param model * the model name, or null to use the default */ - public void asyncTraceEvaluate(@NotNull BaseScorer scorer, String model) { + public void asyncTraceEvaluate(BaseScorer scorer, String model) { safeExecute("evaluate trace scorer", () -> { if (!isEvaluationEnabled()) { return; @@ -283,7 +279,7 @@ public void asyncTraceEvaluate(@NotNull BaseScorer scorer, String model) { * @param scorer * the scorer to evaluate */ - public void asyncTraceEvaluate(@NotNull BaseScorer scorer) { + public void asyncTraceEvaluate(BaseScorer scorer) { asyncTraceEvaluate(scorer, null); } @@ -374,7 +370,7 @@ public void setOutput(Object output, Type type) { * @param runnable * the code to execute within the span context */ - public void span(@NotNull String spanName, @NotNull Runnable runnable) { + public void span(String spanName, Runnable runnable) { Span span = getTracer().spanBuilder(spanName) .startSpan(); try (Scope scope = span.makeCurrent()) { @@ -398,7 +394,7 @@ public void span(@NotNull String spanName, @NotNull Runnable runnable) { * @throws Exception * if the callable throws an exception */ - public T span(@NotNull String spanName, @NotNull java.util.concurrent.Callable callable) throws Exception { + public T span(String spanName, java.util.concurrent.Callable callable) throws Exception { Span span = getTracer().spanBuilder(spanName) .startSpan(); try (Scope scope = span.makeCurrent()) { @@ -426,14 +422,14 @@ public Tracer getTracer() { * the name of the span * @return the newly created span */ - public static Span span(@NotNull String spanName) { + public static Span span(String spanName) { return GlobalOpenTelemetry.get() .getTracer(TRACER_NAME) .spanBuilder(spanName) .startSpan(); } - protected Optional resolveProjectId(@NotNull String name) { + protected Optional resolveProjectId(String name) { try { ResolveProjectNameRequest request = new ResolveProjectNameRequest(); request.setProjectName(name); @@ -445,11 +441,11 @@ protected Optional resolveProjectId(@NotNull String name) { } } - private String buildEndpoint(@NotNull String baseUrl) { + private String buildEndpoint(String baseUrl) { return baseUrl.endsWith("/") ? baseUrl + "otel/v1/traces" : baseUrl + "/otel/v1/traces"; } - private JudgmentSpanExporter createJudgmentSpanExporter(@NotNull String projectId) { + private JudgmentSpanExporter createJudgmentSpanExporter(String projectId) { return JudgmentSpanExporter.builder() .endpoint(buildEndpoint(configuration.apiUrl())) .apiKey(configuration.apiKey()) @@ -468,8 +464,8 @@ private String getModelName(String model) { .orElse(Env.JUDGMENT_DEFAULT_GPT_MODEL); } - private ExampleEvaluationRun createEvaluationRun(@NotNull BaseScorer scorer, @NotNull Example example, - String model, @NotNull String traceId, String spanId) { + private ExampleEvaluationRun createEvaluationRun(BaseScorer scorer, Example example, String model, String traceId, + String spanId) { String runId = generateRunId("async_evaluate_", spanId); String modelName = getModelName(model); @@ -480,8 +476,8 @@ private ExampleEvaluationRun createEvaluationRun(@NotNull BaseScorer scorer, @No return evaluationRun; } - private TraceEvaluationRun createTraceEvaluationRun(@NotNull BaseScorer scorer, String model, - @NotNull String traceId, String spanId) { + private TraceEvaluationRun createTraceEvaluationRun(BaseScorer scorer, String model, String traceId, + String spanId) { String evalName = generateRunId("async_trace_evaluate_", spanId); String modelName = getModelName(model); @@ -495,7 +491,7 @@ private TraceEvaluationRun createTraceEvaluationRun(@NotNull BaseScorer scorer, .build(); } - private void enqueueEvaluation(@NotNull ExampleEvaluationRun evaluationRun) { + private void enqueueEvaluation(ExampleEvaluationRun evaluationRun) { try { apiClient.addToRunEvalQueue(evaluationRun); } catch (Exception e) { diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java index 36da85a..db8defe 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java @@ -2,8 +2,6 @@ import java.util.Optional; -import org.jetbrains.annotations.NotNull; - import com.judgmentlabs.judgeval.Version; import io.opentelemetry.api.GlobalOpenTelemetry; @@ -102,7 +100,7 @@ public static final class TracerBuilder { * the configuration to use * @return this builder for method chaining */ - public TracerBuilder configuration(@NotNull TracerConfiguration configuration) { + public TracerBuilder configuration(TracerConfiguration configuration) { this.configuration = configuration; return this; } @@ -114,7 +112,7 @@ public TracerBuilder configuration(@NotNull TracerConfiguration configuration) { * the serializer to use * @return this builder for method chaining */ - public TracerBuilder serializer(@NotNull ISerializer serializer) { + public TracerBuilder serializer(ISerializer serializer) { this.serializer = serializer; return this; } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java b/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java index 31b3bbf..bd296ea 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java @@ -2,8 +2,6 @@ import java.util.Optional; -import org.jetbrains.annotations.NotNull; - /** * Configuration for the Judgment Tracer that controls how tracing and * evaluation behave. @@ -33,7 +31,7 @@ public final class TracerConfiguration { private final String apiUrl; private final boolean enableEvaluation; - private TracerConfiguration(@NotNull Builder builder) { + private TracerConfiguration(Builder builder) { this.projectName = Optional.ofNullable(builder.projectName) .map(String::trim) .filter(name -> !name.isEmpty()) @@ -115,22 +113,22 @@ public static final class Builder { private String apiUrl = com.judgmentlabs.judgeval.Env.JUDGMENT_API_URL; private boolean enableEvaluation = true; - public Builder projectName(@NotNull String projectName) { + public Builder projectName(String projectName) { this.projectName = projectName; return this; } - public Builder apiKey(@NotNull String apiKey) { + public Builder apiKey(String apiKey) { this.apiKey = apiKey; return this; } - public Builder organizationId(@NotNull String organizationId) { + public Builder organizationId(String organizationId) { this.organizationId = organizationId; return this; } - public Builder apiUrl(@NotNull String apiUrl) { + public Builder apiUrl(String apiUrl) { this.apiUrl = apiUrl; return this; } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java index 8cf7a10..fd6bdb7 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java @@ -3,8 +3,6 @@ import java.util.Collection; import java.util.Optional; -import org.jetbrains.annotations.NotNull; - import com.judgmentlabs.judgeval.utils.Logger; import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; @@ -36,8 +34,7 @@ public class JudgmentSpanExporter implements SpanExporter { * @throws IllegalArgumentException * if projectId is null or empty */ - protected JudgmentSpanExporter(@NotNull String endpoint, @NotNull String apiKey, @NotNull String organizationId, - @NotNull String projectId) { + protected JudgmentSpanExporter(String endpoint, String apiKey, String organizationId, String projectId) { if (projectId.isEmpty()) { throw new IllegalArgumentException("projectId is required for JudgmentSpanExporter"); } @@ -110,7 +107,7 @@ private Builder() { * the endpoint URL * @return this builder for method chaining */ - public Builder endpoint(@NotNull String endpoint) { + public Builder endpoint(String endpoint) { this.endpoint = endpoint; return this; } @@ -122,7 +119,7 @@ public Builder endpoint(@NotNull String endpoint) { * the API key * @return this builder for method chaining */ - public Builder apiKey(@NotNull String apiKey) { + public Builder apiKey(String apiKey) { this.apiKey = apiKey; return this; } @@ -134,7 +131,7 @@ public Builder apiKey(@NotNull String apiKey) { * the organization ID * @return this builder for method chaining */ - public Builder organizationId(@NotNull String organizationId) { + public Builder organizationId(String organizationId) { this.organizationId = organizationId; return this; } @@ -146,7 +143,7 @@ public Builder organizationId(@NotNull String organizationId) { * the project ID * @return this builder for method chaining */ - public Builder projectId(@NotNull String projectId) { + public Builder projectId(String projectId) { this.projectId = projectId; return this; } From 9cd5569db39a1243992234254d3136e426d2960d Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 29 Oct 2025 16:42:52 -0700 Subject: [PATCH 07/11] chore: regen client --- .../internal/api/JudgmentAsyncClient.java | 16 +++++++--- .../internal/api/JudgmentSyncClient.java | 31 ++++++++----------- .../internal/api/models/BaseScorer.java | 4 +-- .../internal/api/models/EvalResults.java | 5 ++- .../internal/api/models/EvalResultsFetch.java | 8 ++--- .../judgeval/internal/api/models/Example.java | 3 +- .../api/models/ExampleEvaluationRun.java | 6 ++-- .../api/models/FetchPromptScorersRequest.java | 3 +- .../models/FetchPromptScorersResponse.java | 4 +-- .../internal/api/models/OtelTraceSpan.java | 11 ++----- .../internal/api/models/PromptScorer.java | 13 +++----- .../api/models/ResolveProjectNameRequest.java | 4 +-- .../models/ResolveProjectNameResponse.java | 4 +-- .../api/models/SavePromptScorerRequest.java | 6 ++-- .../api/models/SavePromptScorerResponse.java | 4 +-- .../internal/api/models/ScorerConfig.java | 4 +-- .../internal/api/models/ScorerData.java | 10 +++--- .../api/models/ScorerExistsRequest.java | 4 +-- .../api/models/ScorerExistsResponse.java | 7 ++--- .../internal/api/models/ScoringResult.java | 5 ++- .../api/models/TraceEvaluationRun.java | 6 ++-- 21 files changed, 69 insertions(+), 89 deletions(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java index f6fcd59..0ad218d 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java @@ -34,8 +34,7 @@ private String buildUrl(String path, Map queryParams) { StringBuilder url = new StringBuilder(baseUrl).append(path); if (!queryParams.isEmpty()) { url.append("?"); - String queryString = queryParams.entrySet() - .stream() + String queryString = queryParams.entrySet().stream() .map(entry -> entry.getKey() + "=" + entry.getValue()) .reduce("", (a, b) -> a.isEmpty() ? b : a + "&" + b); url.append(queryString); @@ -48,8 +47,14 @@ private String buildUrl(String path) { } private String[] buildHeaders() { - return new String[] { "Content-Type", "application/json", "Authorization", "Bearer " + apiKey, - "X-Organization-Id", organizationId }; + return new String[] { + "Content-Type", + "application/json", + "Authorization", + "Bearer " + apiKey, + "X-Organization-Id", + organizationId + }; } private T handleResponse(HttpResponse response) { @@ -182,4 +187,5 @@ public CompletableFuture projectsResolve(ResolveProj return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } -} + +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java index 9579f7c..d0f1f5d 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java @@ -11,17 +11,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.judgmentlabs.judgeval.internal.api.models.EvalResults; -import com.judgmentlabs.judgeval.internal.api.models.EvalResultsFetch; -import com.judgmentlabs.judgeval.internal.api.models.ExampleEvaluationRun; -import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersRequest; -import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersResponse; -import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameRequest; -import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameResponse; -import com.judgmentlabs.judgeval.internal.api.models.SavePromptScorerRequest; -import com.judgmentlabs.judgeval.internal.api.models.SavePromptScorerResponse; -import com.judgmentlabs.judgeval.internal.api.models.ScorerExistsRequest; -import com.judgmentlabs.judgeval.internal.api.models.ScorerExistsResponse; +import com.judgmentlabs.judgeval.internal.api.models.*; public class JudgmentSyncClient { private final HttpClient client; @@ -44,8 +34,7 @@ private String buildUrl(String path, Map queryParams) { StringBuilder url = new StringBuilder(baseUrl).append(path); if (!queryParams.isEmpty()) { url.append("?"); - String queryString = queryParams.entrySet() - .stream() + String queryString = queryParams.entrySet().stream() .map(entry -> entry.getKey() + "=" + entry.getValue()) .reduce("", (a, b) -> a.isEmpty() ? b : a + "&" + b); url.append(queryString); @@ -58,8 +47,14 @@ private String buildUrl(String path) { } private String[] buildHeaders() { - return new String[] { "Content-Type", "application/json", "Authorization", "Bearer " + apiKey, - "X-Organization-Id", organizationId }; + return new String[] { + "Content-Type", + "application/json", + "Authorization", + "Bearer " + apiKey, + "X-Organization-Id", + organizationId + }; } private T handleResponse(HttpResponse response) throws IOException { @@ -110,8 +105,7 @@ public Object fetchExperimentRun(EvalResultsFetch payload) throws IOException, I return handleResponse(response); } - public ScorerExistsResponse scorerExists(ScorerExistsRequest payload) - throws IOException, InterruptedException { + public ScorerExistsResponse scorerExists(ScorerExistsRequest payload) throws IOException, InterruptedException { String url = buildUrl("/scorer_exists/"); String jsonPayload = mapper.writeValueAsString(payload); HttpRequest request = HttpRequest.newBuilder() @@ -161,4 +155,5 @@ public ResolveProjectNameResponse projectsResolve(ResolveProjectNameRequest payl HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return mapper.readValue(response.body(), ResolveProjectNameResponse.class); } -} + +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java index abdb4f1..235a640 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java @@ -10,7 +10,6 @@ public class BaseScorer { @JsonProperty("score_type") - private String scoreType; @JsonProperty("threshold") private Double threshold; @@ -42,6 +41,7 @@ public class BaseScorer { private String user; @JsonProperty("server_hosted") private Boolean serverHosted; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -206,4 +206,4 @@ public int hashCode() { success, model, modelClient, strictMode, error, additionalMetadata, user, serverHosted, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java index bc7fc54..9d7494b 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java @@ -11,11 +11,10 @@ public class EvalResults { @JsonProperty("results") - private List results; @JsonProperty("run") - private Object run; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -59,4 +58,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(results, run, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java index 0390748..9e77977 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java @@ -10,11 +10,10 @@ public class EvalResultsFetch { @JsonProperty("experiment_run_id") - private String experimentRunId; @JsonProperty("project_name") - private String projectName; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -50,8 +49,7 @@ public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) return false; EvalResultsFetch other = (EvalResultsFetch) obj; - return Objects.equals(experimentRunId, other.experimentRunId) - && Objects.equals(projectName, other.projectName) + return Objects.equals(experimentRunId, other.experimentRunId) && Objects.equals(projectName, other.projectName) && Objects.equals(additionalProperties, other.additionalProperties); } @@ -59,4 +57,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(experimentRunId, projectName, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java index 4af1f35..92b8c63 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java @@ -15,6 +15,7 @@ public class Example { private String createdAt; @JsonProperty("name") private String name; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -66,4 +67,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(exampleId, createdAt, name, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java index 6446d5e..847e4f2 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java @@ -13,10 +13,8 @@ public class ExampleEvaluationRun { @JsonProperty("id") private String id; @JsonProperty("project_name") - private String projectName; @JsonProperty("eval_name") - private String evalName; @JsonProperty("custom_scorers") private List customScorers; @@ -27,12 +25,12 @@ public class ExampleEvaluationRun { @JsonProperty("created_at") private String createdAt; @JsonProperty("examples") - private List examples; @JsonProperty("trace_span_id") private String traceSpanId; @JsonProperty("trace_id") private String traceId; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -145,4 +143,4 @@ public int hashCode() { return Objects.hash(id, projectName, evalName, customScorers, judgmentScorers, model, createdAt, examples, traceSpanId, traceId, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java index fc43367..31f4c24 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java @@ -14,6 +14,7 @@ public class FetchPromptScorersRequest { private List names; @JsonProperty("is_trace") private Boolean isTrace; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -57,4 +58,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(names, isTrace, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java index 0950dde..d6521eb 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java @@ -11,8 +11,8 @@ public class FetchPromptScorersResponse { @JsonProperty("scorers") - private List scorers; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -48,4 +48,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(scorers, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java index 142c843..949082d 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java @@ -11,21 +11,16 @@ public class OtelTraceSpan { @JsonProperty("organization_id") - private String organizationId; @JsonProperty("project_id") private String projectId; @JsonProperty("user_id") - private String userId; @JsonProperty("timestamp") - private String timestamp; @JsonProperty("trace_id") - private String traceId; @JsonProperty("span_id") - private String spanId; @JsonProperty("parent_span_id") private String parentSpanId; @@ -51,6 +46,7 @@ public class OtelTraceSpan { private List events; @JsonProperty("links") private List links; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -231,7 +227,6 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(organizationId, projectId, userId, timestamp, traceId, spanId, parentSpanId, traceState, spanName, spanKind, serviceName, resourceAttributes, spanAttributes, duration, statusCode, - statusMessage, - events, links, Objects.hashCode(additionalProperties)); + statusMessage, events, links, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java index 8419bc4..3c0662c 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java @@ -10,22 +10,16 @@ public class PromptScorer { @JsonProperty("id") - private String id; @JsonProperty("user_id") - private String userId; @JsonProperty("organization_id") - private String organizationId; @JsonProperty("name") - private String name; @JsonProperty("prompt") - private String prompt; @JsonProperty("threshold") - private Double threshold; @JsonProperty("model") private String model; @@ -41,6 +35,7 @@ public class PromptScorer { private Boolean isTrace; @JsonProperty("is_bucket_rubric") private Boolean isBucketRubric; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -176,7 +171,7 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return Objects.hash(id, userId, organizationId, name, prompt, threshold, model, options, description, - createdAt, updatedAt, isTrace, isBucketRubric, Objects.hashCode(additionalProperties)); + return Objects.hash(id, userId, organizationId, name, prompt, threshold, model, options, description, createdAt, + updatedAt, isTrace, isBucketRubric, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java index d95f327..d3a2d33 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java @@ -10,8 +10,8 @@ public class ResolveProjectNameRequest { @JsonProperty("project_name") - private String projectName; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -47,4 +47,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(projectName, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java index e18ccdd..9e587ee 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java @@ -10,8 +10,8 @@ public class ResolveProjectNameResponse { @JsonProperty("project_id") - private String projectId; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -47,4 +47,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(projectId, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java index ecd3df8..c795d16 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java @@ -10,13 +10,10 @@ public class SavePromptScorerRequest { @JsonProperty("name") - private String name; @JsonProperty("prompt") - private String prompt; @JsonProperty("threshold") - private Double threshold; @JsonProperty("model") private String model; @@ -26,6 +23,7 @@ public class SavePromptScorerRequest { private Object options; @JsonProperty("description") private String description; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -113,4 +111,4 @@ public int hashCode() { return Objects.hash(name, prompt, threshold, model, isTrace, options, description, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java index 4f7ff7c..bb1f0b4 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java @@ -10,8 +10,8 @@ public class SavePromptScorerResponse { @JsonProperty("scorer_response") - private PromptScorer scorerResponse; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -47,4 +47,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(scorerResponse, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java index 2c193a9..72b095d 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java @@ -11,7 +11,6 @@ public class ScorerConfig { @JsonProperty("score_type") - private String scoreType; @JsonProperty("name") private String name; @@ -25,6 +24,7 @@ public class ScorerConfig { private List requiredParams; @JsonProperty("kwargs") private Object kwargs; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -112,4 +112,4 @@ public int hashCode() { return Objects.hash(scoreType, name, threshold, model, strictMode, requiredParams, kwargs, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java index 7b3d96d..ccca512 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java @@ -12,13 +12,10 @@ public class ScorerData { @JsonProperty("id") private String id; @JsonProperty("name") - private String name; @JsonProperty("threshold") - private Double threshold; @JsonProperty("success") - private Boolean success; @JsonProperty("score") private Double score; @@ -32,6 +29,7 @@ public class ScorerData { private String error; @JsonProperty("additional_metadata") private Object additionalMetadata; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -135,8 +133,8 @@ public boolean equals(Object obj) { && Objects.equals(threshold, other.threshold) && Objects.equals(success, other.success) && Objects.equals(score, other.score) && Objects.equals(reason, other.reason) && Objects.equals(strictMode, other.strictMode) - && Objects.equals(evaluationModel, other.evaluationModel) - && Objects.equals(error, other.error) && Objects.equals(additionalMetadata, other.additionalMetadata) + && Objects.equals(evaluationModel, other.evaluationModel) && Objects.equals(error, other.error) + && Objects.equals(additionalMetadata, other.additionalMetadata) && Objects.equals(additionalProperties, other.additionalProperties); } @@ -145,4 +143,4 @@ public int hashCode() { return Objects.hash(id, name, threshold, success, score, reason, strictMode, evaluationModel, error, additionalMetadata, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java index c34fbb6..46e178a 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java @@ -10,8 +10,8 @@ public class ScorerExistsRequest { @JsonProperty("name") - private String name; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -46,4 +46,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(name, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java index 5716b37..29939e6 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java @@ -10,8 +10,8 @@ public class ScorerExistsResponse { @JsonProperty("exists") - private Boolean exists; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -39,12 +39,11 @@ public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) return false; ScorerExistsResponse other = (ScorerExistsResponse) obj; - return Objects.equals(exists, other.exists) - && Objects.equals(additionalProperties, other.additionalProperties); + return Objects.equals(exists, other.exists) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { return Objects.hash(exists, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java index b291fe7..1231e90 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java @@ -11,10 +11,8 @@ public class ScoringResult { @JsonProperty("success") - private Boolean success; @JsonProperty("scorers_data") - private List scorersData; @JsonProperty("name") private String name; @@ -26,6 +24,7 @@ public class ScoringResult { private Double runDuration; @JsonProperty("evaluation_cost") private Double evaluationCost; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -113,4 +112,4 @@ public int hashCode() { return Objects.hash(success, scorersData, name, dataObject, traceId, runDuration, evaluationCost, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java index f205bf6..c19a9f8 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java @@ -13,10 +13,8 @@ public class TraceEvaluationRun { @JsonProperty("id") private String id; @JsonProperty("project_name") - private String projectName; @JsonProperty("eval_name") - private String evalName; @JsonProperty("custom_scorers") private List customScorers; @@ -27,12 +25,12 @@ public class TraceEvaluationRun { @JsonProperty("created_at") private String createdAt; @JsonProperty("trace_and_span_ids") - private List> traceAndSpanIds; @JsonProperty("is_offline") private Boolean isOffline; @JsonProperty("is_bucket_run") private Boolean isBucketRun; + private Map additionalProperties = new HashMap<>(); @JsonAnyGetter @@ -145,4 +143,4 @@ public int hashCode() { return Objects.hash(id, projectName, evalName, customScorers, judgmentScorers, model, createdAt, traceAndSpanIds, isOffline, isBucketRun, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file From 409c25575a9164d6062f7890e1b7ed800613aaa7 Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 29 Oct 2025 16:47:18 -0700 Subject: [PATCH 08/11] fix: setAttribute prevent null key --- .../judgeval/tracer/BaseTracer.java | 50 ++++++++++++++----- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java index 60b1231..1d9f641 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java @@ -90,46 +90,63 @@ private static void withCurrentSpan(java.util.function.Consumer action) { .ifPresent(action); } + private static boolean isValidKey(String key) { + return key != null && !key.isEmpty(); + } + /** * Sets an attribute on the current span by serializing the object value. + * Empty strings and null are valid attribute values, but not valid keys. * * @param key * the attribute key * @param value - * the attribute value, ignored if null + * the attribute value (null and empty strings are valid) */ public void setAttribute(String key, Object value) { - Optional.ofNullable(value) - .ifPresent(v -> setAttribute(key, v, v.getClass())); + if (!isValidKey(key)) { + return; + } + if (value != null) { + setAttribute(key, value, value.getClass()); + } } /** * Sets an attribute on the current span by serializing the object value with - * the specified type. + * the specified type. Empty strings and null are valid attribute values, but + * not valid keys. * * @param key * the attribute key * @param value - * the attribute value, ignored if null + * the attribute value (null and empty strings are valid) * @param type * the type to use for serialization */ public void setAttribute(String key, Object value, Type type) { - Optional.ofNullable(value) - .ifPresent(v -> withCurrentSpan(span -> span.setAttribute(key, serializer.serialize(v, type)))); + if (!isValidKey(key)) { + return; + } + if (value != null) { + withCurrentSpan(span -> span.setAttribute(key, serializer.serialize(value, type))); + } } /** - * Sets a string attribute on the current span. + * Sets a string attribute on the current span. Empty strings and null are valid + * attribute values, but not valid keys. * * @param key * the attribute key * @param value - * the string value, ignored if null + * the string value (null and empty strings are valid) */ public void setAttribute(String key, String value) { - Optional.ofNullable(value) - .ifPresent(v -> withCurrentSpan(span -> span.setAttribute(key, v))); + if (!isValidKey(key)) { + return; + } + withCurrentSpan(span -> span.setAttribute(key, value)); } /** @@ -141,6 +158,9 @@ public void setAttribute(String key, String value) { * the long value */ public void setAttribute(String key, long value) { + if (!isValidKey(key)) { + return; + } withCurrentSpan(span -> span.setAttribute(key, value)); } @@ -153,6 +173,9 @@ public void setAttribute(String key, long value) { * the double value */ public void setAttribute(String key, double value) { + if (!isValidKey(key)) { + return; + } withCurrentSpan(span -> span.setAttribute(key, value)); } @@ -165,6 +188,9 @@ public void setAttribute(String key, double value) { * the boolean value */ public void setAttribute(String key, boolean value) { + if (!isValidKey(key)) { + return; + } withCurrentSpan(span -> span.setAttribute(key, value)); } @@ -291,7 +317,7 @@ public void asyncTraceEvaluate(BaseScorer scorer) { */ public void setAttributes(Map attributes) { Optional.ofNullable(attributes) - .ifPresent(attrs -> withCurrentSpan(span -> attrs.forEach(this::setAttribute))); + .ifPresent(attrs -> attrs.forEach(this::setAttribute)); } /** From b41e2f66c9cb9292913d2fecda02288edbd87e1b Mon Sep 17 00:00:00 2001 From: Abhishek Govindarasu Date: Wed, 29 Oct 2025 17:11:10 -0700 Subject: [PATCH 09/11] fix: record exceptins and reraise on span --- .../java/com/judgmentlabs/judgeval/tracer/BaseTracer.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java index 1d9f641..1f030e4 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java @@ -23,6 +23,7 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Scope; import io.opentelemetry.sdk.trace.export.SpanExporter; @@ -401,6 +402,9 @@ public void span(String spanName, Runnable runnable) { .startSpan(); try (Scope scope = span.makeCurrent()) { runnable.run(); + } catch (Exception e) { + span.setStatus(StatusCode.ERROR).recordException(e); + throw e; } finally { span.end(); } @@ -425,6 +429,9 @@ public T span(String spanName, java.util.concurrent.Callable callable) th .startSpan(); try (Scope scope = span.makeCurrent()) { return callable.call(); + } catch (Exception e) { + span.setStatus(StatusCode.ERROR).recordException(e); + throw e; } finally { span.end(); } From ad1acc50cb2f37a4797d488e68e9f734172e2811 Mon Sep 17 00:00:00 2001 From: Aaryan Divate Date: Wed, 29 Oct 2025 18:10:22 -0700 Subject: [PATCH 10/11] Evaluation Run Fix --- .../judgeval/data/ExampleEvaluationRun.java | 153 ---------------- .../judgeval/data/TraceEvaluationRun.java | 167 ------------------ .../judgeval/tracer/BaseTracer.java | 54 ++++-- 3 files changed, 39 insertions(+), 335 deletions(-) delete mode 100644 src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java delete mode 100644 src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java diff --git a/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java deleted file mode 100644 index 435dd89..0000000 --- a/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java +++ /dev/null @@ -1,153 +0,0 @@ -package com.judgmentlabs.judgeval.data; - -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.Objects; -import java.util.UUID; - -import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; - -public class ExampleEvaluationRun extends com.judgmentlabs.judgeval.internal.api.models.ExampleEvaluationRun { - - private String organizationId; - - public ExampleEvaluationRun() { - super(); - setId(UUID.randomUUID() - .toString()); - setCreatedAt(Instant.now() - .atOffset(ZoneOffset.UTC) - .format(DateTimeFormatter.ISO_INSTANT)); - } - - public ExampleEvaluationRun(String projectName, String evalName, List examples, - List scorers, String model, String organizationId) { - this(); - setProjectName(projectName); - setEvalName(evalName); - @SuppressWarnings("unchecked") - List internalExamples = (List) (List) examples; - setExamples(internalExamples); - setModel(model); - setOrganizationId(organizationId); - setCustomScorers(new java.util.ArrayList<>()); - setJudgmentScorers(scorers); - } - - public void setOrganizationId(String organizationId) { - this.organizationId = organizationId; - } - - public String getOrganizationId() { - return organizationId; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null || getClass() != obj.getClass()) - return false; - if (!super.equals(obj)) - return false; - ExampleEvaluationRun other = (ExampleEvaluationRun) obj; - return Objects.equals(organizationId, other.organizationId); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), organizationId); - } - - public static Builder builder() { - return new Builder(); - } - - public static Builder builder(String projectName, String evalName) { - return new Builder(projectName, evalName); - } - - public static final class Builder { - private String projectName; - private String evalName; - private List examples; - private List scorers; - private String model; - private String organizationId; - - private Builder() { - } - - private Builder(String projectName, String evalName) { - this.projectName = projectName; - this.evalName = evalName; - } - - public Builder projectName(String projectName) { - this.projectName = projectName; - return this; - } - - public Builder evalName(String evalName) { - this.evalName = evalName; - return this; - } - - public Builder examples(List examples) { - this.examples = examples; - return this; - } - - public Builder example(Example example) { - if (this.examples == null) { - this.examples = new java.util.ArrayList<>(); - } - this.examples.add(example); - return this; - } - - public Builder scorers(List scorers) { - this.scorers = scorers; - return this; - } - - public Builder scorer(ScorerConfig scorer) { - if (this.scorers == null) { - this.scorers = new java.util.ArrayList<>(); - } - this.scorers.add(scorer); - return this; - } - - public Builder model(String model) { - this.model = model; - return this; - } - - public Builder organizationId(String organizationId) { - this.organizationId = organizationId; - return this; - } - - public ExampleEvaluationRun build() { - if (projectName == null || projectName.trim() - .isEmpty()) { - throw new IllegalArgumentException("Project name is required"); - } - if (evalName == null || evalName.trim() - .isEmpty()) { - throw new IllegalArgumentException("Evaluation name is required"); - } - if (examples == null || examples.isEmpty()) { - throw new IllegalArgumentException("At least one example is required"); - } - if (scorers == null || scorers.isEmpty()) { - throw new IllegalArgumentException("At least one scorer is required"); - } - - return new ExampleEvaluationRun(projectName, evalName, examples, scorers, model, organizationId); - } - } -} diff --git a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java deleted file mode 100644 index 2687700..0000000 --- a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java +++ /dev/null @@ -1,167 +0,0 @@ -package com.judgmentlabs.judgeval.data; - -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.Objects; -import java.util.UUID; - -import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; - -public class TraceEvaluationRun extends com.judgmentlabs.judgeval.internal.api.models.TraceEvaluationRun { - - private String organizationId; - - public TraceEvaluationRun() { - super(); - setId(UUID.randomUUID() - .toString()); - setCreatedAt(Instant.now() - .atOffset(ZoneOffset.UTC) - .format(DateTimeFormatter.ISO_INSTANT)); - setIsOffline(false); - } - - public TraceEvaluationRun(String projectName, String evalName, List scorers, String model, - String organizationId, List> traceAndSpanIds) { - this(); - setProjectName(projectName); - setEvalName(evalName); - setModel(model); - setOrganizationId(organizationId); - setTraceAndSpanIds(convertTraceAndSpanIds(traceAndSpanIds)); - setCustomScorers(new java.util.ArrayList<>()); - setJudgmentScorers(scorers); - } - - private static List> convertTraceAndSpanIds(List> traceAndSpanIds) { - if (traceAndSpanIds == null || traceAndSpanIds.isEmpty()) { - throw new IllegalArgumentException("Trace and span IDs are required for trace evaluations."); - } - - List> converted = new java.util.ArrayList<>(); - for (List pair : traceAndSpanIds) { - if (pair == null || pair.size() != 2) { - throw new IllegalArgumentException("Each trace and span ID pair must contain exactly 2 elements."); - } - converted.add(List.of(pair.get(0), pair.get(1))); - } - return converted; - } - - public void setOrganizationId(String organizationId) { - this.organizationId = organizationId; - } - - public String getOrganizationId() { - return organizationId; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null || getClass() != obj.getClass()) - return false; - if (!super.equals(obj)) - return false; - TraceEvaluationRun other = (TraceEvaluationRun) obj; - return Objects.equals(organizationId, other.organizationId); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), organizationId); - } - - public static Builder builder() { - return new Builder(); - } - - public static Builder builder(String projectName, String evalName) { - return new Builder(projectName, evalName); - } - - public static final class Builder { - private String projectName; - private String evalName; - private List scorers; - private String model; - private String organizationId; - private List> traceAndSpanIds; - - private Builder() { - } - - private Builder(String projectName, String evalName) { - this.projectName = projectName; - this.evalName = evalName; - } - - public Builder projectName(String projectName) { - this.projectName = projectName; - return this; - } - - public Builder evalName(String evalName) { - this.evalName = evalName; - return this; - } - - public Builder scorers(List scorers) { - this.scorers = scorers; - return this; - } - - public Builder scorer(ScorerConfig scorer) { - if (this.scorers == null) { - this.scorers = new java.util.ArrayList<>(); - } - this.scorers.add(scorer); - return this; - } - - public Builder model(String model) { - this.model = model; - return this; - } - - public Builder organizationId(String organizationId) { - this.organizationId = organizationId; - return this; - } - - public Builder traceAndSpanIds(List> traceAndSpanIds) { - this.traceAndSpanIds = traceAndSpanIds; - return this; - } - - public Builder traceAndSpanId(String traceId, String spanId) { - if (this.traceAndSpanIds == null) { - this.traceAndSpanIds = new java.util.ArrayList<>(); - } - this.traceAndSpanIds.add(List.of(traceId, spanId)); - return this; - } - - public TraceEvaluationRun build() { - if (projectName == null || projectName.trim() - .isEmpty()) { - throw new IllegalArgumentException("Project name is required"); - } - if (evalName == null || evalName.trim() - .isEmpty()) { - throw new IllegalArgumentException("Evaluation name is required"); - } - if (scorers == null || scorers.isEmpty()) { - throw new IllegalArgumentException("At least one scorer is required"); - } - if (traceAndSpanIds == null || traceAndSpanIds.isEmpty()) { - throw new IllegalArgumentException("At least one trace and span ID pair is required"); - } - - return new TraceEvaluationRun(projectName, evalName, scorers, model, organizationId, traceAndSpanIds); - } - } -} diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java index 1f030e4..da3f660 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java @@ -10,11 +10,11 @@ import com.google.gson.Gson; import com.judgmentlabs.judgeval.Env; import com.judgmentlabs.judgeval.data.Example; -import com.judgmentlabs.judgeval.data.ExampleEvaluationRun; -import com.judgmentlabs.judgeval.data.TraceEvaluationRun; import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameRequest; import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameResponse; +import com.judgmentlabs.judgeval.internal.api.models.TraceEvaluationRun; +import com.judgmentlabs.judgeval.internal.api.models.ExampleEvaluationRun; import com.judgmentlabs.judgeval.scorers.BaseScorer; import com.judgmentlabs.judgeval.tracer.exporters.JudgmentSpanExporter; import com.judgmentlabs.judgeval.tracer.exporters.NoOpSpanExporter; @@ -28,6 +28,11 @@ import io.opentelemetry.context.Scope; import io.opentelemetry.sdk.trace.export.SpanExporter; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.UUID; + public abstract class BaseTracer { public static final String TRACER_NAME = "judgeval"; @@ -502,11 +507,21 @@ private ExampleEvaluationRun createEvaluationRun(BaseScorer scorer, Example exam String runId = generateRunId("async_evaluate_", spanId); String modelName = getModelName(model); - ExampleEvaluationRun evaluationRun = new ExampleEvaluationRun(configuration.projectName(), runId, - List.of(example), List.of(scorer.getScorerConfig()), modelName, configuration.organizationId()); - evaluationRun.setTraceId(traceId); - evaluationRun.setTraceSpanId(spanId); - return evaluationRun; + ExampleEvaluationRun exampleEvaluationRun = new ExampleEvaluationRun(); + exampleEvaluationRun.setProjectName(configuration.projectName()); + exampleEvaluationRun.setEvalName(runId); + exampleEvaluationRun.setJudgmentScorers(List.of(scorer.getScorerConfig())); + exampleEvaluationRun.setModel(modelName); + exampleEvaluationRun.setExamples(List.of(example)); + exampleEvaluationRun.setTraceId(traceId); + exampleEvaluationRun.setTraceSpanId(spanId); + exampleEvaluationRun.setCustomScorers(new java.util.ArrayList<>()); + exampleEvaluationRun.setId(UUID.randomUUID() + .toString()); + exampleEvaluationRun.setCreatedAt(Instant.now() + .atOffset(ZoneOffset.UTC) + .format(DateTimeFormatter.ISO_INSTANT)); + return exampleEvaluationRun; } private TraceEvaluationRun createTraceEvaluationRun(BaseScorer scorer, String model, String traceId, @@ -514,14 +529,23 @@ private TraceEvaluationRun createTraceEvaluationRun(BaseScorer scorer, String mo String evalName = generateRunId("async_trace_evaluate_", spanId); String modelName = getModelName(model); - return TraceEvaluationRun.builder() - .projectName(configuration.projectName()) - .evalName(evalName) - .scorer(scorer.getScorerConfig()) - .model(modelName) - .organizationId(configuration.organizationId()) - .traceAndSpanId(traceId, spanId) - .build(); + TraceEvaluationRun traceEvaluationRun = new TraceEvaluationRun(); + traceEvaluationRun.setProjectName(configuration.projectName()); + traceEvaluationRun.setEvalName(evalName); + traceEvaluationRun.setJudgmentScorers(List.of(scorer.getScorerConfig())); + traceEvaluationRun.setModel(modelName); + traceEvaluationRun.setTraceAndSpanIds(List.of(List.of(traceId, spanId))); + traceEvaluationRun.setIsOffline(false); + traceEvaluationRun.setIsBucketRun(false); + traceEvaluationRun.setCustomScorers(new java.util.ArrayList<>()); + traceEvaluationRun.setId(UUID.randomUUID() + .toString()); + traceEvaluationRun.setCreatedAt(Instant.now() + .atOffset(ZoneOffset.UTC) + .format(DateTimeFormatter.ISO_INSTANT)); + traceEvaluationRun.setIsOffline(false); + + return traceEvaluationRun; } private void enqueueEvaluation(ExampleEvaluationRun evaluationRun) { From b635941ce4d2e2e12a4126d71d47b2f3f75eaee7 Mon Sep 17 00:00:00 2001 From: Aaryan Divate Date: Wed, 29 Oct 2025 18:30:17 -0700 Subject: [PATCH 11/11] Gemini fixes --- .../judgeval/tracer/BaseTracer.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java index da3f660..ea08614 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java @@ -512,7 +512,8 @@ private ExampleEvaluationRun createEvaluationRun(BaseScorer scorer, Example exam exampleEvaluationRun.setEvalName(runId); exampleEvaluationRun.setJudgmentScorers(List.of(scorer.getScorerConfig())); exampleEvaluationRun.setModel(modelName); - exampleEvaluationRun.setExamples(List.of(example)); + com.judgmentlabs.judgeval.internal.api.models.Example internalExample = (com.judgmentlabs.judgeval.internal.api.models.Example) example; + exampleEvaluationRun.setExamples(List.of(internalExample)); exampleEvaluationRun.setTraceId(traceId); exampleEvaluationRun.setTraceSpanId(spanId); exampleEvaluationRun.setCustomScorers(new java.util.ArrayList<>()); @@ -534,7 +535,7 @@ private TraceEvaluationRun createTraceEvaluationRun(BaseScorer scorer, String mo traceEvaluationRun.setEvalName(evalName); traceEvaluationRun.setJudgmentScorers(List.of(scorer.getScorerConfig())); traceEvaluationRun.setModel(modelName); - traceEvaluationRun.setTraceAndSpanIds(List.of(List.of(traceId, spanId))); + traceEvaluationRun.setTraceAndSpanIds(convertTraceAndSpanIds(List.of(List.of(traceId, spanId)))); traceEvaluationRun.setIsOffline(false); traceEvaluationRun.setIsBucketRun(false); traceEvaluationRun.setCustomScorers(new java.util.ArrayList<>()); @@ -543,11 +544,25 @@ private TraceEvaluationRun createTraceEvaluationRun(BaseScorer scorer, String mo traceEvaluationRun.setCreatedAt(Instant.now() .atOffset(ZoneOffset.UTC) .format(DateTimeFormatter.ISO_INSTANT)); - traceEvaluationRun.setIsOffline(false); return traceEvaluationRun; } + private static List> convertTraceAndSpanIds(List> traceAndSpanIds) { + if (traceAndSpanIds == null || traceAndSpanIds.isEmpty()) { + throw new IllegalArgumentException("Trace and span IDs are required for trace evaluations."); + } + + List> converted = new java.util.ArrayList<>(); + for (List pair : traceAndSpanIds) { + if (pair == null || pair.size() != 2) { + throw new IllegalArgumentException("Each trace and span ID pair must contain exactly 2 elements."); + } + converted.add(List.of(pair.get(0), pair.get(1))); + } + return converted; + } + private void enqueueEvaluation(ExampleEvaluationRun evaluationRun) { try { apiClient.addToRunEvalQueue(evaluationRun);