diff --git a/.cursorrules b/.cursorrules new file mode 100644 index 0000000..8422f18 --- /dev/null +++ b/.cursorrules @@ -0,0 +1,227 @@ +# Judgeval Java - Internal API Patterns + +## Context + +The `com.judgmentlabs.judgeval.internal` package contains internal API code that is auto-generated from OpenAPI specifications using `scripts/generate_client.py`, but is explicitly modifiable for internal improvements. This code is NOT part of the public API and can be refactored, cleaned up, and optimized as needed. + +## Internal API Models (`internal/api/models/`) + +### Structure Pattern +- **Package**: `com.judgmentlabs.judgeval.internal.api.models` +- **Naming**: PascalCase class names matching OpenAPI schema names (e.g., `EvalResultsFetch`, `ResolveProjectNameResponse`) +- **Purpose**: Jackson-annotated POJOs for serializing/deserializing JSON requests and responses + +### Field Patterns +- **Field Declaration**: + - Private fields with camelCase naming (e.g., `experimentRunId`, `projectName`) + - Jackson `@JsonProperty` annotation mapping snake_case JSON keys to camelCase fields + - Use wrapper types (String, Integer, Boolean, List, etc.) - never primitives for nullable fields + - All fields are nullable by default (no `@NotNull` annotations) + +### Additional Properties Pattern +All model classes MUST include: +```java +private Map additionalProperties = new HashMap<>(); + +@JsonAnyGetter +public Map getAdditionalProperties() { + return additionalProperties; +} + +@JsonAnySetter +public void setAdditionalProperty(String name, Object value) { + additionalProperties.put(name, value); +} +``` +This allows unknown JSON fields to be preserved during deserialization. + +### Getter/Setter Pattern +- **Getters**: Public methods following JavaBean convention `getFieldName()` returning field type +- **Setters**: Public methods following JavaBean convention `setFieldName(Type fieldName)` returning void +- **Order**: Fields declared first, then additionalProperties, then getters, then setters + +### Equals/HashCode Pattern +- **Equals**: Must include all fields plus `additionalProperties` +- **Implementation**: Use `Objects.equals()` for all field comparisons +- **HashCode**: Use `Objects.hash()` for all fields, with `Objects.hashCode(additionalProperties)` for the Map +- **Null Safety**: Both methods handle null fields safely via `Objects.equals()` + +### Imports +Standard imports: +- `java.util.HashMap` +- `java.util.List` (if using collections) +- `java.util.Map` +- `java.util.Objects` +- `com.fasterxml.jackson.annotation.JsonAnyGetter` +- `com.fasterxml.jackson.annotation.JsonAnySetter` +- `com.fasterxml.jackson.annotation.JsonProperty` + +## Internal API Clients (`internal/api/`) + +### Client Classes +- **JudgmentSyncClient**: Synchronous HTTP client for blocking API calls +- **JudgmentAsyncClient**: Asynchronous HTTP client for non-blocking API calls +- Both share identical structure and helper methods, differing only in execution model + +### Constructor Pattern +```java +public JudgmentSyncClient(String baseUrl, String apiKey, String organizationId) { + this.baseUrl = baseUrl; + this.apiKey = apiKey; + this.organizationId = organizationId; + this.client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build(); + this.mapper = new ObjectMapper(); +} +``` + +### Helper Methods Pattern + +#### buildUrl(String path, Map queryParams) +- Builds full URL from baseUrl + path +- Appends query string from map if non-empty +- Query string format: `key1=value1&key2=value2` +- Returns complete URL string + +#### buildUrl(String path) +- Convenience overload calling `buildUrl(path, new HashMap<>())` + +#### buildHeaders() +- Validates `apiKey` and `organizationId` are not null (throws `IllegalArgumentException`) +- Returns String array with: + - `"Content-Type"`, `"application/json"` + - `"Authorization"`, `"Bearer " + apiKey` + - `"X-Organization-Id"`, `organizationId` + +#### handleResponse(HttpResponse response) +- **Sync**: `throws IOException` +- **Async**: No throws clause (unchecked exceptions) +- Checks status code >= 400, throws `RuntimeException` with status and body +- Attempts to deserialize response body using `mapper.readValue(response.body(), new TypeReference() {})` +- Catches parsing exceptions and wraps in `RuntimeException("Failed to parse response", e)` +- **Issue**: TypeReference generic type erasure makes this unreliable - should use specific class when known + +### Method Naming Pattern +- Method names derived from API path: `/fetch_experiment_run/` → `fetchExperimentRun` +- Path segments separated by underscores/forward slashes become camelCase +- HTTP method determines method signature (GET = no body, POST = has payload) + +### Sync Client Method Pattern +```java +public ReturnType methodName(RequestType payload) throws IOException, InterruptedException { + String url = buildUrl("/api/path/"); + String jsonPayload = mapper.writeValueAsString(payload); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + return handleResponse(response); // or mapper.readValue(response.body(), SpecificType.class); +} +``` + +### Async Client Method Pattern +```java +public CompletableFuture methodName(RequestType payload) { + String url = buildUrl("/api/path/"); + String jsonPayload; + try { + jsonPayload = mapper.writeValueAsString(payload); + } catch (Exception e) { + throw new RuntimeException("Failed to serialize payload", e); + } + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); + return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) + .thenApply(this::handleResponse); +} +``` + +### Return Type Patterns +- **Consistency Issue**: Some methods return `Object`, others return specific response types +- **Preferred**: Return specific response model types when known (e.g., `ScorerExistsResponse`, `ResolveProjectNameResponse`) +- **Fallback**: Use `Object` only when response schema is truly unknown/variable +- **Async**: Return type wrapped in `CompletableFuture` + +### Error Handling Standards + +#### Sync Client +- `IOException` and `InterruptedException` propagate (method signature) +- `RuntimeException` for HTTP errors (4xx/5xx) - thrown by `handleResponse` +- `RuntimeException` for deserialization errors - thrown by `handleResponse` or `mapper.readValue` + +#### Async Client +- All exceptions caught internally and wrapped in `RuntimeException` +- Serialization errors caught immediately and rethrown before async operation +- HTTP/deserialization errors thrown from `handleResponse` in CompletableFuture chain +- No checked exceptions in method signatures + +### Query Parameters Pattern +For GET requests or POST requests with query params: +```java +Map queryParams = new HashMap<>(); +queryParams.put("param_name", paramValue); +String url = buildUrl("/api/path/", queryParams); +``` + +## Code Quality Expectations + +### Documentation +- Add JavaDoc to all public methods in client classes +- Document parameters, return types, and thrown exceptions +- Explain complex business logic or non-obvious patterns + +### Null Safety +- Use `Optional` internally for nullable values when appropriate +- Use `@NotNull` annotations for parameters that must not be null (only if needed for external safety) +- Validate constructor parameters that must not be null + +### Error Messages +- Error messages should be descriptive and include context +- HTTP errors should include status code and response body +- Serialization errors should indicate the operation that failed + +### Code Organization +- Group related methods together +- Keep helper methods private +- Maintain consistent method ordering (constructors, helpers, public API methods) +- Extract common patterns into reusable helpers when duplicated across sync/async clients + +### Type Safety +- Prefer specific types over `Object` for return values +- Use proper generic types for collections +- Avoid raw types and unchecked casts + +### Consistency +- Sync and async clients should have identical API (only differing by CompletableFuture wrapper) +- Method names should be consistent across both clients +- Helper methods should behave identically in both clients + +## Refactoring Opportunities + +### Known Issues +1. **TypeReference generic erasure**: `handleResponse` uses `TypeReference()` which doesn't preserve type information. Consider passing `Class` or using Jackson's `TypeFactory`. + +2. **Return type inconsistency**: Some sync methods return `Object`, others return specific types. Standardize based on OpenAPI response schemas. + +3. **Code duplication**: Sync and async clients share ~90% of code. Consider extracting common base class or using composition. + +4. **Error handling**: Async client wraps serialization errors, sync client doesn't. Consider consistent error handling strategy. + +5. **Missing validation**: No validation of request payloads before serialization. Consider adding validation for required fields. + +6. **Missing documentation**: No JavaDoc comments on any client methods. Should document all public methods. + +### Recommended Improvements +- Extract common client logic to base class `BaseJudgmentClient` +- Standardize return types based on OpenAPI response schemas +- Add request/response validation +- Improve error handling with custom exception types +- Add comprehensive JavaDoc documentation +- Consider using builder pattern for request construction +- Add retry logic for transient failures +- Add request/response logging + diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..a55e7a1 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..c531a7e --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1761770774206 + + + + \ No newline at end of file diff --git a/.vscode/java-formatter.xml b/.vscode/java-formatter.xml new file mode 100644 index 0000000..ac58365 --- /dev/null +++ b/.vscode/java-formatter.xml @@ -0,0 +1,315 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.vscode/settings.json b/.vscode/settings.json index 4f81299..49b061d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,3 +1,4 @@ { - "java.configuration.updateBuildConfiguration": "automatic" + "java.configuration.updateBuildConfiguration": "automatic", + "java.format.settings.url": ".vscode/java-formatter.xml" } diff --git a/pom.xml b/pom.xml index 56c452e..6de0db4 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.judgmentlabs judgeval-java - 0.2.1 + 0.2.2 jar Judgeval Java Java SDK for Judgeval @@ -207,13 +207,13 @@ ${spotless.version} - - 1.19.2 - - + + 4.28 + .vscode/java-formatter.xml + - java,javax,org,com, + java,javax,org,com @@ -225,7 +225,6 @@ ${checkstyle.version} checkstyle.xml - UTF-8 true false diff --git a/scripts/generate_client.py b/scripts/generate_client.py index d043351..54d9bc1 100755 --- a/scripts/generate_client.py +++ b/scripts/generate_client.py @@ -193,19 +193,22 @@ def get_java_type(schema: Dict[str, Any]) -> str: def generate_model_class(className: str, schema: Dict[str, Any]) -> str: + required_fields = set(schema.get("required", [])) + has_required = bool(required_fields) + lines = [ "package com.judgmentlabs.judgeval.internal.api.models;", "", - "import com.fasterxml.jackson.annotation.JsonProperty;", - "import com.fasterxml.jackson.annotation.JsonAnySetter;", "import com.fasterxml.jackson.annotation.JsonAnyGetter;", + "import com.fasterxml.jackson.annotation.JsonAnySetter;", + "import com.fasterxml.jackson.annotation.JsonProperty;", + "import java.util.HashMap;", "import java.util.List;", "import java.util.Map;", - "import java.util.HashMap;", "import java.util.Objects;", - "", - f"public class {className} {{", ] + + lines.extend(["", f"public class {className} {{"]) fields = [] getters = [] @@ -217,13 +220,14 @@ def generate_model_class(className: str, schema: Dict[str, Any]) -> str: for field_name, property_schema in schema["properties"].items(): java_type = get_java_type(property_schema) camel_case_name = to_camel_case(field_name) + is_required = field_name in required_fields - fields.extend( - [ - f' @JsonProperty("{field_name}")', - f" private {java_type} {camel_case_name};", - ] - ) + field_lines = [ + f' @JsonProperty("{field_name}")', + f" private {java_type} {camel_case_name};" + ] + + fields.extend(field_lines) getters.extend( [ @@ -233,9 +237,10 @@ def generate_model_class(className: str, schema: Dict[str, Any]) -> str: ] ) + setter_param = f"{java_type} {camel_case_name}" setters.extend( [ - f" public void set{to_class_name(camel_case_name)}({java_type} {camel_case_name}) {{", + f" public void set{to_class_name(camel_case_name)}({setter_param}) {{", f" this.{camel_case_name} = {camel_case_name};", " }", ] @@ -344,12 +349,9 @@ def generate_method_body( if param["required"]: lines.append(f' queryParams.put("{param_name}", {param_name});') else: - lines.extend( - [ - f" if ({param_name} != null) {{", - f' queryParams.put("{param_name}", {param_name});', - " }", - ] + param_key = param["name"] + lines.append( + f' Optional.ofNullable({param_name}).ifPresent(v -> queryParams.put("{param_key}", v));' ) lines.append( @@ -422,8 +424,8 @@ def generate_client_class( imports = [ "package com.judgmentlabs.judgeval.internal.api;", "", - "import com.fasterxml.jackson.databind.ObjectMapper;", "import com.fasterxml.jackson.core.type.TypeReference;", + "import com.fasterxml.jackson.databind.ObjectMapper;", "import java.io.IOException;", "import java.net.URI;", "import java.net.http.HttpClient;", @@ -431,6 +433,8 @@ def generate_client_class( "import java.net.http.HttpResponse;", "import java.util.HashMap;", "import java.util.Map;", + "import java.util.Objects;", + "import java.util.Optional;", "import com.judgmentlabs.judgeval.internal.api.models.*;", ] @@ -447,9 +451,9 @@ def generate_client_class( " private final String organizationId;", "", f" public {className}(String baseUrl, String apiKey, String organizationId) {{", - " this.baseUrl = baseUrl;", - " this.apiKey = apiKey;", - " this.organizationId = organizationId;", + " this.baseUrl = Objects.requireNonNull(baseUrl, \"Base URL cannot be null\");", + " this.apiKey = Objects.requireNonNull(apiKey, \"API key cannot be null\");", + " this.organizationId = Objects.requireNonNull(organizationId, \"Organization ID cannot be null\");", " this.client = HttpClient.newBuilder()", " .version(HttpClient.Version.HTTP_1_1)", " .build();", @@ -473,9 +477,6 @@ def generate_client_class( " }", "", " private String[] buildHeaders() {", - " if (apiKey == null || organizationId == null) {", - ' throw new IllegalArgumentException("API key and organization ID cannot be null");', - " }", " return new String[] {", ' "Content-Type",', ' "application/json",', @@ -489,42 +490,39 @@ def generate_client_class( ] throws_clause = "" if is_async else " throws IOException" - lines.extend( - [ - f" private T handleResponse(HttpResponse response){throws_clause} {{", - " if (response.statusCode() >= 400) {", - ' throw new RuntimeException("HTTP Error: " + response.statusCode() + " - " + response.body());', - " }", - " try {", - " return mapper.readValue(response.body(), new TypeReference() {});", - " } catch (Exception e) {", - ' throw new RuntimeException("Failed to parse response", e);', - " }", - " }", - "", - ] - ) + lines.append(f" private T handleResponse(HttpResponse response){throws_clause} {{") + lines.append(" if (response.statusCode() >= 400) {") + lines.append(f' throw new RuntimeException("HTTP Error: " + response.statusCode() + " - " + response.body());') + lines.append(" }") + lines.append(" try {") + lines.append(" return mapper.readValue(response.body(), new TypeReference() {});") + lines.append(" } catch (Exception e) {") + lines.append(' throw new RuntimeException("Failed to parse response", e);') + lines.append(" }") + lines.append(" }") + lines.append("") for method_info in methods: - signature = generate_method_signature( - method_info["name"], - method_info["request_type"], - method_info["query_params"], - method_info["response_type"], - is_async, + lines.append( + generate_method_signature( + method_info["name"], + method_info["request_type"], + method_info["query_params"], + method_info["response_type"], + is_async, + ) ) - lines.append(signature) - - body = generate_method_body( - method_info["name"], - method_info["path"], - method_info["method"], - method_info["request_type"], - method_info["query_params"], - method_info["response_type"], - is_async, + lines.append( + generate_method_body( + method_info["name"], + method_info["path"], + method_info["method"], + method_info["request_type"], + method_info["query_params"], + method_info["response_type"], + is_async, + ) ) - lines.append(body) lines.append(" }") lines.append("") diff --git a/src/main/java/com/judgmentlabs/judgeval/Env.java b/src/main/java/com/judgmentlabs/judgeval/Env.java index e2a07e8..e3b7440 100644 --- a/src/main/java/com/judgmentlabs/judgeval/Env.java +++ b/src/main/java/com/judgmentlabs/judgeval/Env.java @@ -1,16 +1,16 @@ package com.judgmentlabs.judgeval; public final class Env { - private Env() {} + private Env() { + } - public static final String JUDGMENT_API_KEY = getEnvVar("JUDGMENT_API_KEY"); - public static final String JUDGMENT_ORG_ID = getEnvVar("JUDGMENT_ORG_ID"); - public static final String JUDGMENT_API_URL = - getEnvVar("JUDGMENT_API_URL", "https://api.judgmentlabs.ai"); - public static final String JUDGMENT_DEFAULT_GPT_MODEL = - getEnvVar("JUDGMENT_DEFAULT_GPT_MODEL", "gpt-4.1"); - public static final String JUDGMENT_NO_COLOR = getEnvVar("JUDGMENT_NO_COLOR"); - public static final String JUDGMENT_LOG_LEVEL = getEnvVar("JUDGMENT_LOG_LEVEL", "warn"); + public static final String JUDGMENT_API_KEY = getEnvVar("JUDGMENT_API_KEY"); + public static final String JUDGMENT_ORG_ID = getEnvVar("JUDGMENT_ORG_ID"); + public static final String JUDGMENT_API_URL = getEnvVar("JUDGMENT_API_URL", + "https://api.judgmentlabs.ai"); + public static final String JUDGMENT_DEFAULT_GPT_MODEL = getEnvVar("JUDGMENT_DEFAULT_GPT_MODEL", "gpt-4.1"); + public static final String JUDGMENT_NO_COLOR = getEnvVar("JUDGMENT_NO_COLOR"); + public static final String JUDGMENT_LOG_LEVEL = getEnvVar("JUDGMENT_LOG_LEVEL", "warn"); private static String getEnvVar(String varName) { return getEnvVar(varName, null); diff --git a/src/main/java/com/judgmentlabs/judgeval/Version.java b/src/main/java/com/judgmentlabs/judgeval/Version.java index f8d252e..d97531b 100644 --- a/src/main/java/com/judgmentlabs/judgeval/Version.java +++ b/src/main/java/com/judgmentlabs/judgeval/Version.java @@ -2,7 +2,8 @@ public final class Version { - private Version() {} + private Version() { + } /** * Gets the current SDK version. @@ -11,8 +12,6 @@ private Version() {} */ public static String getVersion() { Package pkg = Version.class.getPackage(); - return (pkg != null && pkg.getImplementationVersion() != null) - ? pkg.getImplementationVersion() - : "unknown"; + return (pkg != null && pkg.getImplementationVersion() != null) ? pkg.getImplementationVersion() : "unknown"; } } diff --git a/src/main/java/com/judgmentlabs/judgeval/data/Example.java b/src/main/java/com/judgmentlabs/judgeval/data/Example.java index ebfc049..a09ec10 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/Example.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/Example.java @@ -7,8 +7,10 @@ public class Example extends com.judgmentlabs.judgeval.internal.api.models.Examp public Example() { super(); - setExampleId(UUID.randomUUID().toString()); - setCreatedAt(Instant.now().toString()); + setExampleId(UUID.randomUUID() + .toString()); + setCreatedAt(Instant.now() + .toString()); setName(null); } diff --git a/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java deleted file mode 100644 index fb64636..0000000 --- a/src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java +++ /dev/null @@ -1,152 +0,0 @@ -package com.judgmentlabs.judgeval.data; - -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.Objects; -import java.util.UUID; - -import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; - -public class ExampleEvaluationRun - extends com.judgmentlabs.judgeval.internal.api.models.ExampleEvaluationRun { - - private String organizationId; - - public ExampleEvaluationRun() { - super(); - setId(UUID.randomUUID().toString()); - setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); - } - - public ExampleEvaluationRun( - String projectName, - String evalName, - List examples, - List scorers, - String model, - String organizationId) { - this(); - setProjectName(projectName); - setEvalName(evalName); - @SuppressWarnings("unchecked") - List internalExamples = - (List) (List) examples; - setExamples(internalExamples); - setModel(model); - setOrganizationId(organizationId); - setCustomScorers(new java.util.ArrayList<>()); - setJudgmentScorers(scorers); - } - - public void setOrganizationId(String organizationId) { - this.organizationId = organizationId; - } - - public String getOrganizationId() { - return organizationId; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - if (!super.equals(obj)) return false; - ExampleEvaluationRun other = (ExampleEvaluationRun) obj; - return Objects.equals(organizationId, other.organizationId); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), organizationId); - } - - public static Builder builder() { - return new Builder(); - } - - public static Builder builder(String projectName, String evalName) { - return new Builder(projectName, evalName); - } - - public static final class Builder { - private String projectName; - private String evalName; - private List examples; - private List scorers; - private String model; - private String organizationId; - - private Builder() {} - - private Builder(String projectName, String evalName) { - this.projectName = projectName; - this.evalName = evalName; - } - - public Builder projectName(String projectName) { - this.projectName = projectName; - return this; - } - - public Builder evalName(String evalName) { - this.evalName = evalName; - return this; - } - - public Builder examples(List examples) { - this.examples = examples; - return this; - } - - public Builder example(Example example) { - if (this.examples == null) { - this.examples = new java.util.ArrayList<>(); - } - this.examples.add(example); - return this; - } - - public Builder scorers(List scorers) { - this.scorers = scorers; - return this; - } - - public Builder scorer(ScorerConfig scorer) { - if (this.scorers == null) { - this.scorers = new java.util.ArrayList<>(); - } - this.scorers.add(scorer); - return this; - } - - public Builder model(String model) { - this.model = model; - return this; - } - - public Builder organizationId(String organizationId) { - this.organizationId = organizationId; - return this; - } - - public ExampleEvaluationRun build() { - if (projectName == null || projectName.trim().isEmpty()) { - throw new IllegalArgumentException("Project name is required"); - } - if (evalName == null || evalName.trim().isEmpty()) { - throw new IllegalArgumentException("Evaluation name is required"); - } - if (examples == null || examples.isEmpty()) { - throw new IllegalArgumentException("At least one example is required"); - } - if (scorers == null || scorers.isEmpty()) { - throw new IllegalArgumentException("At least one scorer is required"); - } - - return new ExampleEvaluationRun( - projectName, evalName, examples, scorers, model, organizationId); - } - } -} diff --git a/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java b/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java index 8df8b2a..0f262c7 100644 --- a/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java +++ b/src/main/java/com/judgmentlabs/judgeval/data/ScoringResult.java @@ -22,9 +22,7 @@ public Builder success(Boolean success) { public Builder scorersData(List scorersData) { @SuppressWarnings("unchecked") - List internalList = - (List) - (List) scorersData; + List internalList = (List) (List) scorersData; result.setScorersData(internalList); return this; } @@ -33,13 +31,16 @@ public Builder scorerData(ScorerData scorerData) { if (result.getScorersData() == null) { result.setScorersData(new java.util.ArrayList<>()); } - result.getScorersData().add(scorerData); + result.getScorersData() + .add(scorerData); return this; } public Builder dataObject(Example dataObject) { - // Store Example in additional properties since setDataObject expects TraceSpan - // This indicates a potential API design issue - ScoringResult may be + // Store Example in additional properties since setDataObject + // expects TraceSpan + // This indicates a potential API design issue - ScoringResult may + // be // trace-specific if (dataObject != null) { result.setAdditionalProperty("example", dataObject); diff --git a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java deleted file mode 100644 index 311b31a..0000000 --- a/src/main/java/com/judgmentlabs/judgeval/data/TraceEvaluationRun.java +++ /dev/null @@ -1,168 +0,0 @@ -package com.judgmentlabs.judgeval.data; - -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.Objects; -import java.util.UUID; - -import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; - -public class TraceEvaluationRun - extends com.judgmentlabs.judgeval.internal.api.models.TraceEvaluationRun { - - private String organizationId; - - public TraceEvaluationRun() { - super(); - setId(UUID.randomUUID().toString()); - setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT)); - setIsOffline(false); - } - - public TraceEvaluationRun( - String projectName, - String evalName, - List scorers, - String model, - String organizationId, - List> traceAndSpanIds) { - this(); - setProjectName(projectName); - setEvalName(evalName); - setModel(model); - setOrganizationId(organizationId); - setTraceAndSpanIds(convertTraceAndSpanIds(traceAndSpanIds)); - setCustomScorers(new java.util.ArrayList<>()); - setJudgmentScorers(scorers); - } - - private static List> convertTraceAndSpanIds(List> traceAndSpanIds) { - if (traceAndSpanIds == null || traceAndSpanIds.isEmpty()) { - throw new IllegalArgumentException( - "Trace and span IDs are required for trace evaluations."); - } - - List> converted = new java.util.ArrayList<>(); - for (List pair : traceAndSpanIds) { - if (pair == null || pair.size() != 2) { - throw new IllegalArgumentException( - "Each trace and span ID pair must contain exactly 2 elements."); - } - converted.add(List.of(pair.get(0), pair.get(1))); - } - return converted; - } - - public void setOrganizationId(String organizationId) { - this.organizationId = organizationId; - } - - public String getOrganizationId() { - return organizationId; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - if (!super.equals(obj)) return false; - TraceEvaluationRun other = (TraceEvaluationRun) obj; - return Objects.equals(organizationId, other.organizationId); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), organizationId); - } - - public static Builder builder() { - return new Builder(); - } - - public static Builder builder(String projectName, String evalName) { - return new Builder(projectName, evalName); - } - - public static final class Builder { - private String projectName; - private String evalName; - private List scorers; - private String model; - private String organizationId; - private List> traceAndSpanIds; - - private Builder() {} - - private Builder(String projectName, String evalName) { - this.projectName = projectName; - this.evalName = evalName; - } - - public Builder projectName(String projectName) { - this.projectName = projectName; - return this; - } - - public Builder evalName(String evalName) { - this.evalName = evalName; - return this; - } - - public Builder scorers(List scorers) { - this.scorers = scorers; - return this; - } - - public Builder scorer(ScorerConfig scorer) { - if (this.scorers == null) { - this.scorers = new java.util.ArrayList<>(); - } - this.scorers.add(scorer); - return this; - } - - public Builder model(String model) { - this.model = model; - return this; - } - - public Builder organizationId(String organizationId) { - this.organizationId = organizationId; - return this; - } - - public Builder traceAndSpanIds(List> traceAndSpanIds) { - this.traceAndSpanIds = traceAndSpanIds; - return this; - } - - public Builder traceAndSpanId(String traceId, String spanId) { - if (this.traceAndSpanIds == null) { - this.traceAndSpanIds = new java.util.ArrayList<>(); - } - this.traceAndSpanIds.add(List.of(traceId, spanId)); - return this; - } - - public TraceEvaluationRun build() { - if (projectName == null || projectName.trim().isEmpty()) { - throw new IllegalArgumentException("Project name is required"); - } - if (evalName == null || evalName.trim().isEmpty()) { - throw new IllegalArgumentException("Evaluation name is required"); - } - if (scorers == null || scorers.isEmpty()) { - throw new IllegalArgumentException("At least one scorer is required"); - } - if (traceAndSpanIds == null || traceAndSpanIds.isEmpty()) { - throw new IllegalArgumentException( - "At least one trace and span ID pair is required"); - } - - return new TraceEvaluationRun( - projectName, evalName, scorers, model, organizationId, traceAndSpanIds); - } - } -} diff --git a/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentAPIError.java b/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentAPIError.java index b200c0c..8126891 100644 --- a/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentAPIError.java +++ b/src/main/java/com/judgmentlabs/judgeval/exceptions/JudgmentAPIError.java @@ -1,7 +1,7 @@ package com.judgmentlabs.judgeval.exceptions; public class JudgmentAPIError extends RuntimeException { - private final int statusCode; + private final int statusCode; private final String detail; public JudgmentAPIError(int statusCode, String detail) { diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java index 6fed7da..0ad218d 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentAsyncClient.java @@ -6,6 +6,7 @@ import java.net.http.HttpResponse; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.concurrent.CompletableFuture; import com.fasterxml.jackson.core.type.TypeReference; @@ -13,17 +14,19 @@ import com.judgmentlabs.judgeval.internal.api.models.*; public class JudgmentAsyncClient { - private final HttpClient client; + private final HttpClient client; private final ObjectMapper mapper; - private final String baseUrl; - private final String apiKey; - private final String organizationId; + private final String baseUrl; + private final String apiKey; + private final String organizationId; public JudgmentAsyncClient(String baseUrl, String apiKey, String organizationId) { - this.baseUrl = baseUrl; - this.apiKey = apiKey; - this.organizationId = organizationId; - this.client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build(); + this.baseUrl = Objects.requireNonNull(baseUrl, "Base URL cannot be null"); + this.apiKey = Objects.requireNonNull(apiKey, "API key cannot be null"); + this.organizationId = Objects.requireNonNull(organizationId, "Organization ID cannot be null"); + this.client = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_1_1) + .build(); this.mapper = new ObjectMapper(); } @@ -31,10 +34,9 @@ private String buildUrl(String path, Map queryParams) { StringBuilder url = new StringBuilder(baseUrl).append(path); if (!queryParams.isEmpty()) { url.append("?"); - String queryString = - queryParams.entrySet().stream() - .map(entry -> entry.getKey() + "=" + entry.getValue()) - .reduce("", (a, b) -> a.isEmpty() ? b : a + "&" + b); + String queryString = queryParams.entrySet().stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .reduce("", (a, b) -> a.isEmpty() ? b : a + "&" + b); url.append(queryString); } return url.toString(); @@ -45,26 +47,23 @@ private String buildUrl(String path) { } private String[] buildHeaders() { - if (apiKey == null || organizationId == null) { - throw new IllegalArgumentException("API key and organization ID cannot be null"); - } return new String[] { - "Content-Type", - "application/json", - "Authorization", - "Bearer " + apiKey, - "X-Organization-Id", - organizationId + "Content-Type", + "application/json", + "Authorization", + "Bearer " + apiKey, + "X-Organization-Id", + organizationId }; } private T handleResponse(HttpResponse response) { if (response.statusCode() >= 400) { - throw new RuntimeException( - "HTTP Error: " + response.statusCode() + " - " + response.body()); + throw new RuntimeException("HTTP Error: " + response.statusCode() + " - " + response.body()); } try { - return mapper.readValue(response.body(), new TypeReference() {}); + return mapper.readValue(response.body(), new TypeReference() { + }); } catch (Exception e) { throw new RuntimeException("Failed to parse response", e); } @@ -78,12 +77,11 @@ public CompletableFuture addToRunEvalQueue(ExampleEvaluationRun payload) } catch (Exception e) { throw new RuntimeException("Failed to serialize payload", e); } - HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } @@ -96,12 +94,11 @@ public CompletableFuture logEvalResults(EvalResults payload) { } catch (Exception e) { throw new RuntimeException("Failed to serialize payload", e); } - HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } @@ -114,24 +111,11 @@ public CompletableFuture fetchExperimentRun(EvalResultsFetch payload) { } catch (Exception e) { throw new RuntimeException("Failed to serialize payload", e); } - HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); - return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) - .thenApply(this::handleResponse); - } - - public CompletableFuture getEvaluationStatus( - String experiment_run_id, String project_name) { - Map queryParams = new HashMap<>(); - queryParams.put("experiment_run_id", experiment_run_id); - queryParams.put("project_name", project_name); - String url = buildUrl("/get_evaluation_status/", queryParams); - HttpRequest request = - HttpRequest.newBuilder().GET().uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } @@ -144,12 +128,11 @@ public CompletableFuture scorerExists(ScorerExistsRequest } catch (Exception e) { throw new RuntimeException("Failed to serialize payload", e); } - HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } @@ -162,18 +145,16 @@ public CompletableFuture saveScorer(SavePromptScorerRe } catch (Exception e) { throw new RuntimeException("Failed to serialize payload", e); } - HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } - public CompletableFuture fetchScorers( - FetchPromptScorersRequest payload) { + public CompletableFuture fetchScorers(FetchPromptScorersRequest payload) { String url = buildUrl("/fetch_scorers/"); String jsonPayload; try { @@ -181,18 +162,16 @@ public CompletableFuture fetchScorers( } catch (Exception e) { throw new RuntimeException("Failed to serialize payload", e); } - HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } - public CompletableFuture projectsResolve( - ResolveProjectNameRequest payload) { + public CompletableFuture projectsResolve(ResolveProjectNameRequest payload) { String url = buildUrl("/projects/resolve/"); String jsonPayload; try { @@ -200,13 +179,13 @@ public CompletableFuture projectsResolve( } catch (Exception e) { throw new RuntimeException("Failed to serialize payload", e); } - HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); return client.sendAsync(request, HttpResponse.BodyHandlers.ofString()) .thenApply(this::handleResponse); } -} + +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java index 5ea1b80..d0f1f5d 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/JudgmentSyncClient.java @@ -7,23 +7,26 @@ import java.net.http.HttpResponse; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.judgmentlabs.judgeval.internal.api.models.*; public class JudgmentSyncClient { - private final HttpClient client; + private final HttpClient client; private final ObjectMapper mapper; - private final String baseUrl; - private final String apiKey; - private final String organizationId; + private final String baseUrl; + private final String apiKey; + private final String organizationId; public JudgmentSyncClient(String baseUrl, String apiKey, String organizationId) { - this.baseUrl = baseUrl; - this.apiKey = apiKey; - this.organizationId = organizationId; - this.client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build(); + this.baseUrl = Objects.requireNonNull(baseUrl, "Base URL cannot be null"); + this.apiKey = Objects.requireNonNull(apiKey, "API key cannot be null"); + this.organizationId = Objects.requireNonNull(organizationId, "Organization ID cannot be null"); + this.client = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_1_1) + .build(); this.mapper = new ObjectMapper(); } @@ -31,10 +34,9 @@ private String buildUrl(String path, Map queryParams) { StringBuilder url = new StringBuilder(baseUrl).append(path); if (!queryParams.isEmpty()) { url.append("?"); - String queryString = - queryParams.entrySet().stream() - .map(entry -> entry.getKey() + "=" + entry.getValue()) - .reduce("", (a, b) -> a.isEmpty() ? b : a + "&" + b); + String queryString = queryParams.entrySet().stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .reduce("", (a, b) -> a.isEmpty() ? b : a + "&" + b); url.append(queryString); } return url.toString(); @@ -45,41 +47,36 @@ private String buildUrl(String path) { } private String[] buildHeaders() { - if (apiKey == null || organizationId == null) { - throw new IllegalArgumentException("API key and organization ID cannot be null"); - } return new String[] { - "Content-Type", - "application/json", - "Authorization", - "Bearer " + apiKey, - "X-Organization-Id", - organizationId + "Content-Type", + "application/json", + "Authorization", + "Bearer " + apiKey, + "X-Organization-Id", + organizationId }; } private T handleResponse(HttpResponse response) throws IOException { if (response.statusCode() >= 400) { - throw new RuntimeException( - "HTTP Error: " + response.statusCode() + " - " + response.body()); + throw new RuntimeException("HTTP Error: " + response.statusCode() + " - " + response.body()); } try { - return mapper.readValue(response.body(), new TypeReference() {}); + return mapper.readValue(response.body(), new TypeReference() { + }); } catch (Exception e) { throw new RuntimeException("Failed to parse response", e); } } - public Object addToRunEvalQueue(ExampleEvaluationRun payload) - throws IOException, InterruptedException { + public Object addToRunEvalQueue(ExampleEvaluationRun payload) throws IOException, InterruptedException { String url = buildUrl("/add_to_run_eval_queue/"); String jsonPayload = mapper.writeValueAsString(payload); - HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return handleResponse(response); } @@ -87,52 +84,35 @@ public Object addToRunEvalQueue(ExampleEvaluationRun payload) public Object logEvalResults(EvalResults payload) throws IOException, InterruptedException { String url = buildUrl("/log_eval_results/"); String jsonPayload = mapper.writeValueAsString(payload); - HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return handleResponse(response); } - public Object fetchExperimentRun(EvalResultsFetch payload) - throws IOException, InterruptedException { + public Object fetchExperimentRun(EvalResultsFetch payload) throws IOException, InterruptedException { String url = buildUrl("/fetch_experiment_run/"); String jsonPayload = mapper.writeValueAsString(payload); - HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); - return handleResponse(response); - } - - public Object getEvaluationStatus(String experiment_run_id, String project_name) - throws IOException, InterruptedException { - Map queryParams = new HashMap<>(); - queryParams.put("experiment_run_id", experiment_run_id); - queryParams.put("project_name", project_name); - String url = buildUrl("/get_evaluation_status/", queryParams); - HttpRequest request = - HttpRequest.newBuilder().GET().uri(URI.create(url)).headers(buildHeaders()).build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return handleResponse(response); } - public ScorerExistsResponse scorerExists(ScorerExistsRequest payload) - throws IOException, InterruptedException { + public ScorerExistsResponse scorerExists(ScorerExistsRequest payload) throws IOException, InterruptedException { String url = buildUrl("/scorer_exists/"); String jsonPayload = mapper.writeValueAsString(payload); - HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return mapper.readValue(response.body(), ScorerExistsResponse.class); } @@ -141,12 +121,11 @@ public SavePromptScorerResponse saveScorer(SavePromptScorerRequest payload) throws IOException, InterruptedException { String url = buildUrl("/save_scorer/"); String jsonPayload = mapper.writeValueAsString(payload); - HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return mapper.readValue(response.body(), SavePromptScorerResponse.class); } @@ -155,12 +134,11 @@ public FetchPromptScorersResponse fetchScorers(FetchPromptScorersRequest payload throws IOException, InterruptedException { String url = buildUrl("/fetch_scorers/"); String jsonPayload = mapper.writeValueAsString(payload); - HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return mapper.readValue(response.body(), FetchPromptScorersResponse.class); } @@ -169,13 +147,13 @@ public ResolveProjectNameResponse projectsResolve(ResolveProjectNameRequest payl throws IOException, InterruptedException { String url = buildUrl("/projects/resolve/"); String jsonPayload = mapper.writeValueAsString(payload); - HttpRequest request = - HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) - .uri(URI.create(url)) - .headers(buildHeaders()) - .build(); + HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers.ofString(jsonPayload)) + .uri(URI.create(url)) + .headers(buildHeaders()) + .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); return mapper.readValue(response.body(), ResolveProjectNameResponse.class); } -} + +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java index 72811fb..235a640 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/BaseScorer.java @@ -10,52 +10,37 @@ public class BaseScorer { @JsonProperty("score_type") - private String scoreType; - + private String scoreType; @JsonProperty("threshold") - private Double threshold; - + private Double threshold; @JsonProperty("name") - private String name; - + private String name; @JsonProperty("class_name") - private String className; - + private String className; @JsonProperty("score") - private Double score; - + private Double score; @JsonProperty("score_breakdown") - private Object scoreBreakdown; - + private Object scoreBreakdown; @JsonProperty("reason") - private String reason; - + private String reason; @JsonProperty("using_native_model") - private Boolean usingNativeModel; - + private Boolean usingNativeModel; @JsonProperty("success") - private Boolean success; - + private Boolean success; @JsonProperty("model") - private String model; - + private String model; @JsonProperty("model_client") - private Object modelClient; - + private Object modelClient; @JsonProperty("strict_mode") - private Boolean strictMode; - + private Boolean strictMode; @JsonProperty("error") - private String error; - + private String error; @JsonProperty("additional_metadata") - private Object additionalMetadata; - + private Object additionalMetadata; @JsonProperty("user") - private String user; - + private String user; @JsonProperty("server_hosted") - private Boolean serverHosted; + private Boolean serverHosted; private Map additionalProperties = new HashMap<>(); @@ -199,47 +184,26 @@ public void setServerHosted(Boolean serverHosted) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; BaseScorer other = (BaseScorer) obj; - return Objects.equals(scoreType, other.scoreType) - && Objects.equals(threshold, other.threshold) - && Objects.equals(name, other.name) - && Objects.equals(className, other.className) - && Objects.equals(score, other.score) - && Objects.equals(scoreBreakdown, other.scoreBreakdown) - && Objects.equals(reason, other.reason) - && Objects.equals(usingNativeModel, other.usingNativeModel) - && Objects.equals(success, other.success) - && Objects.equals(model, other.model) - && Objects.equals(modelClient, other.modelClient) - && Objects.equals(strictMode, other.strictMode) - && Objects.equals(error, other.error) - && Objects.equals(additionalMetadata, other.additionalMetadata) - && Objects.equals(user, other.user) - && Objects.equals(serverHosted, other.serverHosted) + return Objects.equals(scoreType, other.scoreType) && Objects.equals(threshold, other.threshold) + && Objects.equals(name, other.name) && Objects.equals(className, other.className) + && Objects.equals(score, other.score) && Objects.equals(scoreBreakdown, other.scoreBreakdown) + && Objects.equals(reason, other.reason) && Objects.equals(usingNativeModel, other.usingNativeModel) + && Objects.equals(success, other.success) && Objects.equals(model, other.model) + && Objects.equals(modelClient, other.modelClient) && Objects.equals(strictMode, other.strictMode) + && Objects.equals(error, other.error) && Objects.equals(additionalMetadata, other.additionalMetadata) + && Objects.equals(user, other.user) && Objects.equals(serverHosted, other.serverHosted) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash( - scoreType, - threshold, - name, - className, - score, - scoreBreakdown, - reason, - usingNativeModel, - success, - model, - modelClient, - strictMode, - error, - additionalMetadata, - user, - serverHosted, + return Objects.hash(scoreType, threshold, name, className, score, scoreBreakdown, reason, usingNativeModel, + success, model, modelClient, strictMode, error, additionalMetadata, user, serverHosted, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java index 7a5e5c5..9d7494b 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResults.java @@ -12,9 +12,8 @@ public class EvalResults { @JsonProperty("results") private List results; - @JsonProperty("run") - private Object run; + private Object run; private Map additionalProperties = new HashMap<>(); @@ -46,11 +45,12 @@ public void setRun(Object run) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; EvalResults other = (EvalResults) obj; - return Objects.equals(results, other.results) - && Objects.equals(run, other.run) + return Objects.equals(results, other.results) && Objects.equals(run, other.run) && Objects.equals(additionalProperties, other.additionalProperties); } @@ -58,4 +58,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(results, run, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java index 92ef79c..9e77977 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/EvalResultsFetch.java @@ -10,10 +10,9 @@ public class EvalResultsFetch { @JsonProperty("experiment_run_id") - private String experimentRunId; - + private String experimentRunId; @JsonProperty("project_name") - private String projectName; + private String projectName; private Map additionalProperties = new HashMap<>(); @@ -45,11 +44,12 @@ public void setProjectName(String projectName) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; EvalResultsFetch other = (EvalResultsFetch) obj; - return Objects.equals(experimentRunId, other.experimentRunId) - && Objects.equals(projectName, other.projectName) + return Objects.equals(experimentRunId, other.experimentRunId) && Objects.equals(projectName, other.projectName) && Objects.equals(additionalProperties, other.additionalProperties); } @@ -57,4 +57,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(experimentRunId, projectName, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java index 23ca12b..92b8c63 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/Example.java @@ -10,13 +10,11 @@ public class Example { @JsonProperty("example_id") - private String exampleId; - + private String exampleId; @JsonProperty("created_at") - private String createdAt; - + private String createdAt; @JsonProperty("name") - private String name; + private String name; private Map additionalProperties = new HashMap<>(); @@ -56,17 +54,17 @@ public void setName(String name) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; Example other = (Example) obj; - return Objects.equals(exampleId, other.exampleId) - && Objects.equals(createdAt, other.createdAt) - && Objects.equals(name, other.name) - && Objects.equals(additionalProperties, other.additionalProperties); + return Objects.equals(exampleId, other.exampleId) && Objects.equals(createdAt, other.createdAt) + && Objects.equals(name, other.name) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { return Objects.hash(exampleId, createdAt, name, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java index e942be4..847e4f2 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ExampleEvaluationRun.java @@ -11,34 +11,25 @@ public class ExampleEvaluationRun { @JsonProperty("id") - private String id; - + private String id; @JsonProperty("project_name") - private String projectName; - + private String projectName; @JsonProperty("eval_name") - private String evalName; - + private String evalName; @JsonProperty("custom_scorers") - private List customScorers; - + private List customScorers; @JsonProperty("judgment_scorers") - private List judgmentScorers; - + private List judgmentScorers; @JsonProperty("model") - private String model; - + private String model; @JsonProperty("created_at") - private String createdAt; - + private String createdAt; @JsonProperty("examples") - private List examples; - + private List examples; @JsonProperty("trace_span_id") - private String traceSpanId; - + private String traceSpanId; @JsonProperty("trace_id") - private String traceId; + private String traceId; private Map additionalProperties = new HashMap<>(); @@ -134,35 +125,22 @@ public void setTraceId(String traceId) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ExampleEvaluationRun other = (ExampleEvaluationRun) obj; - return Objects.equals(id, other.id) - && Objects.equals(projectName, other.projectName) - && Objects.equals(evalName, other.evalName) - && Objects.equals(customScorers, other.customScorers) - && Objects.equals(judgmentScorers, other.judgmentScorers) - && Objects.equals(model, other.model) - && Objects.equals(createdAt, other.createdAt) - && Objects.equals(examples, other.examples) - && Objects.equals(traceSpanId, other.traceSpanId) - && Objects.equals(traceId, other.traceId) + return Objects.equals(id, other.id) && Objects.equals(projectName, other.projectName) + && Objects.equals(evalName, other.evalName) && Objects.equals(customScorers, other.customScorers) + && Objects.equals(judgmentScorers, other.judgmentScorers) && Objects.equals(model, other.model) + && Objects.equals(createdAt, other.createdAt) && Objects.equals(examples, other.examples) + && Objects.equals(traceSpanId, other.traceSpanId) && Objects.equals(traceId, other.traceId) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash( - id, - projectName, - evalName, - customScorers, - judgmentScorers, - model, - createdAt, - examples, - traceSpanId, - traceId, - Objects.hashCode(additionalProperties)); - } -} + return Objects.hash(id, projectName, evalName, customScorers, judgmentScorers, model, createdAt, examples, + traceSpanId, traceId, Objects.hashCode(additionalProperties)); + } +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java index 49bd079..31f4c24 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersRequest.java @@ -11,7 +11,9 @@ public class FetchPromptScorersRequest { @JsonProperty("names") - private List names; + private List names; + @JsonProperty("is_trace") + private Boolean isTrace; private Map additionalProperties = new HashMap<>(); @@ -29,21 +31,31 @@ public List getNames() { return names; } + public Boolean getIsTrace() { + return isTrace; + } + public void setNames(List names) { this.names = names; } + public void setIsTrace(Boolean isTrace) { + this.isTrace = isTrace; + } + @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; FetchPromptScorersRequest other = (FetchPromptScorersRequest) obj; - return Objects.equals(names, other.names) + return Objects.equals(names, other.names) && Objects.equals(isTrace, other.isTrace) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash(names, Objects.hashCode(additionalProperties)); + return Objects.hash(names, isTrace, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java index 7bfe803..d6521eb 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/FetchPromptScorersResponse.java @@ -11,7 +11,7 @@ public class FetchPromptScorersResponse { @JsonProperty("scorers") - private List scorers; + private List scorers; private Map additionalProperties = new HashMap<>(); @@ -35,8 +35,10 @@ public void setScorers(List scorers) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; FetchPromptScorersResponse other = (FetchPromptScorersResponse) obj; return Objects.equals(scorers, other.scorers) && Objects.equals(additionalProperties, other.additionalProperties); @@ -46,4 +48,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(scorers, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java index 9755864..949082d 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/OtelTraceSpan.java @@ -11,82 +11,41 @@ public class OtelTraceSpan { @JsonProperty("organization_id") - private String organizationId; - + private String organizationId; @JsonProperty("project_id") - private String projectId; - + private String projectId; @JsonProperty("user_id") - private String userId; - + private String userId; @JsonProperty("timestamp") - private String timestamp; - + private String timestamp; @JsonProperty("trace_id") - private String traceId; - + private String traceId; @JsonProperty("span_id") - private String spanId; - + private String spanId; @JsonProperty("parent_span_id") - private String parentSpanId; - + private String parentSpanId; @JsonProperty("trace_state") - private String traceState; - + private String traceState; @JsonProperty("span_name") - private String spanName; - + private String spanName; @JsonProperty("span_kind") - private String spanKind; - + private String spanKind; @JsonProperty("service_name") - private String serviceName; - + private String serviceName; @JsonProperty("resource_attributes") - private Object resourceAttributes; - + private Object resourceAttributes; @JsonProperty("span_attributes") - private Object spanAttributes; - + private Object spanAttributes; @JsonProperty("duration") - private Integer duration; - + private Integer duration; @JsonProperty("status_code") - private String statusCode; - + private Integer statusCode; @JsonProperty("status_message") - private String statusMessage; - + private String statusMessage; @JsonProperty("events") - private List events; - + private List events; @JsonProperty("links") - private List links; - - @JsonProperty("legacy_span_id") - private String legacySpanId; - - @JsonProperty("inputs") - private Object inputs; - - @JsonProperty("output") - private Object output; - - @JsonProperty("error") - private Object error; - - @JsonProperty("agent_id") - private String agentId; - - @JsonProperty("cumulative_llm_cost") - private Double cumulativeLlmCost; - - @JsonProperty("state_after") - private Object stateAfter; - - @JsonProperty("state_before") - private Object stateBefore; + private List links; private Map additionalProperties = new HashMap<>(); @@ -156,7 +115,7 @@ public Integer getDuration() { return duration; } - public String getStatusCode() { + public Integer getStatusCode() { return statusCode; } @@ -172,38 +131,6 @@ public List getLinks() { return links; } - public String getLegacySpanId() { - return legacySpanId; - } - - public Object getInputs() { - return inputs; - } - - public Object getOutput() { - return output; - } - - public Object getError() { - return error; - } - - public String getAgentId() { - return agentId; - } - - public Double getCumulativeLlmCost() { - return cumulativeLlmCost; - } - - public Object getStateAfter() { - return stateAfter; - } - - public Object getStateBefore() { - return stateBefore; - } - public void setOrganizationId(String organizationId) { this.organizationId = organizationId; } @@ -260,7 +187,7 @@ public void setDuration(Integer duration) { this.duration = duration; } - public void setStatusCode(String statusCode) { + public void setStatusCode(Integer statusCode) { this.statusCode = statusCode; } @@ -276,101 +203,30 @@ public void setLinks(List links) { this.links = links; } - public void setLegacySpanId(String legacySpanId) { - this.legacySpanId = legacySpanId; - } - - public void setInputs(Object inputs) { - this.inputs = inputs; - } - - public void setOutput(Object output) { - this.output = output; - } - - public void setError(Object error) { - this.error = error; - } - - public void setAgentId(String agentId) { - this.agentId = agentId; - } - - public void setCumulativeLlmCost(Double cumulativeLlmCost) { - this.cumulativeLlmCost = cumulativeLlmCost; - } - - public void setStateAfter(Object stateAfter) { - this.stateAfter = stateAfter; - } - - public void setStateBefore(Object stateBefore) { - this.stateBefore = stateBefore; - } - @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; OtelTraceSpan other = (OtelTraceSpan) obj; - return Objects.equals(organizationId, other.organizationId) - && Objects.equals(projectId, other.projectId) - && Objects.equals(userId, other.userId) - && Objects.equals(timestamp, other.timestamp) - && Objects.equals(traceId, other.traceId) - && Objects.equals(spanId, other.spanId) - && Objects.equals(parentSpanId, other.parentSpanId) - && Objects.equals(traceState, other.traceState) - && Objects.equals(spanName, other.spanName) - && Objects.equals(spanKind, other.spanKind) + return Objects.equals(organizationId, other.organizationId) && Objects.equals(projectId, other.projectId) + && Objects.equals(userId, other.userId) && Objects.equals(timestamp, other.timestamp) + && Objects.equals(traceId, other.traceId) && Objects.equals(spanId, other.spanId) + && Objects.equals(parentSpanId, other.parentSpanId) && Objects.equals(traceState, other.traceState) + && Objects.equals(spanName, other.spanName) && Objects.equals(spanKind, other.spanKind) && Objects.equals(serviceName, other.serviceName) && Objects.equals(resourceAttributes, other.resourceAttributes) - && Objects.equals(spanAttributes, other.spanAttributes) - && Objects.equals(duration, other.duration) - && Objects.equals(statusCode, other.statusCode) - && Objects.equals(statusMessage, other.statusMessage) - && Objects.equals(events, other.events) - && Objects.equals(links, other.links) - && Objects.equals(legacySpanId, other.legacySpanId) - && Objects.equals(inputs, other.inputs) - && Objects.equals(output, other.output) - && Objects.equals(error, other.error) - && Objects.equals(agentId, other.agentId) - && Objects.equals(cumulativeLlmCost, other.cumulativeLlmCost) - && Objects.equals(stateAfter, other.stateAfter) - && Objects.equals(stateBefore, other.stateBefore) + && Objects.equals(spanAttributes, other.spanAttributes) && Objects.equals(duration, other.duration) + && Objects.equals(statusCode, other.statusCode) && Objects.equals(statusMessage, other.statusMessage) + && Objects.equals(events, other.events) && Objects.equals(links, other.links) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash( - organizationId, - projectId, - userId, - timestamp, - traceId, - spanId, - parentSpanId, - traceState, - spanName, - spanKind, - serviceName, - resourceAttributes, - spanAttributes, - duration, - statusCode, - statusMessage, - events, - links, - legacySpanId, - inputs, - output, - error, - agentId, - cumulativeLlmCost, - stateAfter, - stateBefore, - Objects.hashCode(additionalProperties)); - } -} + return Objects.hash(organizationId, projectId, userId, timestamp, traceId, spanId, parentSpanId, traceState, + spanName, spanKind, serviceName, resourceAttributes, spanAttributes, duration, statusCode, + statusMessage, events, links, Objects.hashCode(additionalProperties)); + } +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java index 8327092..3c0662c 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/PromptScorer.java @@ -9,26 +9,32 @@ import com.fasterxml.jackson.annotation.JsonProperty; public class PromptScorer { + @JsonProperty("id") + private String id; + @JsonProperty("user_id") + private String userId; + @JsonProperty("organization_id") + private String organizationId; @JsonProperty("name") - private String name; - + private String name; @JsonProperty("prompt") - private String prompt; - + private String prompt; @JsonProperty("threshold") - private Double threshold; - + private Double threshold; + @JsonProperty("model") + private String model; @JsonProperty("options") - private Object options; - + private Object options; + @JsonProperty("description") + private String description; @JsonProperty("created_at") - private String createdAt; - + private String createdAt; @JsonProperty("updated_at") - private String updatedAt; - + private String updatedAt; @JsonProperty("is_trace") - private Boolean isTrace; + private Boolean isTrace; + @JsonProperty("is_bucket_rubric") + private Boolean isBucketRubric; private Map additionalProperties = new HashMap<>(); @@ -42,6 +48,18 @@ public void setAdditionalProperty(String name, Object value) { additionalProperties.put(name, value); } + public String getId() { + return id; + } + + public String getUserId() { + return userId; + } + + public String getOrganizationId() { + return organizationId; + } + public String getName() { return name; } @@ -54,10 +72,18 @@ public Double getThreshold() { return threshold; } + public String getModel() { + return model; + } + public Object getOptions() { return options; } + public String getDescription() { + return description; + } + public String getCreatedAt() { return createdAt; } @@ -70,6 +96,22 @@ public Boolean getIsTrace() { return isTrace; } + public Boolean getIsBucketRubric() { + return isBucketRubric; + } + + public void setId(String id) { + this.id = id; + } + + public void setUserId(String userId) { + this.userId = userId; + } + + public void setOrganizationId(String organizationId) { + this.organizationId = organizationId; + } + public void setName(String name) { this.name = name; } @@ -82,10 +124,18 @@ public void setThreshold(Double threshold) { this.threshold = threshold; } + public void setModel(String model) { + this.model = model; + } + public void setOptions(Object options) { this.options = options; } + public void setDescription(String description) { + this.description = description; + } + public void setCreatedAt(String createdAt) { this.createdAt = createdAt; } @@ -98,31 +148,30 @@ public void setIsTrace(Boolean isTrace) { this.isTrace = isTrace; } + public void setIsBucketRubric(Boolean isBucketRubric) { + this.isBucketRubric = isBucketRubric; + } + @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; PromptScorer other = (PromptScorer) obj; - return Objects.equals(name, other.name) - && Objects.equals(prompt, other.prompt) - && Objects.equals(threshold, other.threshold) - && Objects.equals(options, other.options) - && Objects.equals(createdAt, other.createdAt) - && Objects.equals(updatedAt, other.updatedAt) - && Objects.equals(isTrace, other.isTrace) + return Objects.equals(id, other.id) && Objects.equals(userId, other.userId) + && Objects.equals(organizationId, other.organizationId) && Objects.equals(name, other.name) + && Objects.equals(prompt, other.prompt) && Objects.equals(threshold, other.threshold) + && Objects.equals(model, other.model) && Objects.equals(options, other.options) + && Objects.equals(description, other.description) && Objects.equals(createdAt, other.createdAt) + && Objects.equals(updatedAt, other.updatedAt) && Objects.equals(isTrace, other.isTrace) + && Objects.equals(isBucketRubric, other.isBucketRubric) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash( - name, - prompt, - threshold, - options, - createdAt, - updatedAt, - isTrace, - Objects.hashCode(additionalProperties)); - } -} + return Objects.hash(id, userId, organizationId, name, prompt, threshold, model, options, description, createdAt, + updatedAt, isTrace, isBucketRubric, Objects.hashCode(additionalProperties)); + } +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java index cf1cb41..d3a2d33 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameRequest.java @@ -10,7 +10,7 @@ public class ResolveProjectNameRequest { @JsonProperty("project_name") - private String projectName; + private String projectName; private Map additionalProperties = new HashMap<>(); @@ -34,8 +34,10 @@ public void setProjectName(String projectName) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ResolveProjectNameRequest other = (ResolveProjectNameRequest) obj; return Objects.equals(projectName, other.projectName) && Objects.equals(additionalProperties, other.additionalProperties); @@ -45,4 +47,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(projectName, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java index d668f8b..9e587ee 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ResolveProjectNameResponse.java @@ -10,7 +10,7 @@ public class ResolveProjectNameResponse { @JsonProperty("project_id") - private String projectId; + private String projectId; private Map additionalProperties = new HashMap<>(); @@ -34,8 +34,10 @@ public void setProjectId(String projectId) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ResolveProjectNameResponse other = (ResolveProjectNameResponse) obj; return Objects.equals(projectId, other.projectId) && Objects.equals(additionalProperties, other.additionalProperties); @@ -45,4 +47,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(projectId, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java index 62be159..c795d16 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerRequest.java @@ -10,19 +10,19 @@ public class SavePromptScorerRequest { @JsonProperty("name") - private String name; - + private String name; @JsonProperty("prompt") - private String prompt; - + private String prompt; @JsonProperty("threshold") - private Double threshold; - - @JsonProperty("options") - private Object options; - + private Double threshold; + @JsonProperty("model") + private String model; @JsonProperty("is_trace") - private Boolean isTrace; + private Boolean isTrace; + @JsonProperty("options") + private Object options; + @JsonProperty("description") + private String description; private Map additionalProperties = new HashMap<>(); @@ -48,14 +48,22 @@ public Double getThreshold() { return threshold; } - public Object getOptions() { - return options; + public String getModel() { + return model; } public Boolean getIsTrace() { return isTrace; } + public Object getOptions() { + return options; + } + + public String getDescription() { + return description; + } + public void setName(String name) { this.name = name; } @@ -68,30 +76,39 @@ public void setThreshold(Double threshold) { this.threshold = threshold; } - public void setOptions(Object options) { - this.options = options; + public void setModel(String model) { + this.model = model; } public void setIsTrace(Boolean isTrace) { this.isTrace = isTrace; } + public void setOptions(Object options) { + this.options = options; + } + + public void setDescription(String description) { + this.description = description; + } + @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; SavePromptScorerRequest other = (SavePromptScorerRequest) obj; - return Objects.equals(name, other.name) - && Objects.equals(prompt, other.prompt) - && Objects.equals(threshold, other.threshold) - && Objects.equals(options, other.options) - && Objects.equals(isTrace, other.isTrace) + return Objects.equals(name, other.name) && Objects.equals(prompt, other.prompt) + && Objects.equals(threshold, other.threshold) && Objects.equals(model, other.model) + && Objects.equals(isTrace, other.isTrace) && Objects.equals(options, other.options) + && Objects.equals(description, other.description) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash( - name, prompt, threshold, options, isTrace, Objects.hashCode(additionalProperties)); + return Objects.hash(name, prompt, threshold, model, isTrace, options, description, + Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java index e126892..bb1f0b4 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/SavePromptScorerResponse.java @@ -9,11 +9,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; public class SavePromptScorerResponse { - @JsonProperty("message") - private String message; - - @JsonProperty("name") - private String name; + @JsonProperty("scorer_response") + private PromptScorer scorerResponse; private Map additionalProperties = new HashMap<>(); @@ -27,34 +24,27 @@ public void setAdditionalProperty(String name, Object value) { additionalProperties.put(name, value); } - public String getMessage() { - return message; - } - - public String getName() { - return name; - } - - public void setMessage(String message) { - this.message = message; + public PromptScorer getScorerResponse() { + return scorerResponse; } - public void setName(String name) { - this.name = name; + public void setScorerResponse(PromptScorer scorerResponse) { + this.scorerResponse = scorerResponse; } @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; SavePromptScorerResponse other = (SavePromptScorerResponse) obj; - return Objects.equals(message, other.message) - && Objects.equals(name, other.name) + return Objects.equals(scorerResponse, other.scorerResponse) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash(message, name, Objects.hashCode(additionalProperties)); + return Objects.hash(scorerResponse, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java index d312ee1..72b095d 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerConfig.java @@ -11,22 +11,19 @@ public class ScorerConfig { @JsonProperty("score_type") - private String scoreType; - + private String scoreType; @JsonProperty("name") - private String name; - + private String name; @JsonProperty("threshold") - private Double threshold; - + private Double threshold; + @JsonProperty("model") + private String model; @JsonProperty("strict_mode") - private Boolean strictMode; - + private Boolean strictMode; @JsonProperty("required_params") - private List requiredParams; - + private List requiredParams; @JsonProperty("kwargs") - private Object kwargs; + private Object kwargs; private Map additionalProperties = new HashMap<>(); @@ -52,6 +49,10 @@ public Double getThreshold() { return threshold; } + public String getModel() { + return model; + } + public Boolean getStrictMode() { return strictMode; } @@ -76,6 +77,10 @@ public void setThreshold(Double threshold) { this.threshold = threshold; } + public void setModel(String model) { + this.model = model; + } + public void setStrictMode(Boolean strictMode) { this.strictMode = strictMode; } @@ -90,27 +95,21 @@ public void setKwargs(Object kwargs) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ScorerConfig other = (ScorerConfig) obj; - return Objects.equals(scoreType, other.scoreType) - && Objects.equals(name, other.name) - && Objects.equals(threshold, other.threshold) - && Objects.equals(strictMode, other.strictMode) - && Objects.equals(requiredParams, other.requiredParams) + return Objects.equals(scoreType, other.scoreType) && Objects.equals(name, other.name) + && Objects.equals(threshold, other.threshold) && Objects.equals(model, other.model) + && Objects.equals(strictMode, other.strictMode) && Objects.equals(requiredParams, other.requiredParams) && Objects.equals(kwargs, other.kwargs) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash( - scoreType, - name, - threshold, - strictMode, - requiredParams, - kwargs, + return Objects.hash(scoreType, name, threshold, model, strictMode, requiredParams, kwargs, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java index 62a7419..ccca512 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerData.java @@ -10,34 +10,25 @@ public class ScorerData { @JsonProperty("id") - private String id; - + private String id; @JsonProperty("name") - private String name; - + private String name; @JsonProperty("threshold") - private Double threshold; - + private Double threshold; @JsonProperty("success") - private Boolean success; - + private Boolean success; @JsonProperty("score") - private Double score; - + private Double score; @JsonProperty("reason") - private String reason; - + private String reason; @JsonProperty("strict_mode") - private Boolean strictMode; - + private Boolean strictMode; @JsonProperty("evaluation_model") - private String evaluationModel; - + private String evaluationModel; @JsonProperty("error") - private String error; - + private String error; @JsonProperty("additional_metadata") - private Object additionalMetadata; + private Object additionalMetadata; private Map additionalProperties = new HashMap<>(); @@ -133,35 +124,23 @@ public void setAdditionalMetadata(Object additionalMetadata) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ScorerData other = (ScorerData) obj; - return Objects.equals(id, other.id) - && Objects.equals(name, other.name) - && Objects.equals(threshold, other.threshold) - && Objects.equals(success, other.success) - && Objects.equals(score, other.score) - && Objects.equals(reason, other.reason) + return Objects.equals(id, other.id) && Objects.equals(name, other.name) + && Objects.equals(threshold, other.threshold) && Objects.equals(success, other.success) + && Objects.equals(score, other.score) && Objects.equals(reason, other.reason) && Objects.equals(strictMode, other.strictMode) - && Objects.equals(evaluationModel, other.evaluationModel) - && Objects.equals(error, other.error) + && Objects.equals(evaluationModel, other.evaluationModel) && Objects.equals(error, other.error) && Objects.equals(additionalMetadata, other.additionalMetadata) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash( - id, - name, - threshold, - success, - score, - reason, - strictMode, - evaluationModel, - error, - additionalMetadata, - Objects.hashCode(additionalProperties)); - } -} + return Objects.hash(id, name, threshold, success, score, reason, strictMode, evaluationModel, error, + additionalMetadata, Objects.hashCode(additionalProperties)); + } +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java index 809e4d4..46e178a 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsRequest.java @@ -10,7 +10,7 @@ public class ScorerExistsRequest { @JsonProperty("name") - private String name; + private String name; private Map additionalProperties = new HashMap<>(); @@ -34,15 +34,16 @@ public void setName(String name) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ScorerExistsRequest other = (ScorerExistsRequest) obj; - return Objects.equals(name, other.name) - && Objects.equals(additionalProperties, other.additionalProperties); + return Objects.equals(name, other.name) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { return Objects.hash(name, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java index 1dafef1..29939e6 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScorerExistsResponse.java @@ -10,7 +10,7 @@ public class ScorerExistsResponse { @JsonProperty("exists") - private Boolean exists; + private Boolean exists; private Map additionalProperties = new HashMap<>(); @@ -34,15 +34,16 @@ public void setExists(Boolean exists) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ScorerExistsResponse other = (ScorerExistsResponse) obj; - return Objects.equals(exists, other.exists) - && Objects.equals(additionalProperties, other.additionalProperties); + return Objects.equals(exists, other.exists) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { return Objects.hash(exists, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java index 215d06f..1231e90 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/ScoringResult.java @@ -11,25 +11,19 @@ public class ScoringResult { @JsonProperty("success") - private Boolean success; - + private Boolean success; @JsonProperty("scorers_data") - private List scorersData; - + private List scorersData; @JsonProperty("name") - private String name; - + private String name; @JsonProperty("data_object") - private Object dataObject; - + private Object dataObject; @JsonProperty("trace_id") - private String traceId; - + private String traceId; @JsonProperty("run_duration") - private Double runDuration; - + private Double runDuration; @JsonProperty("evaluation_cost") - private Double evaluationCost; + private Double evaluationCost; private Map additionalProperties = new HashMap<>(); @@ -101,29 +95,21 @@ public void setEvaluationCost(Double evaluationCost) { @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; ScoringResult other = (ScoringResult) obj; - return Objects.equals(success, other.success) - && Objects.equals(scorersData, other.scorersData) - && Objects.equals(name, other.name) - && Objects.equals(dataObject, other.dataObject) - && Objects.equals(traceId, other.traceId) - && Objects.equals(runDuration, other.runDuration) + return Objects.equals(success, other.success) && Objects.equals(scorersData, other.scorersData) + && Objects.equals(name, other.name) && Objects.equals(dataObject, other.dataObject) + && Objects.equals(traceId, other.traceId) && Objects.equals(runDuration, other.runDuration) && Objects.equals(evaluationCost, other.evaluationCost) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash( - success, - scorersData, - name, - dataObject, - traceId, - runDuration, - evaluationCost, + return Objects.hash(success, scorersData, name, dataObject, traceId, runDuration, evaluationCost, Objects.hashCode(additionalProperties)); } -} +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java index 0961e3d..c19a9f8 100644 --- a/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java +++ b/src/main/java/com/judgmentlabs/judgeval/internal/api/models/TraceEvaluationRun.java @@ -11,31 +11,25 @@ public class TraceEvaluationRun { @JsonProperty("id") - private String id; - + private String id; @JsonProperty("project_name") - private String projectName; - + private String projectName; @JsonProperty("eval_name") - private String evalName; - + private String evalName; @JsonProperty("custom_scorers") - private List customScorers; - + private List customScorers; @JsonProperty("judgment_scorers") - private List judgmentScorers; - + private List judgmentScorers; @JsonProperty("model") - private String model; - + private String model; @JsonProperty("created_at") - private String createdAt; - + private String createdAt; @JsonProperty("trace_and_span_ids") - private List> traceAndSpanIds; - + private List> traceAndSpanIds; @JsonProperty("is_offline") - private Boolean isOffline; + private Boolean isOffline; + @JsonProperty("is_bucket_run") + private Boolean isBucketRun; private Map additionalProperties = new HashMap<>(); @@ -85,6 +79,10 @@ public Boolean getIsOffline() { return isOffline; } + public Boolean getIsBucketRun() { + return isBucketRun; + } + public void setId(String id) { this.id = id; } @@ -121,35 +119,28 @@ public void setIsOffline(Boolean isOffline) { this.isOffline = isOffline; } + public void setIsBucketRun(Boolean isBucketRun) { + this.isBucketRun = isBucketRun; + } + @Override public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; TraceEvaluationRun other = (TraceEvaluationRun) obj; - return Objects.equals(id, other.id) - && Objects.equals(projectName, other.projectName) - && Objects.equals(evalName, other.evalName) - && Objects.equals(customScorers, other.customScorers) - && Objects.equals(judgmentScorers, other.judgmentScorers) - && Objects.equals(model, other.model) - && Objects.equals(createdAt, other.createdAt) - && Objects.equals(traceAndSpanIds, other.traceAndSpanIds) - && Objects.equals(isOffline, other.isOffline) + return Objects.equals(id, other.id) && Objects.equals(projectName, other.projectName) + && Objects.equals(evalName, other.evalName) && Objects.equals(customScorers, other.customScorers) + && Objects.equals(judgmentScorers, other.judgmentScorers) && Objects.equals(model, other.model) + && Objects.equals(createdAt, other.createdAt) && Objects.equals(traceAndSpanIds, other.traceAndSpanIds) + && Objects.equals(isOffline, other.isOffline) && Objects.equals(isBucketRun, other.isBucketRun) && Objects.equals(additionalProperties, other.additionalProperties); } @Override public int hashCode() { - return Objects.hash( - id, - projectName, - evalName, - customScorers, - judgmentScorers, - model, - createdAt, - traceAndSpanIds, - isOffline, - Objects.hashCode(additionalProperties)); - } -} + return Objects.hash(id, projectName, evalName, customScorers, judgmentScorers, model, createdAt, + traceAndSpanIds, isOffline, isBucketRun, Objects.hashCode(additionalProperties)); + } +} \ No newline at end of file diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java index a11e6f1..6ad60e3 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/APIScorer.java @@ -3,16 +3,17 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import com.fasterxml.jackson.annotation.JsonIgnore; import com.judgmentlabs.judgeval.data.APIScorerType; import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; -public class APIScorer extends com.judgmentlabs.judgeval.internal.api.models.BaseScorer - implements BaseScorer { +public class APIScorer extends com.judgmentlabs.judgeval.internal.api.models.BaseScorer implements BaseScorer { private APIScorerType scoreType; - @JsonIgnore private List requiredParams; + @JsonIgnore + private List requiredParams; public APIScorer(APIScorerType scoreType) { super(); @@ -27,8 +28,7 @@ public APIScorer(APIScorerType scoreType) { public void setThreshold(double threshold) { if (threshold < 0 || threshold > 1) { - throw new IllegalArgumentException( - "Threshold must be between 0 and 1, got: " + threshold); + throw new IllegalArgumentException("Threshold must be between 0 and 1, got: " + threshold); } super.setThreshold(threshold); } @@ -47,20 +47,21 @@ public void setRequiredParams(List requiredParams) { @Override public Double getThreshold() { - Double threshold = super.getThreshold(); - return threshold != null ? threshold : 0.5; + return Optional.ofNullable(super.getThreshold()) + .orElse(0.5); } @Override public String getName() { - Object name = super.getName(); - return name != null ? name.toString() : null; + return Optional.ofNullable(super.getName()) + .map(Object::toString) + .orElse(null); } @Override public Boolean getStrictMode() { - Boolean strictMode = super.getStrictMode(); - return strictMode != null ? strictMode : false; + return Optional.ofNullable(super.getStrictMode()) + .orElse(false); } @Override @@ -72,7 +73,8 @@ public ScorerConfig getScorerConfig() { cfg.setStrictMode(getStrictMode()); cfg.setRequiredParams(getRequiredParams()); Map kwargs = new HashMap<>(); - if (getAdditionalProperties() != null) kwargs.putAll(getAdditionalProperties()); + if (getAdditionalProperties() != null) + kwargs.putAll(getAdditionalProperties()); cfg.setKwargs(kwargs); return cfg; } @@ -86,7 +88,8 @@ public static final class Builder { private Builder(Class scorerClass) { try { - this.scorer = scorerClass.getDeclaredConstructor().newInstance(); + this.scorer = scorerClass.getDeclaredConstructor() + .newInstance(); } catch (Exception e) { throw new RuntimeException("Failed to create scorer instance", e); } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java index ce73512..3a5b231 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/BaseScorer.java @@ -3,8 +3,8 @@ import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig; /** - * Minimal interface for scorers used by BaseTracer. Only requires the essential methods needed for - * evaluation. + * Minimal interface for scorers used by BaseTracer. Only requires the essential + * methods needed for evaluation. */ public interface BaseScorer { /** diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java index 483d4e9..e0ec743 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerCorrectnessScorer.java @@ -20,6 +20,7 @@ public static AnswerCorrectnessScorer create() { } public static AnswerCorrectnessScorer create(double threshold) { - return builder().threshold(threshold).build(); + return builder().threshold(threshold) + .build(); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java index ece195b..7434399 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/AnswerRelevancyScorer.java @@ -20,6 +20,7 @@ public static AnswerRelevancyScorer create() { } public static AnswerRelevancyScorer create(double threshold) { - return builder().threshold(threshold).build(); + return builder().threshold(threshold) + .build(); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/DerailmentScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/DerailmentScorer.java index b949f4f..431728d 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/DerailmentScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/DerailmentScorer.java @@ -17,6 +17,7 @@ public static DerailmentScorer create() { } public static DerailmentScorer create(double threshold) { - return builder().threshold(threshold).build(); + return builder().threshold(threshold) + .build(); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java index dd4ca09..bb53d23 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/FaithfulnessScorer.java @@ -20,6 +20,7 @@ public static FaithfulnessScorer create() { } public static FaithfulnessScorer create(double threshold) { - return builder().threshold(threshold).build(); + return builder().threshold(threshold) + .build(); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java index 96416b8..da71351 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/InstructionAdherenceScorer.java @@ -21,6 +21,7 @@ public static InstructionAdherenceScorer create() { } public static InstructionAdherenceScorer create(double threshold) { - return builder().threshold(threshold).build(); + return builder().threshold(threshold) + .build(); } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java index e0d666f..01f1860 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/BasePromptScorer.java @@ -1,8 +1,10 @@ package com.judgmentlabs.judgeval.scorers.api_scorers.prompt_scorer; -import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import com.judgmentlabs.judgeval.Env; import com.judgmentlabs.judgeval.data.APIScorerType; @@ -10,162 +12,72 @@ import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersRequest; import com.judgmentlabs.judgeval.internal.api.models.FetchPromptScorersResponse; -import com.judgmentlabs.judgeval.internal.api.models.SavePromptScorerRequest; -import com.judgmentlabs.judgeval.internal.api.models.SavePromptScorerResponse; import com.judgmentlabs.judgeval.internal.api.models.ScorerExistsRequest; import com.judgmentlabs.judgeval.internal.api.models.ScorerExistsResponse; import com.judgmentlabs.judgeval.scorers.APIScorer; -import com.judgmentlabs.judgeval.utils.Logger; public abstract class BasePromptScorer extends APIScorer { - protected String prompt; - protected Map options; - protected String judgmentApiKey; - protected String organizationId; - - protected BasePromptScorer( - APIScorerType scoreType, - String name, - String prompt, - double threshold, - Map options, - String judgmentApiKey, - String organizationId) { + private static final Map cache = new ConcurrentHashMap<>(); + + protected String prompt; + protected Map options; + protected String judgmentApiKey; + protected String organizationId; + + protected BasePromptScorer(APIScorerType scoreType, String name, String prompt, double threshold, + Map options, String judgmentApiKey, String organizationId) { super(scoreType); this.prompt = prompt; this.options = options; this.judgmentApiKey = judgmentApiKey; this.organizationId = organizationId; setName(name); - setThreshold(threshold); + super.setThreshold(threshold); } public static boolean scorerExists(String name, String judgmentApiKey, String organizationId) { try { - JudgmentSyncClient client = - new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); + JudgmentSyncClient client = new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); ScorerExistsRequest request = new ScorerExistsRequest(); request.setName(name); ScorerExistsResponse response = client.scorerExists(request); return Boolean.TRUE.equals(response.getExists()); - } catch (JudgmentAPIError e) { - if (e.getStatusCode() == 500) { - throw new JudgmentAPIError( - e.getStatusCode(), - "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " - + e.getMessage()); - } - throw new JudgmentAPIError( - e.getStatusCode(), "Failed to check if scorer exists: " + e.getMessage()); - } catch (IOException | InterruptedException e) { - throw new JudgmentAPIError( - 500, - "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " - + e.getMessage()); + } catch (Exception e) { + throw new JudgmentAPIError(500, "Failed to check if scorer exists: " + e.getMessage()); } } - public static com.judgmentlabs.judgeval.internal.api.models.PromptScorer fetchPromptScorer( - String name, String judgmentApiKey, String organizationId) { + public static com.judgmentlabs.judgeval.internal.api.models.PromptScorer fetchPromptScorer(String name, + String judgmentApiKey, String organizationId) { + CacheKey key = new CacheKey(name, judgmentApiKey, organizationId); + com.judgmentlabs.judgeval.internal.api.models.PromptScorer cached = cache.get(key); + if (cached != null) { + return cached; + } + try { - JudgmentSyncClient client = - new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); + JudgmentSyncClient client = new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); FetchPromptScorersRequest request = new FetchPromptScorersRequest(); request.setNames(java.util.Collections.singletonList(name)); FetchPromptScorersResponse response = client.fetchScorers(request); - if (response.getScorers() == null || response.getScorers().isEmpty()) { - throw new JudgmentAPIError( - 404, "Failed to fetch prompt scorer '" + name + "': not found"); - } + com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorer = Optional.ofNullable(response) + .map(FetchPromptScorersResponse::getScorers) + .filter(scorers -> scorers != null && !scorers.isEmpty()) + .map(scorers -> scorers.get(0)) + .orElseThrow( + () -> new JudgmentAPIError(404, "Failed to fetch prompt scorer '" + name + "': not found")); - return response.getScorers().get(0); + cache.put(key, scorer); + return scorer; } catch (JudgmentAPIError e) { - if (e.getStatusCode() == 500) { - throw new JudgmentAPIError( - e.getStatusCode(), - "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " - + e.getMessage()); - } - throw new JudgmentAPIError( - e.getStatusCode(), - "Failed to fetch prompt scorer '" + name + "': " + e.getMessage()); - } catch (IOException | InterruptedException e) { - throw new JudgmentAPIError( - 500, - "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " - + e.getMessage()); + throw e; + } catch (Exception e) { + throw new JudgmentAPIError(500, "Failed to fetch prompt scorer '" + name + "': " + e.getMessage()); } } - public static String pushPromptScorer( - String name, - String prompt, - double threshold, - Map options, - String judgmentApiKey, - String organizationId, - Boolean isTrace) { - try { - JudgmentSyncClient client = - new JudgmentSyncClient(Env.JUDGMENT_API_URL, judgmentApiKey, organizationId); - SavePromptScorerRequest request = new SavePromptScorerRequest(); - request.setName(name); - request.setPrompt(prompt); - request.setThreshold(threshold); - Map apiOptions = null; - if (options != null) { - apiOptions = new HashMap<>(); - for (Map.Entry entry : options.entrySet()) { - apiOptions.put(entry.getKey(), entry.getValue()); - } - } - request.setOptions(apiOptions); - request.setIsTrace(isTrace); - - SavePromptScorerResponse response = client.saveScorer(request); - return response != null ? response.getName() : null; - } catch (JudgmentAPIError e) { - if (e.getStatusCode() == 500) { - throw new JudgmentAPIError( - e.getStatusCode(), - "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " - + e.getMessage()); - } - throw new JudgmentAPIError( - e.getStatusCode(), "Failed to save prompt scorer: " + e.getMessage()); - } catch (IOException | InterruptedException e) { - throw new JudgmentAPIError( - 500, - "The server is temporarily unavailable. Please try your request again in a few moments. Error details: " - + e.getMessage()); - } - } - - public void setThreshold(double threshold) { - super.setThreshold(threshold); - pushPromptScorer(); - } - - public void setPrompt(String prompt) { - this.prompt = prompt; - pushPromptScorer(); - Logger.info("Successfully updated prompt for " + getName()); - } - - public void setOptions(Map options) { - this.options = options; - pushPromptScorer(); - Logger.info("Successfully updated options for " + getName()); - } - - public void appendToPrompt(String promptAddition) { - this.prompt += promptAddition; - pushPromptScorer(); - Logger.info("Successfully appended to prompt for " + getName()); - } - public Double getThreshold() { return super.getThreshold(); } @@ -175,7 +87,9 @@ public String getPrompt() { } public Map getOptions() { - return options != null ? new HashMap<>(options) : null; + return Optional.ofNullable(options) + .map(HashMap::new) + .orElse(null); } public String getScorerName() { @@ -191,29 +105,39 @@ public Map getConfig() { return config; } - protected void pushPromptScorer() { - pushPromptScorer( - getName(), - prompt, - getThreshold(), - options, - judgmentApiKey, - organizationId, - isTrace()); - } - protected abstract boolean isTrace(); + private static final class CacheKey { + private final String name; + private final String apiKey; + private final String organizationId; + + CacheKey(String name, String apiKey, String organizationId) { + this.name = name; + this.apiKey = apiKey; + this.organizationId = organizationId; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; + CacheKey that = (CacheKey) obj; + return Objects.equals(name, that.name) && Objects.equals(apiKey, that.apiKey) + && Objects.equals(organizationId, that.organizationId); + } + + @Override + public int hashCode() { + return Objects.hash(name, apiKey, organizationId); + } + } + @Override public String toString() { - return "PromptScorer(name=" - + getName() - + ", prompt=" - + prompt - + ", threshold=" - + getThreshold() - + ", options=" - + options - + ")"; + return "PromptScorer(name=" + getName() + ", prompt=" + prompt + ", threshold=" + getThreshold() + + ", options=" + options + ")"; } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java index d01cc96..7ebd225 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/PromptScorer.java @@ -2,6 +2,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.Optional; import com.judgmentlabs.judgeval.Env; import com.judgmentlabs.judgeval.data.APIScorerType; @@ -11,31 +12,13 @@ public class PromptScorer extends BasePromptScorer { public PromptScorer(String name, String prompt, double threshold, Map options) { - super( - APIScorerType.PROMPT_SCORER, - name, - prompt, - threshold, - options, - Env.JUDGMENT_API_KEY, + super(APIScorerType.PROMPT_SCORER, name, prompt, threshold, options, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); } - public PromptScorer( - String name, - String prompt, - double threshold, - Map options, - String judgmentApiKey, - String organizationId) { - super( - APIScorerType.PROMPT_SCORER, - name, - prompt, - threshold, - options, - judgmentApiKey, - organizationId); + public PromptScorer(String name, String prompt, double threshold, Map options, + String judgmentApiKey, String organizationId) { + super(APIScorerType.PROMPT_SCORER, name, prompt, threshold, options, judgmentApiKey, organizationId); } public static PromptScorer get(String name) { @@ -43,8 +26,8 @@ public static PromptScorer get(String name) { } public static PromptScorer get(String name, String judgmentApiKey, String organizationId) { - com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorerConfig = - fetchPromptScorer(name, judgmentApiKey, organizationId); + com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorerConfig = fetchPromptScorer(name, + judgmentApiKey, organizationId); if (Boolean.TRUE.equals(scorerConfig.getIsTrace())) { throw new JudgmentAPIError(400, "Scorer with name " + name + " is not a PromptScorer"); @@ -64,47 +47,8 @@ public static PromptScorer get(String name, String judgmentApiKey, String organi } } - return new PromptScorer( - name, - scorerConfig.getPrompt(), - scorerConfig.getThreshold() != null ? scorerConfig.getThreshold() : 0.5, - options, - judgmentApiKey, - organizationId); - } - - public static PromptScorer create(String name, String prompt) { - return create(name, prompt, 0.5, null); - } - - public static PromptScorer create(String name, String prompt, double threshold) { - return create(name, prompt, threshold, null); - } - - public static PromptScorer create( - String name, String prompt, double threshold, Map options) { - return create(name, prompt, threshold, options, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); - } - - public static PromptScorer create( - String name, - String prompt, - double threshold, - Map options, - String judgmentApiKey, - String organizationId) { - if (!scorerExists(name, judgmentApiKey, organizationId)) { - pushPromptScorer( - name, prompt, threshold, options, judgmentApiKey, organizationId, false); - return new PromptScorer( - name, prompt, threshold, options, judgmentApiKey, organizationId); - } else { - throw new JudgmentAPIError( - 400, - "Scorer with name " - + name - + " already exists. Either use the existing scorer with the get() method or use a new name."); - } + return new PromptScorer(name, scorerConfig.getPrompt(), Optional.ofNullable(scorerConfig.getThreshold()) + .orElse(0.5), options, judgmentApiKey, organizationId); } @Override @@ -132,73 +76,4 @@ protected boolean isTrace() { return false; } - public static Builder builder() { - return new Builder(); - } - - public static Builder builder(String name, String prompt) { - return new Builder(name, prompt); - } - - public static final class Builder { - private String name; - private String prompt; - private double threshold = 0.5; - private Map options; - private String judgmentApiKey = Env.JUDGMENT_API_KEY; - private String organizationId = Env.JUDGMENT_ORG_ID; - - private Builder() {} - - private Builder(String name, String prompt) { - this.name = name; - this.prompt = prompt; - } - - public Builder name(String name) { - this.name = name; - return this; - } - - public Builder prompt(String prompt) { - this.prompt = prompt; - return this; - } - - public Builder threshold(double threshold) { - this.threshold = threshold; - return this; - } - - public Builder options(Map options) { - this.options = options; - return this; - } - - public Builder option(String key, Double value) { - if (this.options == null) { - this.options = new HashMap<>(); - } - this.options.put(key, value); - return this; - } - - public Builder judgmentApiKey(String judgmentApiKey) { - this.judgmentApiKey = judgmentApiKey; - return this; - } - - public Builder organizationId(String organizationId) { - this.organizationId = organizationId; - return this; - } - - public PromptScorer build() { - if (name == null || prompt == null) { - throw new IllegalArgumentException("Name and prompt are required"); - } - return new PromptScorer( - name, prompt, threshold, options, judgmentApiKey, organizationId); - } - } } diff --git a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java index 94d0b03..fe813bb 100644 --- a/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java +++ b/src/main/java/com/judgmentlabs/judgeval/scorers/api_scorers/prompt_scorer/TracePromptScorer.java @@ -2,6 +2,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.Optional; import com.judgmentlabs.judgeval.Env; import com.judgmentlabs.judgeval.data.APIScorerType; @@ -10,33 +11,14 @@ public class TracePromptScorer extends BasePromptScorer { - public TracePromptScorer( - String name, String prompt, double threshold, Map options) { - super( - APIScorerType.TRACE_PROMPT_SCORER, - name, - prompt, - threshold, - options, - Env.JUDGMENT_API_KEY, + public TracePromptScorer(String name, String prompt, double threshold, Map options) { + super(APIScorerType.TRACE_PROMPT_SCORER, name, prompt, threshold, options, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); } - public TracePromptScorer( - String name, - String prompt, - double threshold, - Map options, - String judgmentApiKey, - String organizationId) { - super( - APIScorerType.TRACE_PROMPT_SCORER, - name, - prompt, - threshold, - options, - judgmentApiKey, - organizationId); + public TracePromptScorer(String name, String prompt, double threshold, Map options, + String judgmentApiKey, String organizationId) { + super(APIScorerType.TRACE_PROMPT_SCORER, name, prompt, threshold, options, judgmentApiKey, organizationId); } public static TracePromptScorer get(String name) { @@ -44,12 +26,11 @@ public static TracePromptScorer get(String name) { } public static TracePromptScorer get(String name, String judgmentApiKey, String organizationId) { - com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorerConfig = - fetchPromptScorer(name, judgmentApiKey, organizationId); + com.judgmentlabs.judgeval.internal.api.models.PromptScorer scorerConfig = fetchPromptScorer(name, + judgmentApiKey, organizationId); if (!Boolean.TRUE.equals(scorerConfig.getIsTrace())) { - throw new JudgmentAPIError( - 400, "Scorer with name " + name + " is not a TracePromptScorer"); + throw new JudgmentAPIError(400, "Scorer with name " + name + " is not a TracePromptScorer"); } Map options = null; @@ -66,47 +47,10 @@ public static TracePromptScorer get(String name, String judgmentApiKey, String o } } - return new TracePromptScorer( - name, - scorerConfig.getPrompt(), - scorerConfig.getThreshold() != null ? scorerConfig.getThreshold() : 0.5, - options, - judgmentApiKey, - organizationId); - } - - public static TracePromptScorer create(String name, String prompt) { - return create(name, prompt, 0.5, null); - } - - public static TracePromptScorer create(String name, String prompt, double threshold) { - return create(name, prompt, threshold, null); - } - - public static TracePromptScorer create( - String name, String prompt, double threshold, Map options) { - return create(name, prompt, threshold, options, Env.JUDGMENT_API_KEY, Env.JUDGMENT_ORG_ID); - } - - public static TracePromptScorer create( - String name, - String prompt, - double threshold, - Map options, - String judgmentApiKey, - String organizationId) { - if (!scorerExists(name, judgmentApiKey, organizationId)) { - pushPromptScorer( - name, prompt, threshold, options, judgmentApiKey, organizationId, true); - return new TracePromptScorer( - name, prompt, threshold, options, judgmentApiKey, organizationId); - } else { - throw new JudgmentAPIError( - 400, - "Scorer with name " - + name - + " already exists. Either use the existing scorer with the get() method or use a new name."); - } + return new TracePromptScorer(name, scorerConfig.getPrompt(), + Optional.ofNullable(scorerConfig.getThreshold()) + .orElse(0.5), + options, judgmentApiKey, organizationId); } @Override @@ -133,74 +77,4 @@ public ScorerConfig getScorerConfig() { protected boolean isTrace() { return true; } - - public static Builder builder() { - return new Builder(); - } - - public static Builder builder(String name, String prompt) { - return new Builder(name, prompt); - } - - public static final class Builder { - private String name; - private String prompt; - private double threshold = 0.5; - private Map options; - private String judgmentApiKey = Env.JUDGMENT_API_KEY; - private String organizationId = Env.JUDGMENT_ORG_ID; - - private Builder() {} - - private Builder(String name, String prompt) { - this.name = name; - this.prompt = prompt; - } - - public Builder name(String name) { - this.name = name; - return this; - } - - public Builder prompt(String prompt) { - this.prompt = prompt; - return this; - } - - public Builder threshold(double threshold) { - this.threshold = threshold; - return this; - } - - public Builder options(Map options) { - this.options = options; - return this; - } - - public Builder option(String key, Double value) { - if (this.options == null) { - this.options = new HashMap<>(); - } - this.options.put(key, value); - return this; - } - - public Builder judgmentApiKey(String judgmentApiKey) { - this.judgmentApiKey = judgmentApiKey; - return this; - } - - public Builder organizationId(String organizationId) { - this.organizationId = organizationId; - return this; - } - - public TracePromptScorer build() { - if (name == null || prompt == null) { - throw new IllegalArgumentException("Name and prompt are required"); - } - return new TracePromptScorer( - name, prompt, threshold, options, judgmentApiKey, organizationId); - } - } } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java index 36b8773..ea08614 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/BaseTracer.java @@ -10,11 +10,11 @@ import com.google.gson.Gson; import com.judgmentlabs.judgeval.Env; import com.judgmentlabs.judgeval.data.Example; -import com.judgmentlabs.judgeval.data.ExampleEvaluationRun; -import com.judgmentlabs.judgeval.data.TraceEvaluationRun; import com.judgmentlabs.judgeval.internal.api.JudgmentSyncClient; import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameRequest; import com.judgmentlabs.judgeval.internal.api.models.ResolveProjectNameResponse; +import com.judgmentlabs.judgeval.internal.api.models.TraceEvaluationRun; +import com.judgmentlabs.judgeval.internal.api.models.ExampleEvaluationRun; import com.judgmentlabs.judgeval.scorers.BaseScorer; import com.judgmentlabs.judgeval.tracer.exporters.JudgmentSpanExporter; import com.judgmentlabs.judgeval.tracer.exporters.NoOpSpanExporter; @@ -23,300 +23,544 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Scope; import io.opentelemetry.sdk.trace.export.SpanExporter; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.UUID; + public abstract class BaseTracer { - public static final String TRACER_NAME = "judgeval"; + public static final String TRACER_NAME = "judgeval"; protected final TracerConfiguration configuration; - protected final JudgmentSyncClient apiClient; - protected final ISerializer serializer; - protected final ObjectMapper jacksonMapper; - protected final String projectId; + protected final JudgmentSyncClient apiClient; + protected final ISerializer serializer; + protected final ObjectMapper jacksonMapper; + protected final Optional projectId; - protected BaseTracer( - TracerConfiguration configuration, ISerializer serializer, boolean initialize) { + protected BaseTracer(TracerConfiguration configuration, ISerializer serializer, boolean initialize) { this.configuration = Objects.requireNonNull(configuration, "Configuration cannot be null"); - this.apiClient = - new JudgmentSyncClient( - configuration.apiUrl(), - configuration.apiKey(), - configuration.organizationId()); + this.apiClient = new JudgmentSyncClient(configuration.apiUrl(), configuration.apiKey(), + configuration.organizationId()); this.serializer = Objects.requireNonNull(serializer, "Serializer cannot be null"); this.jacksonMapper = new ObjectMapper(); this.projectId = resolveProjectId(configuration.projectName()); - if (this.projectId == null) { - Logger.error( - "Failed to resolve project " - + configuration.projectName() - + ", please create it first at https://app.judgmentlabs.ai/org/" - + configuration.organizationId() - + "/projects. Skipping Judgment export."); - } + this.projectId.ifPresentOrElse(id -> { + }, () -> Logger.error("Failed to resolve project " + configuration.projectName() + + ", please create it first at https://app.judgmentlabs.ai/org/" + configuration.organizationId() + + "/projects. Skipping Judgment export.")); if (initialize) { initialize(); } } + /** + * Initializes the tracer with OpenTelemetry SDK configuration and span + * exporters. Must be implemented by subclasses. + */ public abstract void initialize(); /** - * Gets the SpanExporter for OpenTelemetry integration. Returns NoOpSpanExporter if project ID - * cannot be resolved. + * Gets the span exporter for sending traces to the Judgment Labs backend. + * Returns a NoOpSpanExporter if the project ID is not resolved. + * + * @return the configured SpanExporter instance */ public SpanExporter getSpanExporter() { - if (projectId == null) { - Logger.error( - "Project not resolved; cannot create exporter, returning NoOpSpanExporter"); - return new NoOpSpanExporter(); - } - return createJudgmentSpanExporter(projectId); + return projectId.map(this::createJudgmentSpanExporter) + .orElseGet(() -> { + Logger.error("Project not resolved; cannot create exporter, returning NoOpSpanExporter"); + return new NoOpSpanExporter(); + }); } - /** Sets the span kind attribute on the current span. Common kinds: "span", "llm", "tool". */ + /** + * Sets the kind of the current span (e.g., "llm", "tool", "span"). + * + * @param kind + * the span kind to set, ignored if null + */ public void setSpanKind(String kind) { + Optional.ofNullable(kind) + .ifPresent(k -> withCurrentSpan( + span -> span.setAttribute(JudgevalTraceKeys.AttributeKeys.JUDGMENT_SPAN_KIND, k))); + } + + private static void withCurrentSpan(java.util.function.Consumer action) { Optional.ofNullable(Span.current()) - .filter(span -> kind != null) - .ifPresent( - span -> - span.setAttribute( - OpenTelemetryKeys.AttributeKeys.JUDGMENT_SPAN_KIND, kind)); + .ifPresent(action); + } + + private static boolean isValidKey(String key) { + return key != null && !key.isEmpty(); } - /** Sets a custom attribute on the current span. Value is serialized to JSON. */ + /** + * Sets an attribute on the current span by serializing the object value. + * Empty strings and null are valid attribute values, but not valid keys. + * + * @param key + * the attribute key + * @param value + * the attribute value (null and empty strings are valid) + */ public void setAttribute(String key, Object value) { - Optional.ofNullable(Span.current()) - .ifPresent(span -> span.setAttribute(key, serializer.serialize(value))); + if (!isValidKey(key)) { + return; + } + if (value != null) { + setAttribute(key, value, value.getClass()); + } } - /** Sets a custom attribute on the current span with specific type for JSON serialization. */ + /** + * Sets an attribute on the current span by serializing the object value with + * the specified type. Empty strings and null are valid attribute values, but + * not valid keys. + * + * @param key + * the attribute key + * @param value + * the attribute value (null and empty strings are valid) + * @param type + * the type to use for serialization + */ public void setAttribute(String key, Object value, Type type) { - Optional.ofNullable(Span.current()) - .ifPresent(span -> span.setAttribute(key, serializer.serialize(value, type))); + if (!isValidKey(key)) { + return; + } + if (value != null) { + withCurrentSpan(span -> span.setAttribute(key, serializer.serialize(value, type))); + } } - /** Asynchronously evaluates a scorer with an example using current trace context. */ - public void asyncEvaluate(BaseScorer scorer, Example example, String model) { + /** + * Sets a string attribute on the current span. Empty strings and null are valid + * attribute values, but not valid keys. + * + * @param key + * the attribute key + * @param value + * the string value (null and empty strings are valid) + */ + public void setAttribute(String key, String value) { + if (!isValidKey(key)) { + return; + } + withCurrentSpan(span -> span.setAttribute(key, value)); + } + + /** + * Sets a long attribute on the current span. + * + * @param key + * the attribute key + * @param value + * the long value + */ + public void setAttribute(String key, long value) { + if (!isValidKey(key)) { + return; + } + withCurrentSpan(span -> span.setAttribute(key, value)); + } + + /** + * Sets a double attribute on the current span. + * + * @param key + * the attribute key + * @param value + * the double value + */ + public void setAttribute(String key, double value) { + if (!isValidKey(key)) { + return; + } + withCurrentSpan(span -> span.setAttribute(key, value)); + } + + /** + * Sets a boolean attribute on the current span. + * + * @param key + * the attribute key + * @param value + * the boolean value + */ + public void setAttribute(String key, boolean value) { + if (!isValidKey(key)) { + return; + } + withCurrentSpan(span -> span.setAttribute(key, value)); + } + + private Optional getSampledSpanContext() { + return Optional.ofNullable(Span.current()) + .filter(span -> span.getSpanContext() + .isSampled()) + .map(Span::getSpanContext); + } + + private Optional getSampledSpan() { + return Optional.ofNullable(Span.current()) + .filter(span -> span.getSpanContext() + .isSampled()); + } + + private boolean isEvaluationEnabled() { + return configuration.enableEvaluation(); + } + + private void logEvaluationInfo(String method, String traceId, String spanId, String scorerName) { + Logger.info(method + ": project=" + configuration.projectName() + ", traceId=" + traceId + ", spanId=" + + spanId + ", scorer=" + scorerName); + } + + private void safeExecute(String operation, Runnable action) { try { - if (!configuration.enableEvaluation()) { - return; - } + action.run(); + } catch (Exception e) { + Logger.error("Failed to " + operation + ": " + e.getMessage()); + } + } - Span currentSpan = Span.current(); - if (currentSpan == null || !currentSpan.getSpanContext().isSampled()) { + /** + * Asynchronously evaluates a scorer against an example, associating it with the + * current trace. + * + * @param scorer + * the scorer to evaluate + * @param example + * the example to evaluate against + * @param model + * the model name, or null to use the default + */ + public void asyncEvaluate(BaseScorer scorer, Example example, String model) { + safeExecute("evaluate scorer", () -> { + if (!isEvaluationEnabled()) { return; } - SpanContext spanContext = currentSpan.getSpanContext(); - String traceId = spanContext.getTraceId(); - String spanId = spanContext.getSpanId(); - - Logger.info( - "asyncEvaluate: project=" - + configuration.projectName() - + ", traceId=" - + traceId - + ", spanId=" - + spanId - + ", scorer=" - + scorer.getName()); - - ExampleEvaluationRun evaluationRun = - createEvaluationRun(scorer, example, model, traceId, spanId); - enqueueEvaluation(evaluationRun); - } catch (Exception e) { - Logger.error("Failed to evaluate scorer: " + e.getMessage()); - } + getSampledSpanContext().ifPresent(spanContext -> { + String traceId = spanContext.getTraceId(); + String spanId = spanContext.getSpanId(); + + logEvaluationInfo("asyncEvaluate", traceId, spanId, scorer.getName()); + + ExampleEvaluationRun evaluationRun = createEvaluationRun(scorer, example, model, traceId, spanId); + enqueueEvaluation(evaluationRun); + }); + }); } + /** + * Asynchronously evaluates a scorer against an example using the default model. + * + * @param scorer + * the scorer to evaluate + * @param example + * the example to evaluate against + */ public void asyncEvaluate(BaseScorer scorer, Example example) { asyncEvaluate(scorer, example, null); } - /** Asynchronously evaluates a scorer with trace context. Sets evaluation as span attribute. */ + /** + * Asynchronously evaluates a scorer for the current trace, attaching the + * evaluation as a span attribute. + * + * @param scorer + * the scorer to evaluate + * @param model + * the model name, or null to use the default + */ public void asyncTraceEvaluate(BaseScorer scorer, String model) { - try { - if (!configuration.enableEvaluation()) { - return; - } - - Span currentSpan = Span.current(); - if (currentSpan == null || !currentSpan.getSpanContext().isSampled()) { + safeExecute("evaluate trace scorer", () -> { + if (!isEvaluationEnabled()) { return; } - SpanContext spanContext = currentSpan.getSpanContext(); - String traceId = spanContext.getTraceId(); - String spanId = spanContext.getSpanId(); - - Logger.info( - "asyncTraceEvaluate: project=" - + configuration.projectName() - + ", traceId=" - + traceId - + ", spanId=" - + spanId - + ", scorer=" - + scorer.getName()); - - TraceEvaluationRun evaluationRun = - createTraceEvaluationRun(scorer, model, traceId, spanId); - try { - String traceEvalJson = jacksonMapper.writeValueAsString(evaluationRun); - currentSpan.setAttribute( - OpenTelemetryKeys.AttributeKeys.PENDING_TRACE_EVAL, traceEvalJson); - } catch (Exception e) { - Logger.error("Failed to serialize trace evaluation: " + e.getMessage()); - } - } catch (Exception e) { - Logger.error("Failed to evaluate trace scorer: " + e.getMessage()); - } + getSampledSpan().ifPresent(currentSpan -> { + SpanContext spanContext = currentSpan.getSpanContext(); + String traceId = spanContext.getTraceId(); + String spanId = spanContext.getSpanId(); + + logEvaluationInfo("asyncTraceEvaluate", traceId, spanId, scorer.getName()); + + TraceEvaluationRun evaluationRun = createTraceEvaluationRun(scorer, model, traceId, spanId); + try { + String traceEvalJson = jacksonMapper.writeValueAsString(evaluationRun); + currentSpan.setAttribute(JudgevalTraceKeys.AttributeKeys.PENDING_TRACE_EVAL, traceEvalJson); + } catch (Exception e) { + Logger.error("Failed to serialize trace evaluation: " + e.getMessage()); + } + }); + }); } + /** + * Asynchronously evaluates a scorer for the current trace using the default + * model. + * + * @param scorer + * the scorer to evaluate + */ public void asyncTraceEvaluate(BaseScorer scorer) { asyncTraceEvaluate(scorer, null); } + /** + * Sets multiple attributes on the current span from a map. + * + * @param attributes + * the map of attribute key-value pairs, ignored if null + */ public void setAttributes(Map attributes) { - if (attributes == null) { - return; - } - Optional.ofNullable(Span.current()) - .ifPresent( - span -> - attributes.forEach( - (key, value) -> - span.setAttribute( - key, serializer.serialize(value)))); + Optional.ofNullable(attributes) + .ifPresent(attrs -> attrs.forEach(this::setAttribute)); } + /** + * Sets the current span kind to "llm". + */ public void setLLMSpan() { setSpanKind("llm"); } + /** + * Sets the current span kind to "tool". + */ public void setToolSpan() { setSpanKind("tool"); } + /** + * Sets the current span kind to "span". + */ public void setGeneralSpan() { setSpanKind("span"); } + /** + * Sets the input attribute on the current span by serializing the object. + * + * @param input + * the input object to set, ignored if null + */ public void setInput(Object input) { - setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_INPUT, input); + setAttribute(JudgevalTraceKeys.AttributeKeys.JUDGMENT_INPUT, input); } + /** + * Sets the output attribute on the current span by serializing the object. + * + * @param output + * the output object to set, ignored if null + */ public void setOutput(Object output) { - setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_OUTPUT, output); + setAttribute(JudgevalTraceKeys.AttributeKeys.JUDGMENT_OUTPUT, output); } + /** + * Sets the input attribute on the current span with a specific type for + * serialization. + * + * @param input + * the input object to set, ignored if null + * @param type + * the type to use for serialization + */ public void setInput(Object input, Type type) { - setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_INPUT, input, type); + setAttribute(JudgevalTraceKeys.AttributeKeys.JUDGMENT_INPUT, input, type); } + /** + * Sets the output attribute on the current span with a specific type for + * serialization. + * + * @param output + * the output object to set, ignored if null + * @param type + * the type to use for serialization + */ public void setOutput(Object output, Type type) { - setAttribute(OpenTelemetryKeys.AttributeKeys.JUDGMENT_OUTPUT, output, type); + setAttribute(JudgevalTraceKeys.AttributeKeys.JUDGMENT_OUTPUT, output, type); } /** - * Creates a new span with the given name and executes the provided runnable within its scope. - * The span is automatically ended when the runnable completes. + * Creates a new span, executes the runnable within its context, and ends the + * span. + * + * @param spanName + * the name of the span + * @param runnable + * the code to execute within the span context */ public void span(String spanName, Runnable runnable) { - Span span = getTracer().spanBuilder(spanName).startSpan(); + Span span = getTracer().spanBuilder(spanName) + .startSpan(); try (Scope scope = span.makeCurrent()) { runnable.run(); + } catch (Exception e) { + span.setStatus(StatusCode.ERROR).recordException(e); + throw e; } finally { span.end(); } } /** - * Creates a new span with the given name and executes the provided callable within its scope. - * The span is automatically ended when the callable completes. Returns the result of the - * callable. + * Creates a new span, executes the callable within its context, and ends the + * span. + * + * @param + * the return type of the callable + * @param spanName + * the name of the span + * @param callable + * the code to execute within the span context + * @return the result of the callable + * @throws Exception + * if the callable throws an exception */ public T span(String spanName, java.util.concurrent.Callable callable) throws Exception { - Span span = getTracer().spanBuilder(spanName).startSpan(); + Span span = getTracer().spanBuilder(spanName) + .startSpan(); try (Scope scope = span.makeCurrent()) { return callable.call(); + } catch (Exception e) { + span.setStatus(StatusCode.ERROR).recordException(e); + throw e; } finally { span.end(); } } - /** Gets the OpenTelemetry tracer instance. */ + /** + * Gets the OpenTelemetry tracer instance. + * + * @return the configured Tracer instance + */ public Tracer getTracer() { - return GlobalOpenTelemetry.get().getTracer(TRACER_NAME); + return GlobalOpenTelemetry.get() + .getTracer(TRACER_NAME); } /** - * Creates a new span with the given name. The caller is responsible for ending the span - * manually. + * Creates and returns a new span with the given name. The span must be ended + * manually by calling {@link Span#end()}. + * + * @param spanName + * the name of the span + * @return the newly created span */ public static Span span(String spanName) { - return GlobalOpenTelemetry.get().getTracer(TRACER_NAME).spanBuilder(spanName).startSpan(); + return GlobalOpenTelemetry.get() + .getTracer(TRACER_NAME) + .spanBuilder(spanName) + .startSpan(); } - protected String resolveProjectId(String name) { + protected Optional resolveProjectId(String name) { try { ResolveProjectNameRequest request = new ResolveProjectNameRequest(); request.setProjectName(name); ResolveProjectNameResponse response = apiClient.projectsResolve(request); - return Optional.ofNullable(response.getProjectId()).map(Object::toString).orElse(null); + return Optional.ofNullable(response.getProjectId()) + .map(Object::toString); } catch (Exception e) { - return null; + return Optional.empty(); } } + private String buildEndpoint(String baseUrl) { + return baseUrl.endsWith("/") ? baseUrl + "otel/v1/traces" : baseUrl + "/otel/v1/traces"; + } + private JudgmentSpanExporter createJudgmentSpanExporter(String projectId) { - String endpoint = - configuration.apiUrl().endsWith("/") - ? configuration.apiUrl() + "otel/v1/traces" - : configuration.apiUrl() + "/otel/v1/traces"; return JudgmentSpanExporter.builder() - .endpoint(endpoint) + .endpoint(buildEndpoint(configuration.apiUrl())) .apiKey(configuration.apiKey()) .organizationId(configuration.organizationId()) .projectId(projectId) .build(); } - private ExampleEvaluationRun createEvaluationRun( - BaseScorer scorer, Example example, String model, String traceId, String spanId) { - String runId = "async_evaluate_" + (spanId != null ? spanId : System.currentTimeMillis()); - String modelName = model != null ? model : Env.JUDGMENT_DEFAULT_GPT_MODEL; - - ExampleEvaluationRun evaluationRun = - new ExampleEvaluationRun( - configuration.projectName(), - runId, - List.of(example), - List.of(scorer.getScorerConfig()), - modelName, - configuration.organizationId()); - evaluationRun.setTraceId(traceId); - evaluationRun.setTraceSpanId(spanId); - return evaluationRun; - } - - private TraceEvaluationRun createTraceEvaluationRun( - BaseScorer scorer, String model, String traceId, String spanId) { - String evalName = - "async_trace_evaluate_" + (spanId != null ? spanId : System.currentTimeMillis()); - String modelName = model != null ? model : Env.JUDGMENT_DEFAULT_GPT_MODEL; - - return TraceEvaluationRun.builder() - .projectName(configuration.projectName()) - .evalName(evalName) - .scorer(scorer.getScorerConfig()) - .model(modelName) - .organizationId(configuration.organizationId()) - .traceAndSpanId(traceId, spanId) - .build(); + private String generateRunId(String prefix, String spanId) { + return prefix + Optional.ofNullable(spanId) + .orElseGet(() -> String.valueOf(System.currentTimeMillis())); + } + + private String getModelName(String model) { + return Optional.ofNullable(model) + .orElse(Env.JUDGMENT_DEFAULT_GPT_MODEL); + } + + private ExampleEvaluationRun createEvaluationRun(BaseScorer scorer, Example example, String model, String traceId, + String spanId) { + String runId = generateRunId("async_evaluate_", spanId); + String modelName = getModelName(model); + + ExampleEvaluationRun exampleEvaluationRun = new ExampleEvaluationRun(); + exampleEvaluationRun.setProjectName(configuration.projectName()); + exampleEvaluationRun.setEvalName(runId); + exampleEvaluationRun.setJudgmentScorers(List.of(scorer.getScorerConfig())); + exampleEvaluationRun.setModel(modelName); + com.judgmentlabs.judgeval.internal.api.models.Example internalExample = (com.judgmentlabs.judgeval.internal.api.models.Example) example; + exampleEvaluationRun.setExamples(List.of(internalExample)); + exampleEvaluationRun.setTraceId(traceId); + exampleEvaluationRun.setTraceSpanId(spanId); + exampleEvaluationRun.setCustomScorers(new java.util.ArrayList<>()); + exampleEvaluationRun.setId(UUID.randomUUID() + .toString()); + exampleEvaluationRun.setCreatedAt(Instant.now() + .atOffset(ZoneOffset.UTC) + .format(DateTimeFormatter.ISO_INSTANT)); + return exampleEvaluationRun; + } + + private TraceEvaluationRun createTraceEvaluationRun(BaseScorer scorer, String model, String traceId, + String spanId) { + String evalName = generateRunId("async_trace_evaluate_", spanId); + String modelName = getModelName(model); + + TraceEvaluationRun traceEvaluationRun = new TraceEvaluationRun(); + traceEvaluationRun.setProjectName(configuration.projectName()); + traceEvaluationRun.setEvalName(evalName); + traceEvaluationRun.setJudgmentScorers(List.of(scorer.getScorerConfig())); + traceEvaluationRun.setModel(modelName); + traceEvaluationRun.setTraceAndSpanIds(convertTraceAndSpanIds(List.of(List.of(traceId, spanId)))); + traceEvaluationRun.setIsOffline(false); + traceEvaluationRun.setIsBucketRun(false); + traceEvaluationRun.setCustomScorers(new java.util.ArrayList<>()); + traceEvaluationRun.setId(UUID.randomUUID() + .toString()); + traceEvaluationRun.setCreatedAt(Instant.now() + .atOffset(ZoneOffset.UTC) + .format(DateTimeFormatter.ISO_INSTANT)); + + return traceEvaluationRun; + } + + private static List> convertTraceAndSpanIds(List> traceAndSpanIds) { + if (traceAndSpanIds == null || traceAndSpanIds.isEmpty()) { + throw new IllegalArgumentException("Trace and span IDs are required for trace evaluations."); + } + + List> converted = new java.util.ArrayList<>(); + for (List pair : traceAndSpanIds) { + if (pair == null || pair.size() != 2) { + throw new IllegalArgumentException("Each trace and span ID pair must contain exactly 2 elements."); + } + converted.add(List.of(pair.get(0), pair.get(1))); + } + return converted; } private void enqueueEvaluation(ExampleEvaluationRun evaluationRun) { @@ -332,12 +576,21 @@ protected static class GsonSerializer implements ISerializer { @Override public String serialize(Object obj) { - return gson.toJson(obj); + return Optional.ofNullable(obj) + .map(o -> serialize(o, o.getClass())) + .orElse(null); } @Override public String serialize(Object obj, Type type) { - return gson.toJson(obj, type); + try { + return gson.toJson(obj, type); + } catch (Exception e) { + Logger.error("Failed to serialize object: " + e.getMessage()); + return Optional.ofNullable(obj) + .map(Object::toString) + .orElse(null); + } } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/JudgevalTraceKeys.java b/src/main/java/com/judgmentlabs/judgeval/tracer/JudgevalTraceKeys.java new file mode 100644 index 0000000..65e2fd4 --- /dev/null +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/JudgevalTraceKeys.java @@ -0,0 +1,51 @@ +package com.judgmentlabs.judgeval.tracer; + +public final class JudgevalTraceKeys { + public static final class AttributeKeys { + public static final String JUDGMENT_SPAN_KIND = "judgment.span_kind"; + public static final String JUDGMENT_INPUT = "judgment.input"; + public static final String JUDGMENT_OUTPUT = "judgment.output"; + public static final String JUDGMENT_OFFLINE_MODE = "judgment.offline_mode"; + public static final String JUDGMENT_UPDATE_ID = "judgment.update_id"; + public static final String JUDGMENT_CUSTOMER_ID = "judgment.customer_id"; + public static final String JUDGMENT_AGENT_ID = "judgment.agent_id"; + public static final String JUDGMENT_PARENT_AGENT_ID = "judgment.parent_agent_id"; + public static final String JUDGMENT_AGENT_CLASS_NAME = "judgment.agent_class_name"; + public static final String JUDGMENT_AGENT_INSTANCE_NAME = "judgment.agent_instance_name"; + public static final String JUDGMENT_IS_AGENT_ENTRY_POINT = "judgment.is_agent_entry_point"; + public static final String JUDGMENT_CUMULATIVE_LLM_COST = "judgment.cumulative_llm_cost"; + public static final String JUDGMENT_STATE_BEFORE = "judgment.state_before"; + public static final String JUDGMENT_STATE_AFTER = "judgment.state_after"; + public static final String PENDING_TRACE_EVAL = "judgment.pending_trace_eval"; + + public static final String GEN_AI_PROMPT = "gen_ai.prompt"; + public static final String GEN_AI_COMPLETION = "gen_ai.completion"; + public static final String GEN_AI_REQUEST_MODEL = "gen_ai.request.model"; + public static final String GEN_AI_RESPONSE_MODEL = "gen_ai.response.model"; + public static final String GEN_AI_SYSTEM = "gen_ai.system"; + public static final String GEN_AI_USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"; + public static final String GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"; + public static final String GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS = "gen_ai.usage.cache_creation_input_tokens"; + public static final String GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS = "gen_ai.usage.cache_read_input_tokens"; + public static final String GEN_AI_REQUEST_TEMPERATURE = "gen_ai.request.temperature"; + public static final String GEN_AI_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens"; + public static final String GEN_AI_RESPONSE_FINISH_REASONS = "gen_ai.response.finish_reasons"; + + private AttributeKeys() { + } + } + + public static final class ResourceKeys { + public static final String SERVICE_NAME = "service.name"; + public static final String TELEMETRY_SDK_LANGUAGE = "telemetry.sdk.language"; + public static final String TELEMETRY_SDK_NAME = "telemetry.sdk.name"; + public static final String TELEMETRY_SDK_VERSION = "telemetry.sdk.version"; + public static final String JUDGMENT_PROJECT_ID = "judgment.project_id"; + + private ResourceKeys() { + } + } + + private JudgevalTraceKeys() { + } +} diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/OpenTelemetryKeys.java b/src/main/java/com/judgmentlabs/judgeval/tracer/OpenTelemetryKeys.java deleted file mode 100644 index b5016e7..0000000 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/OpenTelemetryKeys.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.judgmentlabs.judgeval.tracer; - -public final class OpenTelemetryKeys { - public static final class AttributeKeys { - public static final String JUDGMENT_SPAN_KIND = "judgment.span_kind"; - public static final String JUDGMENT_INPUT = "judgment.input"; - public static final String JUDGMENT_OUTPUT = "judgment.output"; - public static final String JUDGMENT_OFFLINE_MODE = "judgment.offline_mode"; - public static final String JUDGMENT_UPDATE_ID = "judgment.update_id"; - public static final String JUDGMENT_CUSTOMER_ID = "judgment.customer_id"; - public static final String JUDGMENT_AGENT_ID = "judgment.agent_id"; - public static final String JUDGMENT_PARENT_AGENT_ID = "judgment.parent_agent_id"; - public static final String JUDGMENT_AGENT_CLASS_NAME = "judgment.agent_class_name"; - public static final String JUDGMENT_AGENT_INSTANCE_NAME = "judgment.agent_instance_name"; - public static final String JUDGMENT_IS_AGENT_ENTRY_POINT = "judgment.is_agent_entry_point"; - public static final String JUDGMENT_CUMULATIVE_LLM_COST = "judgment.cumulative_llm_cost"; - public static final String JUDGMENT_STATE_BEFORE = "judgment.state_before"; - public static final String JUDGMENT_STATE_AFTER = "judgment.state_after"; - public static final String PENDING_TRACE_EVAL = "judgment.pending_trace_eval"; - - private AttributeKeys() {} - } - - public static final class ResourceKeys { - public static final String SERVICE_NAME = "service.name"; - public static final String TELEMETRY_SDK_LANGUAGE = "telemetry.sdk.language"; - public static final String TELEMETRY_SDK_NAME = "telemetry.sdk.name"; - public static final String TELEMETRY_SDK_VERSION = "telemetry.sdk.version"; - public static final String JUDGMENT_PROJECT_ID = "judgment.project_id"; - - private ResourceKeys() {} - } - - private OpenTelemetryKeys() {} -} diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java index d5d394a..db8defe 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/Tracer.java @@ -1,5 +1,7 @@ package com.judgmentlabs.judgeval.tracer; +import java.util.Optional; + import com.judgmentlabs.judgeval.Version; import io.opentelemetry.api.GlobalOpenTelemetry; @@ -21,77 +23,123 @@ */ public final class Tracer extends BaseTracer { - private Tracer( - TracerConfiguration configuration, ISerializer serializer, boolean shouldInitialize) { + private Tracer(TracerConfiguration configuration, ISerializer serializer, boolean shouldInitialize) { super(configuration, serializer, shouldInitialize); } + /** + * Creates a new TracerBuilder for constructing a Tracer instance. + * + * @return a new TracerBuilder + */ public static TracerBuilder builder() { return new TracerBuilder(); } + /** + * Creates a Tracer with default configuration for the given project name. + * + * @param projectName + * the name of the project + * @return a new Tracer instance with default configuration + */ public static Tracer createDefault(String projectName) { - return builder().configuration(TracerConfiguration.createDefault(projectName)).build(); + return builder().configuration(TracerConfiguration.createDefault(projectName)) + .build(); } + /** + * Creates a Tracer with the provided configuration. + * + * @param configuration + * the tracer configuration + * @return a new Tracer instance with the given configuration + */ public static Tracer createWithConfiguration(TracerConfiguration configuration) { - return builder().configuration(configuration).build(); + return builder().configuration(configuration) + .build(); } - /** Initializes the OpenTelemetry SDK with batch span processor and registers it globally. */ + /** + * Initializes the OpenTelemetry SDK with batch span processor and registers it + * globally. + */ @Override public void initialize() { SpanExporter spanExporter = getSpanExporter(); - var resource = - Resource.getDefault() - .merge( - Resource.create( - Attributes.builder() - .put("service.name", configuration.projectName()) - .put("telemetry.sdk.name", TRACER_NAME) - .put("telemetry.sdk.version", Version.getVersion()) - .build())); - - SdkTracerProvider tracerProvider = - SdkTracerProvider.builder() - .setResource(resource) - .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build()) - .build(); - - OpenTelemetry openTelemetry = - OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build(); + var resource = Resource.getDefault() + .merge(Resource.create(Attributes.builder() + .put("service.name", configuration.projectName()) + .put("telemetry.sdk.name", TRACER_NAME) + .put("telemetry.sdk.version", Version.getVersion()) + .build())); + + SdkTracerProvider tracerProvider = SdkTracerProvider.builder() + .setResource(resource) + .addSpanProcessor(BatchSpanProcessor.builder(spanExporter) + .build()) + .build(); + + OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .build(); GlobalOpenTelemetry.set(openTelemetry); } - /** Builder for creating Tracer instances. */ public static final class TracerBuilder { private TracerConfiguration configuration; - private ISerializer serializer = new GsonSerializer(); - private boolean initialize = false; - + private ISerializer serializer = new GsonSerializer(); + private boolean initialize = false; + + /** + * Sets the tracer configuration. + * + * @param configuration + * the configuration to use + * @return this builder for method chaining + */ public TracerBuilder configuration(TracerConfiguration configuration) { this.configuration = configuration; return this; } + /** + * Sets the serializer to use for attribute serialization. + * + * @param serializer + * the serializer to use + * @return this builder for method chaining + */ public TracerBuilder serializer(ISerializer serializer) { this.serializer = serializer; return this; } + /** + * Sets whether to automatically initialize the tracer after construction. + * + * @param initialize + * true to initialize automatically, false otherwise + * @return this builder for method chaining + */ public TracerBuilder initialize(boolean initialize) { this.initialize = initialize; return this; } + /** + * Builds a new Tracer instance with the configured settings. + * + * @return a new Tracer instance + * @throws IllegalArgumentException + * if configuration is not set + */ public Tracer build() { - if (configuration == null) { - throw new IllegalArgumentException("Configuration is required"); - } - - return new Tracer(configuration, serializer, initialize); + return Optional.ofNullable(configuration) + .map(config -> new Tracer(config, serializer, initialize)) + .orElseThrow(() -> new IllegalArgumentException("Configuration is required")); } } } diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java b/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java index 14b49d3..bd296ea 100644 --- a/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java +++ b/src/main/java/com/judgmentlabs/judgeval/tracer/TracerConfiguration.java @@ -1,13 +1,15 @@ package com.judgmentlabs.judgeval.tracer; -import java.util.Objects; +import java.util.Optional; /** - * Configuration for the Judgment Tracer that controls how tracing and evaluation behave. - * - *

This class encapsulates all configuration parameters needed to initialize a {@link Tracer}. - * - *

Example usage: + * Configuration for the Judgment Tracer that controls how tracing and + * evaluation behave. + *

+ * This class encapsulates all configuration parameters needed to initialize a + * {@link Tracer}. + *

+ * Example usage: * * 

{@code
  * TracerConfiguration config = TracerConfiguration.builder()
@@ -23,24 +25,24 @@
  * @see Tracer
  */
 public final class TracerConfiguration {
-    private final String projectName;
-    private final String apiKey;
-    private final String organizationId;
-    private final String apiUrl;
+    private final String  projectName;
+    private final String  apiKey;
+    private final String  organizationId;
+    private final String  apiUrl;
     private final boolean enableEvaluation;
 
     private TracerConfiguration(Builder builder) {
-        this.projectName =
-                Objects.requireNonNull(builder.projectName, "Project name cannot be null").trim();
-        this.apiKey = Objects.requireNonNull(builder.apiKey, "API key cannot be null");
-        this.organizationId =
-                Objects.requireNonNull(builder.organizationId, "Organization ID cannot be null");
-        this.apiUrl = Objects.requireNonNull(builder.apiUrl, "API URL cannot be null");
+        this.projectName = Optional.ofNullable(builder.projectName)
+                .map(String::trim)
+                .filter(name -> !name.isEmpty())
+                .orElseThrow(() -> new IllegalArgumentException("Project name cannot be null or empty"));
+        this.apiKey = Optional.ofNullable(builder.apiKey)
+                .orElseThrow(() -> new IllegalArgumentException("API key cannot be null"));
+        this.organizationId = Optional.ofNullable(builder.organizationId)
+                .orElseThrow(() -> new IllegalArgumentException("Organization ID cannot be null"));
+        this.apiUrl = Optional.ofNullable(builder.apiUrl)
+                .orElseThrow(() -> new IllegalArgumentException("API URL cannot be null"));
         this.enableEvaluation = builder.enableEvaluation;
-
-        if (this.projectName.isEmpty()) {
-            throw new IllegalArgumentException("Project name cannot be empty");
-        }
     }
 
     public String projectName() {
@@ -69,28 +71,30 @@ public static Builder builder() {
 
     /**
      * Creates a default configuration with the given project name.
-     *
-     * 
This method uses default values from environment variables:
-     *
+     * 

+     * This method uses default values from environment variables:
      * 
    
-     *   
  • API Key: {@code Env.JUDGMENT_API_KEY}
-     *   
  • Organization ID: {@code Env.JUDGMENT_ORG_ID}
-     *   
  • API URL: {@code Env.JUDGMENT_API_URL}
-     *   
  • Evaluation: enabled
+     * 
  • API Key: {@code Env.JUDGMENT_API_KEY}
+     * 
  • Organization ID: {@code Env.JUDGMENT_ORG_ID}
+     * 
  • API URL: {@code Env.JUDGMENT_API_URL}
+     * 
  • Evaluation: enabled
      * 

      *
-     * @param projectName the name of the project
+     * @param projectName
+     *            the name of the project
      * @return a new TracerConfiguration with default values
-     * @throws IllegalArgumentException if project name is null or empty
+     * @throws IllegalArgumentException
+     *             if project name is null or empty
      */
     public static TracerConfiguration createDefault(String projectName) {
-        return builder().projectName(projectName).build();
+        return builder().projectName(projectName)
+                .build();
     }
 
     /**
      * Builder for creating TracerConfiguration instances.
-     *
-     * 
Example usage:
+     * 

+     * Example usage:
      *
      * 
{@code
      * TracerConfiguration config = TracerConfiguration.builder()
@@ -103,10 +107,10 @@ public static TracerConfiguration createDefault(String projectName) {
      * }

      */
     public static final class Builder {
-        private String projectName;
-        private String apiKey = com.judgmentlabs.judgeval.Env.JUDGMENT_API_KEY;
-        private String organizationId = com.judgmentlabs.judgeval.Env.JUDGMENT_ORG_ID;
-        private String apiUrl = com.judgmentlabs.judgeval.Env.JUDGMENT_API_URL;
+        private String  projectName;
+        private String  apiKey           = com.judgmentlabs.judgeval.Env.JUDGMENT_API_KEY;
+        private String  organizationId   = com.judgmentlabs.judgeval.Env.JUDGMENT_ORG_ID;
+        private String  apiUrl           = com.judgmentlabs.judgeval.Env.JUDGMENT_API_URL;
         private boolean enableEvaluation = true;
 
         public Builder projectName(String projectName) {
diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java
index 8de675a..fd6bdb7 100644
--- a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java
+++ b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/JudgmentSpanExporter.java
@@ -1,6 +1,7 @@
 package com.judgmentlabs.judgeval.tracer.exporters;
 
 import java.util.Collection;
+import java.util.Optional;
 
 import com.judgmentlabs.judgeval.utils.Logger;
 
@@ -10,10 +11,11 @@
 import io.opentelemetry.sdk.trace.export.SpanExporter;
 
 /**
- * SpanExporter implementation that sends spans to Judgment Labs with project identification.
- *
- * 
This exporter wraps the OTLP HTTP exporter and adds Judgment Labs specific headers and project
- * identification to all exported spans.
+ * SpanExporter implementation that sends spans to Judgment Labs with project
+ * identification.
+ * 

+ * This exporter wraps the OTLP HTTP exporter and adds Judgment Labs specific
+ * headers and project identification to all exported spans.
  */
 public class JudgmentSpanExporter implements SpanExporter {
     private final SpanExporter delegate;
@@ -21,24 +23,27 @@ public class JudgmentSpanExporter implements SpanExporter {
     /**
      * Creates a new JudgmentSpanExporter with the specified configuration.
      *
-     * @param endpoint the OTLP endpoint URL
-     * @param apiKey the API key for authentication
-     * @param organizationId the organization ID
-     * @param projectId the project ID (must not be null or empty)
-     * @throws IllegalArgumentException if projectId is null or empty
+     * @param endpoint
+     *            the OTLP endpoint URL
+     * @param apiKey
+     *            the API key for authentication
+     * @param organizationId
+     *            the organization ID
+     * @param projectId
+     *            the project ID (must not be null or empty)
+     * @throws IllegalArgumentException
+     *             if projectId is null or empty
      */
-    public JudgmentSpanExporter(
-            String endpoint, String apiKey, String organizationId, String projectId) {
-        if (projectId == null || projectId.isEmpty()) {
+    protected JudgmentSpanExporter(String endpoint, String apiKey, String organizationId, String projectId) {
+        if (projectId.isEmpty()) {
             throw new IllegalArgumentException("projectId is required for JudgmentSpanExporter");
         }
-        this.delegate =
-                OtlpHttpSpanExporter.builder()
-                        .setEndpoint(endpoint)
-                        .addHeader("Authorization", "Bearer " + apiKey)
-                        .addHeader("X-Organization-Id", organizationId)
-                        .addHeader("X-Project-Id", projectId)
-                        .build();
+        this.delegate = OtlpHttpSpanExporter.builder()
+                .setEndpoint(endpoint)
+                .addHeader("Authorization", "Bearer " + apiKey)
+                .addHeader("X-Organization-Id", organizationId)
+                .addHeader("X-Project-Id", projectId)
+                .build();
     }
 
     /**
@@ -50,35 +55,56 @@ public static Builder builder() {
         return new Builder();
     }
 
+    /**
+     * Exports the collection of spans to the Judgment Labs backend.
+     *
+     * @param spans
+     *            the collection of spans to export
+     * @return a CompletableResultCode representing the export operation status
+     */
     @Override
     public CompletableResultCode export(Collection spans) {
         Logger.info("Exported " + spans.size() + " spans");
         return delegate.export(spans);
     }
 
+    /**
+     * Flushes any pending span exports.
+     *
+     * @return a CompletableResultCode representing the flush operation status
+     */
     @Override
     public CompletableResultCode flush() {
         return delegate.flush();
     }
 
+    /**
+     * Shuts down this exporter and releases any resources.
+     *
+     * @return a CompletableResultCode representing the shutdown operation status
+     */
     @Override
     public CompletableResultCode shutdown() {
         return delegate.shutdown();
     }
 
-    /** Builder for creating JudgmentSpanExporter instances. */
+    /**
+     * Builder for creating JudgmentSpanExporter instances.
+     */
     public static final class Builder {
         private String endpoint;
         private String apiKey;
         private String organizationId;
         private String projectId;
 
-        private Builder() {}
+        private Builder() {
+        }
 
         /**
          * Sets the OTLP endpoint URL.
          *
-         * @param endpoint the endpoint URL
+         * @param endpoint
+         *            the endpoint URL
          * @return this builder for method chaining
          */
         public Builder endpoint(String endpoint) {
@@ -89,7 +115,8 @@ public Builder endpoint(String endpoint) {
         /**
          * Sets the API key for authentication.
          *
-         * @param apiKey the API key
+         * @param apiKey
+         *            the API key
          * @return this builder for method chaining
          */
         public Builder apiKey(String apiKey) {
@@ -100,7 +127,8 @@ public Builder apiKey(String apiKey) {
         /**
          * Sets the organization ID.
          *
-         * @param organizationId the organization ID
+         * @param organizationId
+         *            the organization ID
          * @return this builder for method chaining
          */
         public Builder organizationId(String organizationId) {
@@ -111,7 +139,8 @@ public Builder organizationId(String organizationId) {
         /**
          * Sets the project ID.
          *
-         * @param projectId the project ID
+         * @param projectId
+         *            the project ID
          * @return this builder for method chaining
          */
         public Builder projectId(String projectId) {
@@ -120,26 +149,31 @@ public Builder projectId(String projectId) {
         }
 
         /**
-         * Builds a new JudgmentSpanExporter with the current configuration.
+         * Builds a new JudgmentSpanExporter instance with the configured settings.
          *
          * @return a new JudgmentSpanExporter instance
-         * @throws IllegalArgumentException if required fields are missing
+         * @throws IllegalArgumentException
+         *             if any required field is null or empty
          */
         public JudgmentSpanExporter build() {
-            if (endpoint == null || endpoint.trim().isEmpty()) {
-                throw new IllegalArgumentException("Endpoint is required");
-            }
-            if (apiKey == null || apiKey.trim().isEmpty()) {
-                throw new IllegalArgumentException("API key is required");
-            }
-            if (organizationId == null || organizationId.trim().isEmpty()) {
-                throw new IllegalArgumentException("Organization ID is required");
-            }
-            if (projectId == null || projectId.trim().isEmpty()) {
-                throw new IllegalArgumentException("Project ID is required");
-            }
-
-            return new JudgmentSpanExporter(endpoint, apiKey, organizationId, projectId);
+            String validEndpoint = Optional.ofNullable(endpoint)
+                    .map(String::trim)
+                    .filter(e -> !e.isEmpty())
+                    .orElseThrow(() -> new IllegalArgumentException("Endpoint is required"));
+            String validApiKey = Optional.ofNullable(apiKey)
+                    .map(String::trim)
+                    .filter(key -> !key.isEmpty())
+                    .orElseThrow(() -> new IllegalArgumentException("API key is required"));
+            String validOrganizationId = Optional.ofNullable(organizationId)
+                    .map(String::trim)
+                    .filter(id -> !id.isEmpty())
+                    .orElseThrow(() -> new IllegalArgumentException("Organization ID is required"));
+            String validProjectId = Optional.ofNullable(projectId)
+                    .map(String::trim)
+                    .filter(id -> !id.isEmpty())
+                    .orElseThrow(() -> new IllegalArgumentException("Project ID is required"));
+
+            return new JudgmentSpanExporter(validEndpoint, validApiKey, validOrganizationId, validProjectId);
         }
     }
 }
diff --git a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/NoOpSpanExporter.java b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/NoOpSpanExporter.java
index 01509e8..8689d1c 100644
--- a/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/NoOpSpanExporter.java
+++ b/src/main/java/com/judgmentlabs/judgeval/tracer/exporters/NoOpSpanExporter.java
@@ -6,17 +6,38 @@
 import io.opentelemetry.sdk.trace.data.SpanData;
 import io.opentelemetry.sdk.trace.export.SpanExporter;
 
+/**
+ * A no-op implementation of SpanExporter that discards all spans. Used as a
+ * fallback when project resolution fails or when spans should not be exported.
+ */
 public class NoOpSpanExporter implements SpanExporter {
+    /**
+     * Discards the collection of spans without exporting.
+     *
+     * @param spans
+     *            the collection of spans (ignored)
+     * @return a successful CompletableResultCode
+     */
     @Override
     public CompletableResultCode export(Collection spans) {
         return CompletableResultCode.ofSuccess();
     }
 
+    /**
+     * Performs a no-op flush operation.
+     *
+     * @return a successful CompletableResultCode
+     */
     @Override
     public CompletableResultCode flush() {
         return CompletableResultCode.ofSuccess();
     }
 
+    /**
+     * Performs a no-op shutdown operation.
+     *
+     * @return a successful CompletableResultCode
+     */
     @Override
     public CompletableResultCode shutdown() {
         return CompletableResultCode.ofSuccess();
diff --git a/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java b/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java
index b711dd1..ff2796d 100644
--- a/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java
+++ b/src/main/java/com/judgmentlabs/judgeval/utils/Logger.java
@@ -8,13 +8,12 @@
 import com.judgmentlabs.judgeval.Env;
 
 public class Logger {
-    private static final String RESET = "\033[0m";
-    private static final String RED = "\033[31m";
-    private static final String YELLOW = "\033[33m";
-    private static final String GRAY = "\033[90m";
+    private static final String            RESET          = "\033[0m";
+    private static final String            RED            = "\033[31m";
+    private static final String            YELLOW         = "\033[33m";
+    private static final String            GRAY           = "\033[90m";
 
-    private static final DateTimeFormatter DATE_FORMATTER =
-            DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+    private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
 
     public enum Level {
         DEBUG(0, GRAY),
@@ -23,7 +22,7 @@ public enum Level {
         ERROR(3, RED),
         CRITICAL(4, RED);
 
-        private final int value;
+        private final int    value;
         private final String color;
 
         Level(int value, String color) {
@@ -40,10 +39,10 @@ public String getColor() {
         }
     }
 
-    private static final AtomicBoolean initialized = new AtomicBoolean(false);
-    private static Level currentLevel = Level.WARNING;
-    private static boolean useColor = true;
-    private static PrintStream output = System.out;
+    private static final AtomicBoolean initialized  = new AtomicBoolean(false);
+    private static Level               currentLevel = Level.WARNING;
+    private static boolean             useColor     = true;
+    private static PrintStream         output       = System.out;
 
     private static void initialize() {
         if (initialized.compareAndSet(false, true)) {
@@ -100,9 +99,9 @@ private static void log(Level level, String message) {
             return;
         }
 
-        String timestamp = LocalDateTime.now().format(DATE_FORMATTER);
-        String formattedMessage =
-                String.format("%s - judgeval - %s - %s", timestamp, level.name(), message);
+        String timestamp = LocalDateTime.now()
+                .format(DATE_FORMATTER);
+        String formattedMessage = String.format("%s - judgeval - %s - %s", timestamp, level.name(), message);
 
         if (useColor) {
             formattedMessage = level.getColor() + formattedMessage + RESET;