feat: prompt scorer, tracer adjustments, logging, and error handling

pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.judgmentlabs</groupId>
     <artifactId>judgeval-java</artifactId>
-    <version>0.1.9</version>
+    <version>0.2.0</version>
     <packaging>jar</packaging>
     <name>Judgeval Java</name>
     <description>Java SDK for Judgeval</description>

scripts/generate_client.py

@@ -13,7 +13,7 @@
     "/add_to_run_eval_queue/",
     "/get_evaluation_status/",
     "/save_scorer/",
-    "/fetch_scorer/",
+    "/fetch_scorers/",
     "/scorer_exists/",
     "/projects/resolve/",
 ]

src/main/java/com/judgmentlabs/judgeval/data/APIScorerType.java

@@ -2,6 +2,7 @@
 
 public enum APIScorerType {
     PROMPT_SCORER("Prompt Scorer"),
+    TRACE_PROMPT_SCORER("Trace Prompt Scorer"),
     FAITHFULNESS("Faithfulness"),
     ANSWER_RELEVANCY("Answer Relevancy"),
     ANSWER_CORRECTNESS("Answer Correctness"),

src/main/java/com/judgmentlabs/judgeval/data/Example.java

@@ -2,6 +2,7 @@
 
 import java.time.Instant;
 import java.util.Map;
+import java.util.UUID;
 
 public class Example extends com.judgmentlabs.judgeval.internal.api.models.Example {
 
@@ -28,7 +29,7 @@ public String getValue() {
 
     public Example() {
         super();
-        setExampleId("");
+        setExampleId(UUID.randomUUID().toString());
         setCreatedAt(Instant.now().toString());
         setName(null);
     }

src/main/java/com/judgmentlabs/judgeval/data/ExampleEvaluationRun.java

@@ -7,7 +7,6 @@
 import java.util.Objects;
 import java.util.UUID;
 
-import com.judgmentlabs.judgeval.internal.api.models.BaseScorer;
 import com.judgmentlabs.judgeval.internal.api.models.ScorerConfig;
 
 public class ExampleEvaluationRun
@@ -21,69 +20,11 @@ public ExampleEvaluationRun() {
         setCreatedAt(Instant.now().atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT));
     }
 
-    public static ExampleEvaluationRun createWithLocalScorers(
-            String projectName,
-            String evalName,
-            List<Example> examples,
-            List<com.judgmentlabs.judgeval.scorers.BaseScorer> localScorers,
-            String model,
-            String organizationId) {
-        ExampleEvaluationRun eval = new ExampleEvaluationRun();
-        eval.setProjectName(projectName);
-        eval.setEvalName(evalName);
-        @SuppressWarnings("unchecked")
-        List<com.judgmentlabs.judgeval.internal.api.models.Example> internalExamples =
-                (List<com.judgmentlabs.judgeval.internal.api.models.Example>) (List<?>) examples;
-        eval.setExamples(internalExamples);
-        eval.setModel(model);
-        eval.setOrganizationId(organizationId);
-
-        if (localScorers != null) {
-            List<BaseScorer> customScorers = new java.util.ArrayList<>();
-
-            for (com.judgmentlabs.judgeval.scorers.BaseScorer scorer : localScorers) {
-                customScorers.add((BaseScorer) scorer);
-            }
-
-            eval.setCustomScorers(customScorers);
-            eval.setJudgmentScorers(new java.util.ArrayList<>());
-        }
-
-        eval.validateScorerLists();
-        return eval;
-    }
-
-    public static ExampleEvaluationRun createWithApiScorers(
-            String projectName,
-            String evalName,
-            List<Example> examples,
-            List<ScorerConfig> apiScorers,
-            String model,
-            String organizationId) {
-        ExampleEvaluationRun eval = new ExampleEvaluationRun();
-        eval.setProjectName(projectName);
-        eval.setEvalName(evalName);
-        @SuppressWarnings("unchecked")
-        List<com.judgmentlabs.judgeval.internal.api.models.Example> internalExamples =
-                (List<com.judgmentlabs.judgeval.internal.api.models.Example>) (List<?>) examples;
-        eval.setExamples(internalExamples);
-        eval.setModel(model);
-        eval.setOrganizationId(organizationId);
-
-        if (apiScorers != null) {
-            eval.setCustomScorers(new java.util.ArrayList<>());
-            eval.setJudgmentScorers(apiScorers);
-        }
-
-        eval.validateScorerLists();
-        return eval;
-    }
-
     public ExampleEvaluationRun(
             String projectName,
             String evalName,
             List<Example> examples,
-            List<Object> scorers,
+            List<ScorerConfig> scorers,
             String model,
             String organizationId) {
         this();
@@ -95,43 +36,8 @@ public ExampleEvaluationRun(
         setExamples(internalExamples);
         setModel(model);
         setOrganizationId(organizationId);
-
-        if (scorers != null) {
-            List<BaseScorer> customScorers = new java.util.ArrayList<>();
-            List<ScorerConfig> judgmentScorers = new java.util.ArrayList<>();
-
-            for (Object scorer : scorers) {
-                if (scorer instanceof com.judgmentlabs.judgeval.scorers.BaseScorer) {
-                    customScorers.add((BaseScorer) scorer);
-                } else if (scorer instanceof ScorerConfig) {
-                    judgmentScorers.add((ScorerConfig) scorer);
-                }
-            }
-
-            setCustomScorers(customScorers);
-            setJudgmentScorers(judgmentScorers);
-        }
-
-        validateScorerLists();
-    }
-
-    private void validateScorerLists() {
-        List<BaseScorer> customScorers = getCustomScorers();
-        List<ScorerConfig> judgmentScorers = getJudgmentScorers();
-
-        if ((customScorers == null || customScorers.isEmpty())
-                && (judgmentScorers == null || judgmentScorers.isEmpty())) {
-            throw new IllegalArgumentException(
-                    "At least one of custom_scorers or judgment_scorers must be provided.");
-        }
-
-        if (customScorers != null
-                && !customScorers.isEmpty()
-                && judgmentScorers != null
-                && !judgmentScorers.isEmpty()) {
-            throw new IllegalArgumentException(
-                    "Only one of custom_scorers or judgment_scorers can be provided, not both.");
-        }
+        setCustomScorers(new java.util.ArrayList<>());
+        setJudgmentScorers(scorers);
     }
 
     public void setOrganizationId(String organizationId) {
@@ -168,9 +74,7 @@ public static final class Builder {
         private String projectName;
         private String evalName;
         private List<Example> examples;
-        private List<com.judgmentlabs.judgeval.scorers.BaseScorer> localScorers;
-        private List<ScorerConfig> apiScorers;
-        private List<Object> mixedScorers;
+        private List<ScorerConfig> scorers;
         private String model;
         private String organizationId;
 
@@ -204,43 +108,16 @@ public Builder example(Example example) {
             return this;
         }
 
-        public Builder localScorers(
-                List<com.judgmentlabs.judgeval.scorers.BaseScorer> localScorers) {
-            this.localScorers = localScorers;
-            return this;
-        }
-
-        public Builder localScorer(com.judgmentlabs.judgeval.scorers.BaseScorer scorer) {
-            if (this.localScorers == null) {
-                this.localScorers = new java.util.ArrayList<>();
-            }
-            this.localScorers.add(scorer);
-            return this;
-        }
-
-        public Builder apiScorers(List<ScorerConfig> apiScorers) {
-            this.apiScorers = apiScorers;
-            return this;
-        }
-
-        public Builder apiScorer(ScorerConfig scorer) {
-            if (this.apiScorers == null) {
-                this.apiScorers = new java.util.ArrayList<>();
-            }
-            this.apiScorers.add(scorer);
+        public Builder scorers(List<ScorerConfig> scorers) {
+            this.scorers = scorers;
             return this;
         }
 
-        public Builder scorers(List<Object> scorers) {
-            this.mixedScorers = scorers;
-            return this;
-        }
-
-        public Builder scorer(Object scorer) {
-            if (this.mixedScorers == null) {
-                this.mixedScorers = new java.util.ArrayList<>();
+        public Builder scorer(ScorerConfig scorer) {
+            if (this.scorers == null) {
+                this.scorers = new java.util.ArrayList<>();
             }
-            this.mixedScorers.add(scorer);
+            this.scorers.add(scorer);
             return this;
         }
 
@@ -264,19 +141,12 @@ public ExampleEvaluationRun build() {
             if (examples == null || examples.isEmpty()) {
                 throw new IllegalArgumentException("At least one example is required");
             }
-
-            if (localScorers != null && !localScorers.isEmpty()) {
-                return createWithLocalScorers(
-                        projectName, evalName, examples, localScorers, model, organizationId);
-            } else if (apiScorers != null && !apiScorers.isEmpty()) {
-                return createWithApiScorers(
-                        projectName, evalName, examples, apiScorers, model, organizationId);
-            } else if (mixedScorers != null && !mixedScorers.isEmpty()) {
-                return new ExampleEvaluationRun(
-                        projectName, evalName, examples, mixedScorers, model, organizationId);
-            } else {
+            if (scorers == null || scorers.isEmpty()) {
                 throw new IllegalArgumentException("At least one scorer is required");
             }
+
+            return new ExampleEvaluationRun(
+                    projectName, evalName, examples, scorers, model, organizationId);
         }
     }
 }