From 661dab2731681c38ea197d8048a0ca904eaff3df Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Mon, 1 Dec 2025 18:45:43 +0100 Subject: [PATCH 01/23] Andy has now a (backward compatible) quality check --- .../cse1110/andy/grade/GradeCalculator.java | 14 ++++++- .../cse1110/andy/grade/GradeValues.java | 39 +++++++++++++++++-- .../cse1110/andy/grade/GradeWeight.java | 21 ++++++++++ .../cse1110/andy/result/QualityResult.java | 37 ++++++++++++++++++ .../tudelft/cse1110/andy/result/Result.java | 30 ++++++++++++++ .../cse1110/andy/result/ResultBuilder.java | 8 ++++ .../java/unit/grade/GradeCalculatorTest.java | 29 ++++++++++++++ .../test/java/unit/grade/GradeWeightTest.java | 11 ++++++ 8 files changed, 184 insertions(+), 5 deletions(-) create mode 100644 andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeCalculator.java b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeCalculator.java index 8b3bc18f0..8fd222936 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeCalculator.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeCalculator.java @@ -18,8 +18,10 @@ public int calculateFinalGrade(GradeValues gradeValues, GradeWeight weights) { * weights.getMetaTestsWeight(); float checkScore = codeChecksScore(gradeValues.getChecksPassed(), gradeValues.getTotalChecks()) * weights.getCodeChecksWeight(); + float qualityScore = qualityScore(gradeValues.getQualityScore()) + * weights.getQualityWeight(); - float finalDecimalGrade = branchScore + mutationScore + metaScore + checkScore; + float finalDecimalGrade = branchScore + mutationScore + metaScore + checkScore + qualityScore; int finalGrade = Math.round(finalDecimalGrade * 100); @@ -98,6 +100,16 @@ public float codeChecksScore(int checksPassed, int totalChecks) { return (float)checksPassed / totalChecks; } + /** + * For now, just a dummy method... + * @param qualityScore - a dummy score + * @return the dummy score + */ + private float qualityScore(int qualityScore) { + // dummy + return 1.0f; + } + /** * @param gradeValues the grade values * @param weights the weights diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java index 11918ed57..792a26697 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java @@ -1,9 +1,6 @@ package nl.tudelft.cse1110.andy.grade; -import nl.tudelft.cse1110.andy.result.CodeChecksResult; -import nl.tudelft.cse1110.andy.result.CoverageResult; -import nl.tudelft.cse1110.andy.result.MetaTestsResult; -import nl.tudelft.cse1110.andy.result.MutationTestingResult; +import nl.tudelft.cse1110.andy.result.*; public class GradeValues { @@ -21,6 +18,8 @@ public class GradeValues { private int penalty; + private int qualityScore; + public int getCoveredBranches() { return coveredBranches; } @@ -86,12 +85,44 @@ public GradeValues setPenalty(int penalty) { return this; } + public int getQualityScore() { + return qualityScore; + } + + public GradeValues setQualityScore(int qualityScore) { + this.qualityScore = qualityScore; + return this; + } + public static GradeValues fromResults(CoverageResult coverageResults, CodeChecksResult codeCheckResults, MutationTestingResult mutationResults, MetaTestsResult metaTestResults, MetaTestsResult penaltyMetaTestResults, CodeChecksResult penaltyCodeCheckResults) { GradeValues grades = new GradeValues(); grades.setBranchGrade(coverageResults.getCoveredBranches(), coverageResults.getTotalNumberOfBranches()); grades.setCheckGrade(codeCheckResults.getNumberOfPassedChecks(), codeCheckResults.getTotalNumberOfChecks()); grades.setMutationGrade(mutationResults.getKilledMutants(), mutationResults.getTotalNumberOfMutants()); grades.setMetaGrade(metaTestResults.getPassedMetaTests(), metaTestResults.getTotalTests()); + grades.setQualityScore(0); // use alternative constructor for quality check (backward compatibility) + + // penalty is equal to the sum of the weights of all failed penalty code checks + grades.setPenalty(penaltyCodeCheckResults.getCheckResults().stream().mapToInt(check -> check.passed() ? 0 : check.getWeight()).sum() + + penaltyMetaTestResults.getMetaTestResults().stream().mapToInt(check -> check.succeeded() ? 0 : check.getWeight()).sum()); + + return grades; + } + + /* Alternative constructor for assignments with quality score */ + public static GradeValues fromResults(CoverageResult coverageResults, + CodeChecksResult codeCheckResults, + MutationTestingResult mutationResults, + MetaTestsResult metaTestResults, + MetaTestsResult penaltyMetaTestResults, + CodeChecksResult penaltyCodeCheckResults, + QualityResult qualityResult) { + GradeValues grades = new GradeValues(); + grades.setBranchGrade(coverageResults.getCoveredBranches(), coverageResults.getTotalNumberOfBranches()); + grades.setCheckGrade(codeCheckResults.getNumberOfPassedChecks(), codeCheckResults.getTotalNumberOfChecks()); + grades.setMutationGrade(mutationResults.getKilledMutants(), mutationResults.getTotalNumberOfMutants()); + grades.setMetaGrade(metaTestResults.getPassedMetaTests(), metaTestResults.getTotalTests()); + grades.setQualityScore(qualityResult.getScore()); // penalty is equal to the sum of the weights of all failed penalty code checks grades.setPenalty(penaltyCodeCheckResults.getCheckResults().stream().mapToInt(check -> check.passed() ? 0 : check.getWeight()).sum() diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeWeight.java b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeWeight.java index 9dd6d27cf..bdffd75b5 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeWeight.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeWeight.java @@ -8,12 +8,14 @@ public class GradeWeight { private final float mutationCoverageWeight; private final float metaTestsWeight; private final float codeChecksWeight; + private final float qualityWeight; public GradeWeight(float branchCoverageWeight, float mutationCoverageWeight, float metaTestsWeight, float codeChecksWeight) { this.branchCoverageWeight = branchCoverageWeight; this.mutationCoverageWeight = mutationCoverageWeight; this.metaTestsWeight = metaTestsWeight; this.codeChecksWeight = codeChecksWeight; + this.qualityWeight = 0.0f; // use alternative constructor for quality check (backward compatibility) // weights have to sum up to 1 float weightSum = branchCoverageWeight + mutationCoverageWeight + metaTestsWeight + codeChecksWeight; @@ -22,6 +24,21 @@ public GradeWeight(float branchCoverageWeight, float mutationCoverageWeight, flo throw new RuntimeException("The weight configuration is wrong! Call the teacher!"); } + /* Alternative constructor for assignments with quality check */ + public GradeWeight(float branchCoverageWeight, float mutationCoverageWeight, float metaTestsWeight, float codeChecksWeight, float qualityWeight) { + this.branchCoverageWeight = branchCoverageWeight; + this.mutationCoverageWeight = mutationCoverageWeight; + this.metaTestsWeight = metaTestsWeight; + this.codeChecksWeight = codeChecksWeight; + this.qualityWeight = qualityWeight; + + // weights have to sum up to 1 + float weightSum = branchCoverageWeight + mutationCoverageWeight + metaTestsWeight + codeChecksWeight + qualityWeight; + float epsilon = Math.abs(1 - weightSum); + if(epsilon > 0.001) + throw new RuntimeException("The weight configuration is wrong! Call the teacher!"); + } + public float getBranchCoverageWeight() { return branchCoverageWeight; } @@ -38,6 +55,10 @@ public float getCodeChecksWeight() { return codeChecksWeight; } + public float getQualityWeight() { + return qualityWeight; + } + public static GradeWeight fromConfig(Map weights) { float coverage = weights.getOrDefault("coverage", 0.0f); float mutation = weights.getOrDefault("mutation", 0.0f); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java new file mode 100644 index 000000000..b7dbb89f8 --- /dev/null +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -0,0 +1,37 @@ +package nl.tudelft.cse1110.andy.result; + +import java.util.Collections; +import java.util.List; + +public class QualityResult { + private int score; // between 0 and 1 + + public QualityResult(int score) { + // this.score = score; + // dummy: + this.score = 1; + } + + public static QualityResult build(int score) { + return new QualityResult(score); + } + + public static QualityResult empty() { + return new QualityResult(0); + } + + public int getScore() { + return score; + } + + public void setScore(int score) { + this.score = score; + } + + @Override + public String toString() { + return "QualityResult{" + + "score=" + score + + '}'; + } +} diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java index 18d0bf374..bd1d6d31d 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java @@ -20,6 +20,7 @@ public class Result { private final double timeInSeconds; private final GradeWeight weights; private final String successMessage; + private final QualityResult qualityResult; public Result(CompilationResult compilation, UnitTestsResult tests, MutationTestingResult mutationTesting, CodeChecksResult codeChecks, CodeChecksResult penaltyCodeChecks, CoverageResult coverage, MetaTestsResult metaTests, MetaTestsResult penaltyMetaTests, int penalty, int finalGrade, GenericFailure genericFailure, double timeInSeconds, GradeWeight weights, String successMessage) { this.compilation = compilation; @@ -37,6 +38,30 @@ public Result(CompilationResult compilation, UnitTestsResult tests, MutationTest this.weights = weights; this.successMessage = successMessage; + this.qualityResult = QualityResult.empty(); + + if(finalGrade < 0 || finalGrade>100) + throw new RuntimeException("Invalid final grade"); + } + + public Result(CompilationResult compilation, UnitTestsResult tests, MutationTestingResult mutationTesting, CodeChecksResult codeChecks, CodeChecksResult penaltyCodeChecks, CoverageResult coverage, MetaTestsResult metaTests, MetaTestsResult penaltyMetaTests, int penalty, int finalGrade, GenericFailure genericFailure, double timeInSeconds, GradeWeight weights, String successMessage, + QualityResult qualityResult) { + this.compilation = compilation; + this.tests = tests; + this.mutationTesting = mutationTesting; + this.codeChecks = codeChecks; + this.penaltyCodeChecks = penaltyCodeChecks; + this.coverage = coverage; + this.metaTests = metaTests; + this.penaltyMetaTests = penaltyMetaTests; + this.penalty = penalty; + this.finalGrade = finalGrade; + this.genericFailure = genericFailure; + this.timeInSeconds = timeInSeconds; + this.weights = weights; + this.successMessage = successMessage; + this.qualityResult = qualityResult; + if(finalGrade < 0 || finalGrade>100) throw new RuntimeException("Invalid final grade"); } @@ -100,6 +125,10 @@ public GenericFailure getGenericFailure() { return genericFailure; } + public QualityResult getQualityResult() { + return qualityResult; + } + public boolean hasFailed() { return !compilation.successful() || tests.hasTestsFailingOrFailures() || hasGenericFailure(); } @@ -122,6 +151,7 @@ public String toString() { ", timeInSeconds=" + timeInSeconds + ", weights=" + weights + ", successMessage=" + successMessage + + ", qualityResult=" + qualityResult + '}'; } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index 69856d294..10a41b6d4 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -50,6 +50,7 @@ public class ResultBuilder { private CoverageResult coverageResults = CoverageResult.empty(); private MetaTestsResult metaTestResults = MetaTestsResult.empty(); private MetaTestsResult penaltyMetaTestResults = MetaTestsResult.empty(); + private QualityResult qualityResult = QualityResult.empty(); public ResultBuilder(Context ctx, GradeCalculator gradeCalculator) { this.ctx = ctx; @@ -249,6 +250,13 @@ public void logPenaltyMetaTests(int score, int totalTests, List this.penaltyMetaTestResults.addResults(score, totalTests, metaTestResults); } + /* + * Quality + */ + public void logQuality() { + // dummy + } + /* * Generic failures */ diff --git a/andy/src/test/java/unit/grade/GradeCalculatorTest.java b/andy/src/test/java/unit/grade/GradeCalculatorTest.java index d7e8ea5eb..ced541697 100644 --- a/andy/src/test/java/unit/grade/GradeCalculatorTest.java +++ b/andy/src/test/java/unit/grade/GradeCalculatorTest.java @@ -89,6 +89,35 @@ private static Stream differentWeights() { ); } + @ParameterizedTest + @MethodSource("withQualityCheck") + void withQualityCheck(int coveredBranches, int totalBranches, + int detectedMutations, int totalMutations, + int metaTestsPassed, int totalMetaTests, + int checksPassed, int totalChecks, int qualityScore, int expectedGrade, + float branchCoverageWeight, float mutationCoverageWeight, + float metaTestsWeight, float codeChecksWeight, float qualityWeight) { + + GradeWeight weights = new GradeWeight(branchCoverageWeight, mutationCoverageWeight, metaTestsWeight, codeChecksWeight, qualityWeight); + + GradeValues grades = new GradeValues(); + grades.setBranchGrade(coveredBranches, totalBranches); + grades.setMutationGrade(detectedMutations, totalMutations); + grades.setMetaGrade(metaTestsPassed, totalMetaTests); + grades.setCheckGrade(checksPassed, totalChecks); + grades.setQualityScore(qualityScore); + + int finalGrade = new GradeCalculator().calculateFinalGrade(grades, weights); + + assertThat(finalGrade).isEqualTo(expectedGrade); + } + + private static Stream withQualityCheck() { + return Stream.of( + of(25, 25, 55, 55, 100, 100, 5, 5, 1, 100, 0.20f, 0.20f, 0.20f, 0.20f, 0.20f) // 0.20*1 + 0.20*1 + 0.20*1 + 0.20*1 + 0.20*1 = 1.0 --> 100 + ); + } + /* * Test where the grade is between 99.5 and 100, should be rounded down to 99 and not * rounded up to 100, as 100 should only be achievable if everything diff --git a/andy/src/test/java/unit/grade/GradeWeightTest.java b/andy/src/test/java/unit/grade/GradeWeightTest.java index d3227dbe4..5ac113c8c 100644 --- a/andy/src/test/java/unit/grade/GradeWeightTest.java +++ b/andy/src/test/java/unit/grade/GradeWeightTest.java @@ -17,4 +17,15 @@ void preCondition() { new GradeWeight(0.33f, 0.33f, 0.33f, 0.01f); new GradeWeight(0.33f, 0.33f, 0.34f, 0f); } + + @Test + void preConditionWithQualityCheck() { + assertThatExceptionOfType(RuntimeException.class).isThrownBy(() -> + new GradeWeight(0.1f, 0.1f, 0.1f, 0.1f, 0.1f) + ).withMessageContaining("weight configuration is wrong"); + + new GradeWeight(0.20f, 0.20f, 0.20f, 0.20f, 0.20f); + new GradeWeight(0.30f, 0.30f, 0.30f, 0.09f, 0.01f); + new GradeWeight(0.25f, 0.25f, 0.25f, 0.25f, 0.0f); + } } From 7b6372c52e1dad56cfce12cd00906314ffb2b306 Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Mon, 1 Dec 2025 20:10:41 +0100 Subject: [PATCH 02/23] Option to assess limited number of meta tests triggered by a test --- .../tudelft/cse1110/andy/config/MetaTest.java | 3 ++- .../execution/metatest/AbstractMetaTest.java | 2 +- .../execution/metatest/MetaTestReport.java | 18 ++++++++++++++++++ .../ExternalProcessMetaTest.java | 9 ++++++--- .../metatest/library/LibraryMetaTest.java | 12 ++++++++---- .../andy/execution/step/RunMetaTestsStep.java | 7 ++++++- .../cse1110/andy/grade/GradeValues.java | 2 +- .../cse1110/andy/grade/GradeWeight.java | 3 ++- .../cse1110/andy/result/QualityResult.java | 16 ++++++++++++++-- .../cse1110/andy/result/ResultBuilder.java | 7 ++++--- 10 files changed, 62 insertions(+), 17 deletions(-) create mode 100644 andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/config/MetaTest.java b/andy/src/main/java/nl/tudelft/cse1110/andy/config/MetaTest.java index 3c12a0ac9..bd067fd8c 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/config/MetaTest.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/config/MetaTest.java @@ -3,6 +3,7 @@ import nl.tudelft.cse1110.andy.execution.Context.Context; import nl.tudelft.cse1110.andy.execution.externalprocess.ExternalProcess; import nl.tudelft.cse1110.andy.execution.metatest.AbstractMetaTestFactory; +import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; public interface MetaTest { String getName(); @@ -11,7 +12,7 @@ public interface MetaTest { String getNameAndWeight(); - boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg) throws Exception; + MetaTestReport execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg) throws Exception; static MetaTest withStringReplacement(int weight, String name, String old, String replacement) { return new AbstractMetaTestFactory().withStringReplacement(weight, name, old, replacement); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/AbstractMetaTest.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/AbstractMetaTest.java index a8333b2f6..63180e5ef 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/AbstractMetaTest.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/AbstractMetaTest.java @@ -31,7 +31,7 @@ public String getNameAndWeight() { } @Override - public abstract boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg) + public abstract MetaTestReport execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg) throws Exception; } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java new file mode 100644 index 000000000..1643dec21 --- /dev/null +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java @@ -0,0 +1,18 @@ +package nl.tudelft.cse1110.andy.execution.metatest; + +public class MetaTestReport { + private int testsRan; + private int testsFound; + private int testsSucceeded; + + public MetaTestReport(int testsRan, int testsSucceeded, int testsFound) { + this.testsRan = testsRan; + this.testsSucceeded = testsSucceeded; + this.testsFound = testsFound; + } + + + public boolean passesTheMetaTest() { + return testsSucceeded < testsRan; + } +} diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/externalprocess/ExternalProcessMetaTest.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/externalprocess/ExternalProcessMetaTest.java index a3891093b..6c6e74508 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/externalprocess/ExternalProcessMetaTest.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/externalprocess/ExternalProcessMetaTest.java @@ -5,6 +5,7 @@ import nl.tudelft.cse1110.andy.execution.Context.Context; import nl.tudelft.cse1110.andy.execution.externalprocess.ExternalProcess; import nl.tudelft.cse1110.andy.execution.metatest.AbstractMetaTest; +import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; import nl.tudelft.cse1110.andy.execution.step.RunJUnitTestsStep; import nl.tudelft.cse1110.andy.result.ResultBuilder; @@ -28,7 +29,7 @@ public void killExternalProcess() { } @Override - public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg) throws Exception { + public MetaTestReport execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg) throws Exception { /* Start the meta external process */ this.startExternalProcess(); @@ -49,8 +50,10 @@ public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfigurat /* Check the result. If there's a failing test, the test suite is good! */ int testsRan = metaResultBuilder.getTestResults().getTestsRan(); int testsSucceeded = metaResultBuilder.getTestResults().getTestsSucceeded(); - boolean passesTheMetaTest = testsSucceeded < testsRan; + int testsFound = metaResultBuilder.getTestResults().getTestsFound(); + // boolean passesTheMetaTest = testsSucceeded < testsRan; + MetaTestReport report = new MetaTestReport(testsRan, testsSucceeded, testsFound); - return passesTheMetaTest; + return report; } } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java index a27758cbb..565656f59 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java @@ -7,6 +7,7 @@ import nl.tudelft.cse1110.andy.execution.Context.ContextDirector; import nl.tudelft.cse1110.andy.execution.ExecutionFlow; import nl.tudelft.cse1110.andy.execution.metatest.AbstractMetaTest; +import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; import nl.tudelft.cse1110.andy.execution.metatest.library.evaluators.MetaEvaluator; import nl.tudelft.cse1110.andy.execution.mode.Action; import nl.tudelft.cse1110.andy.result.CompilationErrorInfo; @@ -35,7 +36,7 @@ public String evaluate(String oldLibraryCode) { } @Override - public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg) throws Exception { + public MetaTestReport execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg) throws Exception { /* Get the student solution, which we will run for each meta test */ String solutionFile = findSolution(dirCfg.getWorkingDir()); @@ -50,7 +51,8 @@ public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfigurat * * We reuse our execution framework to run the code with the meta test. */ - boolean passesTheMetaTest; + // boolean passesTheMetaTest; + MetaTestReport report; /* Copy the library and replace the library by the meta test */ File metaWorkingDir = createTemporaryDirectory("metaWorkplace").toFile(); @@ -67,7 +69,9 @@ public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfigurat /* And check the result. If there's a failing test, the test suite is good! */ int testsRan = metaResult.getTests().getTestsRan(); int testsSucceeded = metaResult.getTests().getTestsSucceeded(); - passesTheMetaTest = testsSucceeded < testsRan; + int testsFound = metaResult.getTests().getTestsFound(); + // passesTheMetaTest = testsSucceeded < testsRan; + report = new MetaTestReport(testsRan, testsSucceeded, testsFound); } finally { /* Clean up the directory */ deleteDirectory(metaWorkingDir); @@ -75,7 +79,7 @@ public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfigurat /* Set the classloader back to the one with the student's original code */ Thread.currentThread().setContextClassLoader(ctx.getClassloaderWithStudentsCode()); } - return passesTheMetaTest; + return report; } private void verifyMetaTestExecution(Result metaResult, String metaFileContent) { diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java index 654abff8e..50fc15d2e 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java @@ -5,6 +5,7 @@ import nl.tudelft.cse1110.andy.config.RunConfiguration; import nl.tudelft.cse1110.andy.execution.Context.Context; import nl.tudelft.cse1110.andy.execution.ExecutionStep; +import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; import nl.tudelft.cse1110.andy.result.MetaTestResult; import nl.tudelft.cse1110.andy.result.ResultBuilder; @@ -30,7 +31,11 @@ public void execute(Context ctx, ResultBuilder result) { for (MetaTest metaTest : metaTests) { - boolean passesTheMetaTest = metaTest.execute(ctx, dirCfg, runCfg); + + MetaTestReport report = metaTest.execute(ctx, dirCfg, runCfg); + result.logQuality(report); + + boolean passesTheMetaTest = report.passesTheMetaTest(); if (passesTheMetaTest) { score += metaTest.getWeight(); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java index 792a26697..ac1d637fa 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java @@ -122,7 +122,7 @@ public static GradeValues fromResults(CoverageResult coverageResults, grades.setCheckGrade(codeCheckResults.getNumberOfPassedChecks(), codeCheckResults.getTotalNumberOfChecks()); grades.setMutationGrade(mutationResults.getKilledMutants(), mutationResults.getTotalNumberOfMutants()); grades.setMetaGrade(metaTestResults.getPassedMetaTests(), metaTestResults.getTotalTests()); - grades.setQualityScore(qualityResult.getScore()); + grades.setQualityScore(qualityResult.computeScore()); // penalty is equal to the sum of the weights of all failed penalty code checks grades.setPenalty(penaltyCodeCheckResults.getCheckResults().stream().mapToInt(check -> check.passed() ? 0 : check.getWeight()).sum() diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeWeight.java b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeWeight.java index bdffd75b5..f5ddae5da 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeWeight.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeWeight.java @@ -64,7 +64,8 @@ public static GradeWeight fromConfig(Map weights) { float mutation = weights.getOrDefault("mutation", 0.0f); float meta = weights.getOrDefault("meta", 0.0f); float codechecks = weights.getOrDefault("codechecks", 0.0f); + float quality = weights.getOrDefault("quality", 0.0f); - return new GradeWeight(coverage, mutation, meta, codechecks); + return new GradeWeight(coverage, mutation, meta, codechecks, quality); } } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index b7dbb89f8..0c7d91de9 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -1,10 +1,12 @@ package nl.tudelft.cse1110.andy.result; -import java.util.Collections; -import java.util.List; +import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; + +import java.util.LinkedList; public class QualityResult { private int score; // between 0 and 1 + private LinkedList metaTestReports; public QualityResult(int score) { // this.score = score; @@ -34,4 +36,14 @@ public String toString() { "score=" + score + '}'; } + + public void considerMetaTest(MetaTestReport metaTestReport) { + metaTestReports.addFirst(metaTestReport); + } + + public int computeScore() { + // dummy + this.score = 1; + return this.score; + } } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index 10a41b6d4..56a4118b2 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -5,6 +5,7 @@ import nl.tudelft.cse1110.andy.execution.Context.Context; import nl.tudelft.cse1110.andy.execution.ExecutionStep; import nl.tudelft.cse1110.andy.execution.externalprocess.ExternalProcess; +import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; import nl.tudelft.cse1110.andy.grade.GradeCalculator; import nl.tudelft.cse1110.andy.grade.GradeValues; import nl.tudelft.cse1110.andy.grade.GradeWeight; @@ -253,8 +254,8 @@ public void logPenaltyMetaTests(int score, int totalTests, List /* * Quality */ - public void logQuality() { - // dummy + public void logQuality(MetaTestReport metaTestReport) { + this.qualityResult.considerMetaTest(metaTestReport); } /* @@ -289,7 +290,7 @@ public Result build() { int finalGrade, penalty; if (ctx.getRunConfiguration() != null) { - GradeValues grades = GradeValues.fromResults(coverageResults, codeCheckResults, mutationResults, metaTestResults, penaltyMetaTestResults, penaltyCodeCheckResults); + GradeValues grades = GradeValues.fromResults(coverageResults, codeCheckResults, mutationResults, metaTestResults, penaltyMetaTestResults, penaltyCodeCheckResults, qualityResult); weights = GradeWeight.fromConfig(ctx.getRunConfiguration().weights()); successMessage = ctx.getRunConfiguration().successMessage(); From 6e6b85450e22efd23ded9a8af0251d25be2f0c2a Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Tue, 6 Jan 2026 18:19:59 +0100 Subject: [PATCH 03/23] Implementing the same logic for RunPenaltyMetaTestsStep and bug fixing in QualityResult --- .../cse1110/andy/execution/step/RunMetaTestsStep.java | 1 - .../andy/execution/step/RunPenaltyMetaTestsStep.java | 6 +++++- .../java/nl/tudelft/cse1110/andy/result/QualityResult.java | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java index 50fc15d2e..d5caca04c 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java @@ -31,7 +31,6 @@ public void execute(Context ctx, ResultBuilder result) { for (MetaTest metaTest : metaTests) { - MetaTestReport report = metaTest.execute(ctx, dirCfg, runCfg); result.logQuality(report); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java index 6041fe20b..597786f00 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java @@ -5,6 +5,7 @@ import nl.tudelft.cse1110.andy.config.RunConfiguration; import nl.tudelft.cse1110.andy.execution.Context.Context; import nl.tudelft.cse1110.andy.execution.ExecutionStep; +import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; import nl.tudelft.cse1110.andy.result.MetaTestResult; import nl.tudelft.cse1110.andy.result.ResultBuilder; @@ -30,7 +31,10 @@ public void execute(Context ctx, ResultBuilder result) { for (MetaTest metaTest : metaTests) { - boolean passesTheMetaTest = metaTest.execute(ctx, dirCfg, runCfg); + MetaTestReport report = metaTest.execute(ctx, dirCfg, runCfg); + result.logQuality(report); + + boolean passesTheMetaTest = report.passesTheMetaTest(); if (passesTheMetaTest) { score += metaTest.getWeight(); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index 0c7d91de9..b97c0773c 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -12,6 +12,7 @@ public QualityResult(int score) { // this.score = score; // dummy: this.score = 1; + metaTestReports = new LinkedList<>(); } public static QualityResult build(int score) { From 40d85350986f8075dff58c3e2fb36fee87c34cb3 Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Tue, 6 Jan 2026 19:06:47 +0100 Subject: [PATCH 04/23] Integration test (and bug fix in ResultBuilder to get a Result with QualityResult) --- .../execution/metatest/MetaTestReport.java | 1 + .../cse1110/andy/result/QualityResult.java | 6 +++++- .../cse1110/andy/result/ResultBuilder.java | 3 ++- .../quality/LimitedNumberOfMetaTestsTest.java | 18 ++++++++++++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 andy/src/test/java/integration/quality/LimitedNumberOfMetaTestsTest.java diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java index 1643dec21..b575ad698 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java @@ -1,6 +1,7 @@ package nl.tudelft.cse1110.andy.execution.metatest; public class MetaTestReport { + private int testsRan; private int testsFound; private int testsSucceeded; diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index b97c0773c..14f5e8545 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -27,6 +27,10 @@ public int getScore() { return score; } + public LinkedList getMetaTestReports() { + return metaTestReports; + } + public void setScore(int score) { this.score = score; } @@ -39,7 +43,7 @@ public String toString() { } public void considerMetaTest(MetaTestReport metaTestReport) { - metaTestReports.addFirst(metaTestReport); + this.metaTestReports.addFirst(metaTestReport); } public int computeScore() { diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index 56a4118b2..e0035e235 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -319,7 +319,8 @@ public Result build() { genericFailureObject, timeInSeconds, weights, - successMessage); + successMessage, + qualityResult); } } diff --git a/andy/src/test/java/integration/quality/LimitedNumberOfMetaTestsTest.java b/andy/src/test/java/integration/quality/LimitedNumberOfMetaTestsTest.java new file mode 100644 index 000000000..678af5c60 --- /dev/null +++ b/andy/src/test/java/integration/quality/LimitedNumberOfMetaTestsTest.java @@ -0,0 +1,18 @@ +package integration.quality; + +import integration.BaseMetaTestsTest; +import nl.tudelft.cse1110.andy.result.Result; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +public class LimitedNumberOfMetaTestsTest extends BaseMetaTestsTest { + + @Test + void allMetaTestsPassing() { + Result result = run("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration"); + + assertThat(result.getQualityResult().getScore()).isEqualTo(1); + assertThat(result.getQualityResult().getMetaTestReports().size()).isEqualTo(4); + } +} From b51a20da852d26f1d2b2ed099522ac2ddedcb74a Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Wed, 7 Jan 2026 18:56:12 +0100 Subject: [PATCH 05/23] Final version of the method to compute score with limit to the number of meta tests each test can trigger. Also an overview of the results can be produced in LimitedNumberOfMetaTestsTest --- .../execution/metatest/MetaTestReport.java | 13 +++++++- .../cse1110/andy/result/QualityResult.java | 27 ++++++++++++++-- .../quality/LimitedNumberOfMetaTestsTest.java | 31 ++++++++++++++++--- 3 files changed, 62 insertions(+), 9 deletions(-) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java index b575ad698..f5e8cfd70 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java @@ -12,8 +12,19 @@ public MetaTestReport(int testsRan, int testsSucceeded, int testsFound) { this.testsFound = testsFound; } - public boolean passesTheMetaTest() { return testsSucceeded < testsRan; } + + public int getTestsRan() { + return testsRan; + } + + public int getTestsFound() { + return testsFound; + } + + public int getTestsSucceeded() { + return testsSucceeded; + } } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index 14f5e8545..5cc8fb0a4 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -47,8 +47,29 @@ public void considerMetaTest(MetaTestReport metaTestReport) { } public int computeScore() { - // dummy - this.score = 1; - return this.score; + + int limitedNumberMetaTestsScore = this.computeLimitedNumberMetaTestsScore(); + + this.score = limitedNumberMetaTestsScore; + + return limitedNumberMetaTestsScore; + } + + /** + * Measures how well tests are isolated. + * Lower scores indicate that individual tests trigger many meta-tests. + */ + private int computeLimitedNumberMetaTestsScore() { + int tolerance = 2; + int penaltyPerExtraTest = 1; + + double score = 100; + for (MetaTestReport r : metaTestReports) { + int triggered = r.getTestsRan() - r.getTestsSucceeded(); + int excess = Math.max(0, triggered - tolerance); + score -= excess * penaltyPerExtraTest; // alternatives: quadratic or logarithmic penalties + } + + return (int) Math.max(0, score); } } diff --git a/andy/src/test/java/integration/quality/LimitedNumberOfMetaTestsTest.java b/andy/src/test/java/integration/quality/LimitedNumberOfMetaTestsTest.java index 678af5c60..aa23dc075 100644 --- a/andy/src/test/java/integration/quality/LimitedNumberOfMetaTestsTest.java +++ b/andy/src/test/java/integration/quality/LimitedNumberOfMetaTestsTest.java @@ -3,16 +3,37 @@ import integration.BaseMetaTestsTest; import nl.tudelft.cse1110.andy.result.Result; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.stream.Stream; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; public class LimitedNumberOfMetaTestsTest extends BaseMetaTestsTest { - @Test - void allMetaTestsPassing() { - Result result = run("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration"); + /** + * For an overview of the results. To be used in production only. + */ + @ParameterizedTest + @MethodSource("testSuitesDST") + void metaTestQualityIsAcceptable( + String libraryFile, + String solutionFile, + String configurationFile + ) { + Result result = run(libraryFile, solutionFile, configurationFile); + + System.out.println("File: " + libraryFile + + "\nQuality score: " + result.getQualityResult().getScore() + "\n\n"); + } - assertThat(result.getQualityResult().getScore()).isEqualTo(1); - assertThat(result.getQualityResult().getMetaTestReports().size()).isEqualTo(4); + static Stream testSuitesDST() { + return Stream.of( + Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration"), + Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration") + ); } } From 49a0dd3ebd86988573771719a2fa6d89943d5c74 Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Tue, 3 Feb 2026 14:20:29 +0100 Subject: [PATCH 06/23] MetaTestReport saves which tests were triggered --- .../andy/execution/metatest/MetaTestReport.java | 17 ++++++++++++++++- .../ExternalProcessMetaTest.java | 5 ++++- .../metatest/library/LibraryMetaTest.java | 6 +++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java index f5e8cfd70..7ca91ce70 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java @@ -1,15 +1,22 @@ package nl.tudelft.cse1110.andy.execution.metatest; +import nl.tudelft.cse1110.andy.result.TestFailureInfo; + +import java.util.List; + public class MetaTestReport { private int testsRan; private int testsFound; private int testsSucceeded; - public MetaTestReport(int testsRan, int testsSucceeded, int testsFound) { + private List testsTriggered; + + public MetaTestReport(int testsRan, int testsSucceeded, int testsFound, List testsTriggered) { this.testsRan = testsRan; this.testsSucceeded = testsSucceeded; this.testsFound = testsFound; + this.testsTriggered = testsTriggered; } public boolean passesTheMetaTest() { @@ -27,4 +34,12 @@ public int getTestsFound() { public int getTestsSucceeded() { return testsSucceeded; } + + public List getTestsTriggered() { + return testsTriggered; + } + + public void setTestsTriggered(List testsTriggered) { + this.testsTriggered = testsTriggered; + } } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/externalprocess/ExternalProcessMetaTest.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/externalprocess/ExternalProcessMetaTest.java index 6c6e74508..e4e1f8636 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/externalprocess/ExternalProcessMetaTest.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/externalprocess/ExternalProcessMetaTest.java @@ -52,7 +52,10 @@ public MetaTestReport execute(Context ctx, DirectoryConfiguration dirCfg, RunCon int testsSucceeded = metaResultBuilder.getTestResults().getTestsSucceeded(); int testsFound = metaResultBuilder.getTestResults().getTestsFound(); // boolean passesTheMetaTest = testsSucceeded < testsRan; - MetaTestReport report = new MetaTestReport(testsRan, testsSucceeded, testsFound); + + var testsFailed = metaResultBuilder.getTestResults().getFailures(); + + MetaTestReport report = new MetaTestReport(testsRan, testsSucceeded, testsFound, testsFailed); return report; } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java index 565656f59..c6632c241 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java @@ -13,6 +13,7 @@ import nl.tudelft.cse1110.andy.result.CompilationErrorInfo; import nl.tudelft.cse1110.andy.result.CompilationResult; import nl.tudelft.cse1110.andy.result.Result; +import nl.tudelft.cse1110.andy.result.TestFailureInfo; import nl.tudelft.cse1110.andy.utils.CodeSnippetUtils; import nl.tudelft.cse1110.andy.utils.FilesUtils; @@ -71,7 +72,10 @@ public MetaTestReport execute(Context ctx, DirectoryConfiguration dirCfg, RunCon int testsSucceeded = metaResult.getTests().getTestsSucceeded(); int testsFound = metaResult.getTests().getTestsFound(); // passesTheMetaTest = testsSucceeded < testsRan; - report = new MetaTestReport(testsRan, testsSucceeded, testsFound); + + var testsFailed = metaResult.getTests().getFailures(); + + report = new MetaTestReport(testsRan, testsSucceeded, testsFound, testsFailed); } finally { /* Clean up the directory */ deleteDirectory(metaWorkingDir); From 6f2220f68bc5c295ed696d628ca92c26d124c132 Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Wed, 4 Feb 2026 13:27:18 +0100 Subject: [PATCH 07/23] StandardPrintWriter includes a quality result (cohesive and isolated tests) --- .../cse1110/andy/result/QualityResult.java | 61 ++++++++---- .../writer/standard/StandardResultWriter.java | 20 ++++ .../quality/LimitedNumberOfMetaTestsTest.java | 39 -------- .../quality/OverviewQualityResults.java | 94 +++++++++++++++++++ 4 files changed, 158 insertions(+), 56 deletions(-) delete mode 100644 andy/src/test/java/integration/quality/LimitedNumberOfMetaTestsTest.java create mode 100644 andy/src/test/java/integration/quality/OverviewQualityResults.java diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index 5cc8fb0a4..49f4a412a 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -2,7 +2,9 @@ import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; +import java.util.HashSet; import java.util.LinkedList; +import java.util.Set; public class QualityResult { private int score; // between 0 and 1 @@ -47,29 +49,54 @@ public void considerMetaTest(MetaTestReport metaTestReport) { } public int computeScore() { + // dummy: + return this.score; + } - int limitedNumberMetaTestsScore = this.computeLimitedNumberMetaTestsScore(); - - this.score = limitedNumberMetaTestsScore; - - return limitedNumberMetaTestsScore; + public long countTests() { + return metaTestReports.getFirst().getTestsFound(); } /** - * Measures how well tests are isolated. - * Lower scores indicate that individual tests trigger many meta-tests. + * Get how many tests cover a single meta-test + * @return the number of such tests */ - private int computeLimitedNumberMetaTestsScore() { - int tolerance = 2; - int penaltyPerExtraTest = 1; - - double score = 100; - for (MetaTestReport r : metaTestReports) { - int triggered = r.getTestsRan() - r.getTestsSucceeded(); - int excess = Math.max(0, triggered - tolerance); - score -= excess * penaltyPerExtraTest; // alternatives: quadratic or logarithmic penalties + public long countCohesiveTests() { + Set cohesiveTests = new HashSet<>(); // list with tests that were triggered once + Set otherTests = new HashSet<>(); // list with tests that were triggered more than once + for (MetaTestReport metaTestReport : metaTestReports) { + for (TestFailureInfo testFailureInfo : metaTestReport.getTestsTriggered()) { + String testName = testFailureInfo.getTestCase(); + if (otherTests.contains(testName)) { + continue; + } else if (cohesiveTests.contains(testName)) { + cohesiveTests.remove(testName); + otherTests.add(testName); + } else { + cohesiveTests.add(testName); + } + } } + return cohesiveTests.size(); + } - return (int) Math.max(0, score); + /** + * Count the number of tests that do not trigger meta-tests already covered by other tests + * @return the number of such tests + */ + public long countIsolatedTests() { + long count = countTests(); + Set unisolatedTests = new HashSet<>(); + for (MetaTestReport metaTestReport : metaTestReports) { + if (metaTestReport.getTestsTriggered().size() == 1) continue; + for (TestFailureInfo testFailureInfo : metaTestReport.getTestsTriggered()) { + String testName = testFailureInfo.getTestCase(); + if (!unisolatedTests.contains(testName)) { + unisolatedTests.add(testName); + count--; + } + } + } + return count; } } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java b/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java index ab40900a8..18a9a4944 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java @@ -1,6 +1,7 @@ package nl.tudelft.cse1110.andy.writer.standard; import nl.tudelft.cse1110.andy.execution.Context.Context; +import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; import nl.tudelft.cse1110.andy.execution.mode.Action; import nl.tudelft.cse1110.andy.execution.mode.Mode; import nl.tudelft.cse1110.andy.execution.mode.ModeActionSelector; @@ -66,6 +67,7 @@ private void writeStdOutFile(Context ctx, Result result) { printMutationTestingResults(result.getMutationTesting()); printCodeCheckResults(ctx, result.getCodeChecks(), result.getPenaltyCodeChecks()); printMetaTestResults(ctx, result.getMetaTests(), result.getPenaltyMetaTests()); + printQualityResults(ctx, result.getQualityResult()); printFinalGrade(ctx, result); printModeAndTimeToRun(ctx, result.getTimeInSeconds()); } @@ -276,6 +278,24 @@ private void printMetaTestResults(Context ctx, MetaTestsResult metaTests, MetaTe } + private void printQualityResults(Context ctx, QualityResult qualityResult) { + + boolean allHints = modeActionSelector(ctx).shouldShowFullHints(); + boolean onlyResult = modeActionSelector(ctx).shouldShowPartialHints(); + + if(!allHints && !onlyResult) + return; + + l("\n--- Quality Results"); + l(String.format("Score: %d", qualityResult.computeScore())); + + if (allHints) { + l(String.format("Cohesive tests: %d/%d", qualityResult.countCohesiveTests(), qualityResult.countTests())); + l(String.format("Independent tests: %d/%d", qualityResult.countIsolatedTests(), qualityResult.countTests())); + } + + } + private void printCoverageResults(CoverageResult coverage) { if(!coverage.wasExecuted()) return; diff --git a/andy/src/test/java/integration/quality/LimitedNumberOfMetaTestsTest.java b/andy/src/test/java/integration/quality/LimitedNumberOfMetaTestsTest.java deleted file mode 100644 index aa23dc075..000000000 --- a/andy/src/test/java/integration/quality/LimitedNumberOfMetaTestsTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package integration.quality; - -import integration.BaseMetaTestsTest; -import nl.tudelft.cse1110.andy.result.Result; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -import java.util.List; -import java.util.stream.Stream; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; - -public class LimitedNumberOfMetaTestsTest extends BaseMetaTestsTest { - - /** - * For an overview of the results. To be used in production only. - */ - @ParameterizedTest - @MethodSource("testSuitesDST") - void metaTestQualityIsAcceptable( - String libraryFile, - String solutionFile, - String configurationFile - ) { - Result result = run(libraryFile, solutionFile, configurationFile); - - System.out.println("File: " + libraryFile + - "\nQuality score: " + result.getQualityResult().getScore() + "\n\n"); - } - - static Stream testSuitesDST() { - return Stream.of( - Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration"), - Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration") - ); - } -} diff --git a/andy/src/test/java/integration/quality/OverviewQualityResults.java b/andy/src/test/java/integration/quality/OverviewQualityResults.java new file mode 100644 index 000000000..f93d078ee --- /dev/null +++ b/andy/src/test/java/integration/quality/OverviewQualityResults.java @@ -0,0 +1,94 @@ +package integration.quality; + +import integration.BaseMetaTestsTest; +import nl.tudelft.cse1110.andy.config.DirectoryConfiguration; +import nl.tudelft.cse1110.andy.execution.Context.Context; +import nl.tudelft.cse1110.andy.execution.mode.Action; +import nl.tudelft.cse1110.andy.execution.mode.Mode; +import nl.tudelft.cse1110.andy.execution.mode.ModeActionSelector; +import nl.tudelft.cse1110.andy.result.*; +import nl.tudelft.cse1110.andy.writer.ResultWriter; +import nl.tudelft.cse1110.andy.writer.standard.CodeSnippetGenerator; +import nl.tudelft.cse1110.andy.writer.standard.RandomAsciiArtGenerator; +import nl.tudelft.cse1110.andy.writer.standard.StandardResultWriter; +import nl.tudelft.cse1110.andy.writer.standard.VersionInformation; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import testutils.ResultTestDataBuilder; +import nl.tudelft.cse1110.andy.result.Result; + +import java.io.File; +import java.io.FileNotFoundException; +import java.nio.file.Path; +import java.util.stream.Stream; + +import static nl.tudelft.cse1110.andy.utils.FilesUtils.concatenateDirectories; +import static nl.tudelft.cse1110.andy.utils.FilesUtils.readFile; +import static org.mockito.Mockito.*; + +/* +This is not a test class. It is a script to run some solutions and capture the output using the writer + */ +public class OverviewQualityResults extends BaseMetaTestsTest { + protected Context ctx = mock(Context.class); + protected VersionInformation versionInformation = new VersionInformation("testVersion", "testBuildTimestamp", "testCommitId"); + protected RandomAsciiArtGenerator asciiArtGenerator = mock(RandomAsciiArtGenerator.class); + protected CodeSnippetGenerator codeSnippetGenerator = mock(CodeSnippetGenerator.class); + protected ResultWriter writer; + + protected ResultWriter buildWriter() { + return new StandardResultWriter(versionInformation, asciiArtGenerator, codeSnippetGenerator); + } + + @TempDir + protected Path reportDir; + + @BeforeEach + void setupMocks() throws FileNotFoundException { + DirectoryConfiguration dirs = new DirectoryConfiguration(null, reportDir.toString()); + when(ctx.getDirectoryConfiguration()).thenReturn(dirs); + ModeActionSelector mas = new ModeActionSelector(Mode.PRACTICE, Action.FULL_WITH_HINTS); + when(ctx.getModeActionSelector()).thenReturn(mas); + when(asciiArtGenerator.getRandomAsciiArt()).thenReturn("random ascii art"); + when(codeSnippetGenerator.generateCodeSnippetFromSolution(any(), anyInt())).thenReturn("arbitrary code snippet"); + } + + @BeforeEach + void createWriter() { + this.writer = buildWriter(); + } + + protected String generatedResult() { + return readFile(new File(concatenateDirectories(reportDir.toString(), "stdout.txt"))); + } + + /** + * For an overview of the results. To be used in production only. + */ + @ParameterizedTest + @MethodSource("testSuitesDST") + void metaTestQualityIsAcceptable( + String libraryFile, + String solutionFile, + String configurationFile + ) { + Result result = run(libraryFile, solutionFile, configurationFile); + + writer.write(ctx, result); + + String output = generatedResult(); + + System.out.println(output); + } + + static Stream testSuitesDST() { + return Stream.of( + Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration"), + Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration") + ); + } +} From a57a4c94ec90781c445a8d44a2d5dcd20e0bc4ac Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Wed, 4 Feb 2026 13:52:46 +0100 Subject: [PATCH 08/23] Add quality to the writer in Weblab + bug fix in counting the number of unit tests --- .../cse1110/andy/result/QualityResult.java | 18 ++++-- .../cse1110/andy/result/ResultBuilder.java | 2 + .../quality/OverviewQualityResults.java | 5 +- .../writer/weblab/WebLabResultWriter.java | 4 ++ .../quality/OverviewQualityResultsWebLab.java | 57 +++++++++++++++++++ 5 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 weblab-runner/src/test/java/weblab/quality/OverviewQualityResultsWebLab.java diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index 49f4a412a..b370cbfd0 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -8,12 +8,13 @@ public class QualityResult { private int score; // between 0 and 1 + private int numUnitTests; private LinkedList metaTestReports; - public QualityResult(int score) { - // this.score = score; + public QualityResult(int numUnitTests) { // dummy: - this.score = 1; + this.score = 0; + this.numUnitTests = numUnitTests; metaTestReports = new LinkedList<>(); } @@ -37,6 +38,14 @@ public void setScore(int score) { this.score = score; } + public int getNumUnitTests() { + return numUnitTests; + } + + public void setNumUnitTests(int numUnitTests) { + this.numUnitTests = numUnitTests; + } + @Override public String toString() { return "QualityResult{" + @@ -50,11 +59,12 @@ public void considerMetaTest(MetaTestReport metaTestReport) { public int computeScore() { // dummy: + this.score = 1; return this.score; } public long countTests() { - return metaTestReports.getFirst().getTestsFound(); + return numUnitTests; } /** diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index e0035e235..579cd79e3 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -104,6 +104,8 @@ public void logJUnitRun(TestExecutionSummary summary, String console) { failures.add(testFailureInfo); } + this.qualityResult.setNumUnitTests(testsFound); + this.testResults = UnitTestsResult.build(testsFound, testsRan, testsSucceeded, failures, console); } diff --git a/andy/src/test/java/integration/quality/OverviewQualityResults.java b/andy/src/test/java/integration/quality/OverviewQualityResults.java index f93d078ee..ce005468a 100644 --- a/andy/src/test/java/integration/quality/OverviewQualityResults.java +++ b/andy/src/test/java/integration/quality/OverviewQualityResults.java @@ -70,7 +70,7 @@ protected String generatedResult() { * For an overview of the results. To be used in production only. */ @ParameterizedTest - @MethodSource("testSuitesDST") + @MethodSource("testSuites") void metaTestQualityIsAcceptable( String libraryFile, String solutionFile, @@ -85,9 +85,8 @@ void metaTestQualityIsAcceptable( System.out.println(output); } - static Stream testSuitesDST() { + static Stream testSuites() { return Stream.of( - Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration"), Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration") ); } diff --git a/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java b/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java index b3e49f973..2b06f6bcc 100644 --- a/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java +++ b/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java @@ -98,7 +98,11 @@ private static void appendMetaScoreElements(Result result, Document doc, Element appendMetaScore(doc, metaElement, "Mutation coverage", result.getMutationTesting().getKilledMutants()); appendMetaScore(doc, metaElement, "Code checks", result.getCodeChecks().getNumberOfPassedChecks()); appendMetaScore(doc, metaElement, "Meta tests", result.getMetaTests().getPassedMetaTests()); + appendMetaScore(doc, metaElement, "Quality score", result.getQualityResult().computeScore()); + appendMetaScore(doc, metaElement, "Cohesive tests", (int) result.getQualityResult().countCohesiveTests()); + appendMetaScore(doc, metaElement, "Independent tests", (int) result.getQualityResult().countIsolatedTests()); + // Existing code checks and meta tests result.getCodeChecks().getCheckResults().forEach(check -> appendMetaScore(doc, metaElement, check.getDescription(), check.passed() ? 1 : 0)); result.getPenaltyCodeChecks().getCheckResults().forEach(check -> appendMetaScore(doc, metaElement, check.getDescription(), check.passed() ? 1 : 0)); result.getMetaTests().getMetaTestResults().forEach(metaTest -> appendMetaScore(doc, metaElement, metaTest.getName(), metaTest.succeeded() ? 1 : 0)); diff --git a/weblab-runner/src/test/java/weblab/quality/OverviewQualityResultsWebLab.java b/weblab-runner/src/test/java/weblab/quality/OverviewQualityResultsWebLab.java new file mode 100644 index 000000000..9f7d25a47 --- /dev/null +++ b/weblab-runner/src/test/java/weblab/quality/OverviewQualityResultsWebLab.java @@ -0,0 +1,57 @@ +package weblab.quality; + +import integration.quality.OverviewQualityResults; +import nl.tudelft.cse1110.andy.result.CompilationErrorInfo; +import nl.tudelft.cse1110.andy.result.Result; +import nl.tudelft.cse1110.andy.writer.ResultWriter; +import nl.tudelft.cse1110.andy.writer.weblab.WebLabResultWriter; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import testutils.ResultTestDataBuilder; + +import java.io.File; +import java.util.stream.Stream; + +import static nl.tudelft.cse1110.andy.utils.FilesUtils.concatenateDirectories; +import static nl.tudelft.cse1110.andy.utils.FilesUtils.readFile; +import static org.assertj.core.api.Assertions.assertThat; +import static utils.WebLabEditorFeedbackJsonTestAssertions.editorFeedbackCompilationError; + +public class OverviewQualityResultsWebLab extends OverviewQualityResults { + + @Override + protected ResultWriter buildWriter() { + return new WebLabResultWriter(versionInformation, asciiArtGenerator, codeSnippetGenerator); + } + + private String editorFeedbackJson() { + return readFile(new File(concatenateDirectories(reportDir.toString(), "editor-feedback.json"))); + } + + /** + * For an overview of the results. To be used in production only. + */ + @ParameterizedTest + @MethodSource("testSuites") + void metaTestQualityIsAcceptable( + String libraryFile, + String solutionFile, + String configurationFile + ) { + Result result = run(libraryFile, solutionFile, configurationFile); + + writer.write(ctx, result); + + String output = generatedResult(); + + System.out.println(output); + } + + static Stream testSuites() { + return Stream.of( + Arguments.of("ZagZigLibrary", "ZagZigAllMutantsKilled", "ZagZigDifferentTotalMutantsConfiguration") + ); + } +} From 59f46af14595b002108b779c275859c3502d3d38 Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Wed, 4 Feb 2026 15:52:36 +0100 Subject: [PATCH 09/23] Adding unit tests for the cohesion and isolation metrics --- .../integration/quality/CohesionTest.java | 91 +++++++++++++++++++ .../integration/quality/IsolationTest.java | 91 +++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 andy/src/test/java/integration/quality/CohesionTest.java create mode 100644 andy/src/test/java/integration/quality/IsolationTest.java diff --git a/andy/src/test/java/integration/quality/CohesionTest.java b/andy/src/test/java/integration/quality/CohesionTest.java new file mode 100644 index 000000000..6657f64c2 --- /dev/null +++ b/andy/src/test/java/integration/quality/CohesionTest.java @@ -0,0 +1,91 @@ +package integration.quality; + +import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; +import nl.tudelft.cse1110.andy.result.QualityResult; +import nl.tudelft.cse1110.andy.result.TestFailureInfo; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class CohesionTest { + + @Test + public void testTriggersOneMetaTestTest() { + QualityResult qualityResult = new QualityResult(1); + + TestFailureInfo failure = new TestFailureInfo("test", "error message"); + MetaTestReport metaTestReport = new MetaTestReport(1, 0, 1, List.of(failure)); + qualityResult.considerMetaTest(metaTestReport); + + assertEquals(1, qualityResult.countCohesiveTests()); + } + + @Test + public void testTriggersNoMetaTestTest() { + QualityResult qualityResult = new QualityResult(1); + + MetaTestReport metaTestReport = new MetaTestReport(1, 1, 1, List.of()); + qualityResult.considerMetaTest(metaTestReport); + + assertEquals(0, qualityResult.countCohesiveTests()); + } + + @Test + public void sameTestTriggersTwoMetaTestsTest() { + QualityResult qualityResult = new QualityResult(1); + + TestFailureInfo failure1 = new TestFailureInfo("test", "error message"); + TestFailureInfo failure2 = new TestFailureInfo("test", "some other error message"); + MetaTestReport metaTestReport1 = new MetaTestReport(1, 1, 1, List.of(failure1)); + MetaTestReport metaTestReport2 = new MetaTestReport(1, 1, 1, List.of(failure2)); + qualityResult.considerMetaTest(metaTestReport1); + qualityResult.considerMetaTest(metaTestReport2); + + assertEquals(0, qualityResult.countCohesiveTests()); + } + + @Test + public void differentTestsTriggerDifferentMetaTestsTest() { + QualityResult qualityResult = new QualityResult(2); + + TestFailureInfo failure1 = new TestFailureInfo("test 1", "error message"); + TestFailureInfo failure2 = new TestFailureInfo("test 2", "some other error message"); + MetaTestReport metaTestReport1 = new MetaTestReport(2, 1, 2, List.of(failure1)); + MetaTestReport metaTestReport2 = new MetaTestReport(2, 1, 2, List.of(failure2)); + qualityResult.considerMetaTest(metaTestReport1); + qualityResult.considerMetaTest(metaTestReport2); + + assertEquals(2, qualityResult.countCohesiveTests()); + } + + @Test + public void differentTestsTriggerSameMetaTestsTest() { + QualityResult qualityResult = new QualityResult(2); + + TestFailureInfo failure1 = new TestFailureInfo("test 1", "error message"); + TestFailureInfo failure2 = new TestFailureInfo("test 2", "some other error message"); + MetaTestReport metaTestReport = new MetaTestReport(2, 0, 2, List.of(failure1, failure2)); + qualityResult.considerMetaTest(metaTestReport); + + assertEquals(2, qualityResult.countCohesiveTests()); + } + + @Test + public void noTestsTest() { + QualityResult qualityResult = new QualityResult(0); + + MetaTestReport metaTestReport = new MetaTestReport(0, 0, 0, List.of()); + qualityResult.considerMetaTest(metaTestReport); + + assertEquals(0, qualityResult.countCohesiveTests()); + } + + @Test + public void noMetaTestsTest() { + QualityResult qualityResult = new QualityResult(1); + + assertEquals(0, qualityResult.countCohesiveTests()); + } +} diff --git a/andy/src/test/java/integration/quality/IsolationTest.java b/andy/src/test/java/integration/quality/IsolationTest.java new file mode 100644 index 000000000..59f479818 --- /dev/null +++ b/andy/src/test/java/integration/quality/IsolationTest.java @@ -0,0 +1,91 @@ +package integration.quality; + +import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; +import nl.tudelft.cse1110.andy.result.QualityResult; +import nl.tudelft.cse1110.andy.result.TestFailureInfo; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class IsolationTest { + + @Test + public void testTriggersOneMetaTestTest() { + QualityResult qualityResult = new QualityResult(1); + + TestFailureInfo failure = new TestFailureInfo("test", "error message"); + MetaTestReport metaTestReport = new MetaTestReport(1, 0, 1, List.of(failure)); + qualityResult.considerMetaTest(metaTestReport); + + assertEquals(1, qualityResult.countIsolatedTests()); + } + + @Test + public void testTriggersNoMetaTestTest() { + QualityResult qualityResult = new QualityResult(1); + + MetaTestReport metaTestReport = new MetaTestReport(1, 1, 1, List.of()); + qualityResult.considerMetaTest(metaTestReport); + + assertEquals(1, qualityResult.countIsolatedTests()); + } + + @Test + public void sameTestTriggersTwoMetaTestsTest() { + QualityResult qualityResult = new QualityResult(1); + + TestFailureInfo failure1 = new TestFailureInfo("test", "error message"); + TestFailureInfo failure2 = new TestFailureInfo("test", "some other error message"); + MetaTestReport metaTestReport1 = new MetaTestReport(1, 1, 1, List.of(failure1)); + MetaTestReport metaTestReport2 = new MetaTestReport(1, 1, 1, List.of(failure2)); + qualityResult.considerMetaTest(metaTestReport1); + qualityResult.considerMetaTest(metaTestReport2); + + assertEquals(1, qualityResult.countIsolatedTests()); + } + + @Test + public void differentTestsTriggerDifferentMetaTestsTest() { + QualityResult qualityResult = new QualityResult(2); + + TestFailureInfo failure1 = new TestFailureInfo("test 1", "error message"); + TestFailureInfo failure2 = new TestFailureInfo("test 2", "some other error message"); + MetaTestReport metaTestReport1 = new MetaTestReport(2, 1, 2, List.of(failure1)); + MetaTestReport metaTestReport2 = new MetaTestReport(2, 1, 2, List.of(failure2)); + qualityResult.considerMetaTest(metaTestReport1); + qualityResult.considerMetaTest(metaTestReport2); + + assertEquals(2, qualityResult.countIsolatedTests()); + } + + @Test + public void differentTestsTriggerSameMetaTestsTest() { + QualityResult qualityResult = new QualityResult(2); + + TestFailureInfo failure1 = new TestFailureInfo("test 1", "error message"); + TestFailureInfo failure2 = new TestFailureInfo("test 2", "some other error message"); + MetaTestReport metaTestReport = new MetaTestReport(2, 0, 2, List.of(failure1, failure2)); + qualityResult.considerMetaTest(metaTestReport); + + assertEquals(0, qualityResult.countIsolatedTests()); + } + + @Test + public void noTestsTest() { + QualityResult qualityResult = new QualityResult(0); + + MetaTestReport metaTestReport = new MetaTestReport(0, 0, 0, List.of()); + qualityResult.considerMetaTest(metaTestReport); + + assertEquals(0, qualityResult.countIsolatedTests()); + } + + @Test + public void noMetaTestsTest() { + QualityResult qualityResult = new QualityResult(1); + + assertEquals(1, qualityResult.countIsolatedTests()); + } +} From 6c2489413142af4b2be10f0ab6a4207e5d5233f7 Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Wed, 4 Feb 2026 17:19:32 +0100 Subject: [PATCH 10/23] QualityResult keeps track of coverage per test in the class under test --- .../step/CollectCoverageInformationStep.java | 103 +++++++++++++++++- .../execution/step/RunJUnitTestsStep.java | 28 ++++- .../andy/execution/step/RunMetaTestsStep.java | 2 +- .../step/RunPenaltyMetaTestsStep.java | 2 +- .../cse1110/andy/result/QualityResult.java | 23 +++- .../cse1110/andy/result/ResultBuilder.java | 14 ++- 6 files changed, 157 insertions(+), 15 deletions(-) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java index e227acedd..f9d484997 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java @@ -8,24 +8,30 @@ import nl.tudelft.cse1110.andy.utils.ClassUtils; import nl.tudelft.cse1110.andy.utils.FilesUtils; import nl.tudelft.cse1110.andy.utils.FromBytesClassLoader; -import org.jacoco.core.analysis.Analyzer; -import org.jacoco.core.analysis.CoverageBuilder; -import org.jacoco.core.analysis.IClassCoverage; +import org.jacoco.core.analysis.*; +import org.jacoco.core.data.ExecutionData; import org.jacoco.core.data.ExecutionDataStore; import org.jacoco.core.data.SessionInfoStore; import org.jacoco.core.instr.Instrumenter; import org.jacoco.core.runtime.IRuntime; +import org.jacoco.core.runtime.LoggerRuntime; import org.jacoco.core.runtime.RuntimeData; import org.jacoco.report.DirectorySourceFileLocator; import org.jacoco.report.FileMultiReportOutput; import org.jacoco.report.IReportVisitor; import org.jacoco.report.html.HTMLFormatter; +import org.junit.platform.engine.discovery.DiscoverySelectors; +import org.junit.platform.launcher.Launcher; +import org.junit.platform.launcher.LauncherDiscoveryRequest; +import org.junit.platform.launcher.TestIdentifier; +import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; +import org.junit.platform.launcher.core.LauncherFactory; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.Collection; +import java.util.*; import static nl.tudelft.cse1110.andy.utils.ClassUtils.clazzNameAsPath; import static nl.tudelft.cse1110.andy.utils.FilesUtils.concatenateDirectories; @@ -76,7 +82,20 @@ public void execute(Context ctx, ResultBuilder result) { /* Generate an HTML report.*/ String testClass = ClassUtils.getTestClass(ctx.getNewClassNames()); this.generateReport(dirCfg, testClass, coverageBuilder, executionData, sessionInfos); - } catch (IOException e) { + + /* + Log the lines covered by each test by scanning the method line by line. + */ + + List tests = result.getQualityResult().getUnitTests().stream() + .map(TestIdentifier::getDisplayName) + .toList(); + + Map>> coveragePerTest = linesCoveredPerTest(ctx, tests); + + result.logCoveragePerTest(coveragePerTest); + + } catch (Exception e) { throw new RuntimeException(e); } finally { /* Restore the old class loader to get the non-instrumented classes back.*/ @@ -84,6 +103,80 @@ public void execute(Context ctx, ResultBuilder result) { } } + private Map>> linesCoveredPerTest(Context ctx, List tests) throws Exception { + + DirectoryConfiguration dirCfg = ctx.getDirectoryConfiguration(); + RunConfiguration runCfg = ctx.getRunConfiguration(); + + Map>> coveragePerTest = new HashMap<>(); + + for (String testName : tests) { + + // 1. Fresh JaCoCo data for this test + RuntimeData data = new RuntimeData(); + LoggerRuntime runtime = new LoggerRuntime(); // or whatever runtime you use + runtime.startup(data); + + // 2. Run exactly one test + runSingleTest(testName); + + // 3. Collect execution data + ExecutionDataStore executionData = new ExecutionDataStore(); + SessionInfoStore sessionInfos = new SessionInfoStore(); + data.collect(executionData, sessionInfos, false); + runtime.shutdown(); + + // 4. Analyze coverage + CoverageBuilder coverageBuilder = new CoverageBuilder(); + Analyzer analyzer = new Analyzer(executionData, coverageBuilder); + + for (String classUnderTest : runCfg.classesUnderTest()) { + try (InputStream in = + getClassAsInputStream(dirCfg.getWorkingDir(), classUnderTest)) { + analyzer.analyzeClass(in, classUnderTest); + } + } + + // 5. Extract covered lines + Map> coveredLines = extractCoveredLines(coverageBuilder); + + coveragePerTest.put(testName, coveredLines); + } + return coveragePerTest; + } + + private void runSingleTest(String testId) { + LauncherDiscoveryRequest request = + LauncherDiscoveryRequestBuilder.request() + .selectors(DiscoverySelectors.selectUniqueId(testId)) + .build(); + + Launcher launcher = LauncherFactory.create(); + launcher.execute(request); + } + + private Map> extractCoveredLines( + CoverageBuilder coverageBuilder) { + + Map> result = new HashMap<>(); + + for (IClassCoverage cc : coverageBuilder.getClasses()) { + String className = cc.getName().replace('/', '.'); + + for (int line = cc.getFirstLine(); line <= cc.getLastLine(); line++) { + ILine l = cc.getLine(line); + if (l.getStatus() == ICounter.FULLY_COVERED || + l.getStatus() == ICounter.PARTLY_COVERED) { + + result + .computeIfAbsent(className, c -> new HashSet<>()) + .add(line); + } + } + } + return result; + } + /**Instrument all classes in a directory.*/ private void instrumentAllInDirectory(Instrumenter instr, File directory, FromBytesClassLoader classLoader, String start) throws IOException { File[] files = directory.listFiles(); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java index 107a72e28..38ac55abd 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java @@ -5,11 +5,9 @@ import nl.tudelft.cse1110.andy.result.ResultBuilder; import nl.tudelft.cse1110.andy.utils.ClassUtils; import nl.tudelft.cse1110.andy.utils.FilesUtils; +import org.junit.platform.engine.discovery.DiscoverySelectors; import org.junit.platform.engine.reporting.ReportEntry; -import org.junit.platform.launcher.Launcher; -import org.junit.platform.launcher.LauncherDiscoveryRequest; -import org.junit.platform.launcher.TestExecutionListener; -import org.junit.platform.launcher.TestIdentifier; +import org.junit.platform.launcher.*; import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; import org.junit.platform.launcher.core.LauncherFactory; import org.junit.platform.launcher.listeners.SummaryGeneratingListener; @@ -20,6 +18,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; import java.util.Properties; import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; @@ -69,6 +69,26 @@ public void execute(Context ctx, ResultBuilder result) { /* Log the junit result */ result.logJUnitRun(summary, output.toString()); + + /* + Log the unit tests available for the quality metrics + */ + request = + LauncherDiscoveryRequestBuilder.request() + .selectors(DiscoverySelectors.selectPackage("org.example")) + .build(); + + launcher = LauncherFactory.create(); + TestPlan plan = launcher.discover(request); + + List unitTests = new ArrayList<>(); + + plan.getRoots().forEach(root -> + plan.getDescendants(root).stream() + .filter(TestIdentifier::isTest) + .forEach(unitTests::add)); + + result.logUnitTests(unitTests); } catch (Exception e) { result.genericFailure(this, e); } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java index d5caca04c..af0693435 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java @@ -32,7 +32,7 @@ public void execute(Context ctx, ResultBuilder result) { for (MetaTest metaTest : metaTests) { MetaTestReport report = metaTest.execute(ctx, dirCfg, runCfg); - result.logQuality(report); + result.logMetaTest(report); boolean passesTheMetaTest = report.passesTheMetaTest(); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java index 597786f00..14cd8ace2 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java @@ -32,7 +32,7 @@ public void execute(Context ctx, ResultBuilder result) { for (MetaTest metaTest : metaTests) { MetaTestReport report = metaTest.execute(ctx, dirCfg, runCfg); - result.logQuality(report); + result.logMetaTest(report); boolean passesTheMetaTest = report.passesTheMetaTest(); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index b370cbfd0..98ddc5110 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -1,15 +1,16 @@ package nl.tudelft.cse1110.andy.result; import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; +import org.junit.platform.launcher.TestIdentifier; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.Set; +import java.util.*; public class QualityResult { private int score; // between 0 and 1 private int numUnitTests; + private List unitTests; private LinkedList metaTestReports; + private Map>> coveragePerTest; public QualityResult(int numUnitTests) { // dummy: @@ -46,6 +47,22 @@ public void setNumUnitTests(int numUnitTests) { this.numUnitTests = numUnitTests; } + public List getUnitTests() { + return unitTests; + } + + public void setUnitTests(List unitTests) { + this.unitTests = unitTests; + } + + public Map>> getCoveragePerTest() { + return coveragePerTest; + } + + public void setCoveragePerTest(Map>> coveragePerTest) { + this.coveragePerTest = coveragePerTest; + } + @Override public String toString() { return "QualityResult{" + diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index 579cd79e3..6b4b07b06 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -256,7 +256,19 @@ public void logPenaltyMetaTests(int score, int totalTests, List /* * Quality */ - public void logQuality(MetaTestReport metaTestReport) { + public QualityResult getQualityResult() { + return qualityResult; + } + + public void logUnitTests(List unitTests) { + this.qualityResult.setUnitTests(unitTests); + } + + public void logCoveragePerTest(Map>> coveragePerTest) { + this.qualityResult.setCoveragePerTest(coveragePerTest); + } + + public void logMetaTest(MetaTestReport metaTestReport) { this.qualityResult.considerMetaTest(metaTestReport); } From 6b0995e9f6ebbde455107118f2952c4a292ee0b2 Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Wed, 4 Feb 2026 17:19:52 +0100 Subject: [PATCH 11/23] Now for only that class --- .../step/CollectCoverageInformationStep.java | 25 +++++++++++-------- .../cse1110/andy/result/QualityResult.java | 6 ++--- .../cse1110/andy/result/ResultBuilder.java | 2 +- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java index f9d484997..ca1a0d24c 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java @@ -38,6 +38,7 @@ public class CollectCoverageInformationStep implements ExecutionStep { + @SuppressWarnings("checkstyle:MethodLength") // in this case, it makes no sense to break down the method any farther @Override public void execute(Context ctx, ResultBuilder result) { // Skip step if disabled @@ -86,12 +87,11 @@ public void execute(Context ctx, ResultBuilder result) { /* Log the lines covered by each test by scanning the method line by line. */ - List tests = result.getQualityResult().getUnitTests().stream() .map(TestIdentifier::getDisplayName) .toList(); - Map>> coveragePerTest = linesCoveredPerTest(ctx, tests); + Map> coveragePerTest = linesCoveredPerTest(ctx, testClass, tests); result.logCoveragePerTest(coveragePerTest); @@ -103,12 +103,12 @@ public void execute(Context ctx, ResultBuilder result) { } } - private Map>> linesCoveredPerTest(Context ctx, List tests) throws Exception { + private Map> linesCoveredPerTest(Context ctx, String testClass, List tests) throws Exception { DirectoryConfiguration dirCfg = ctx.getDirectoryConfiguration(); RunConfiguration runCfg = ctx.getRunConfiguration(); - Map>> coveragePerTest = new HashMap<>(); + Map> coveragePerTest = new HashMap<>(); for (String testName : tests) { @@ -138,7 +138,7 @@ private Map>> linesCoveredPerTest(Context ctx, } // 5. Extract covered lines - Map> coveredLines = extractCoveredLines(coverageBuilder); + Set coveredLines = extractCoveredLines(coverageBuilder, testClass); coveragePerTest.put(testName, coveredLines); } @@ -155,22 +155,25 @@ private void runSingleTest(String testId) { launcher.execute(request); } - private Map> extractCoveredLines( - CoverageBuilder coverageBuilder) { + private Set extractCoveredLines( + CoverageBuilder coverageBuilder, + String testClass) { - Map> result = new HashMap<>(); + Set result = new HashSet<>(); for (IClassCoverage cc : coverageBuilder.getClasses()) { String className = cc.getName().replace('/', '.'); + if (!className.equals(testClass)) { + continue; + } + for (int line = cc.getFirstLine(); line <= cc.getLastLine(); line++) { ILine l = cc.getLine(line); if (l.getStatus() == ICounter.FULLY_COVERED || l.getStatus() == ICounter.PARTLY_COVERED) { - result - .computeIfAbsent(className, c -> new HashSet<>()) - .add(line); + result.add(line); } } } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index 98ddc5110..b01c0ba78 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -10,7 +10,7 @@ public class QualityResult { private int numUnitTests; private List unitTests; private LinkedList metaTestReports; - private Map>> coveragePerTest; + private Map> coveragePerTest; public QualityResult(int numUnitTests) { // dummy: @@ -55,11 +55,11 @@ public void setUnitTests(List unitTests) { this.unitTests = unitTests; } - public Map>> getCoveragePerTest() { + public Map> getCoveragePerTest() { return coveragePerTest; } - public void setCoveragePerTest(Map>> coveragePerTest) { + public void setCoveragePerTest(Map> coveragePerTest) { this.coveragePerTest = coveragePerTest; } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index 6b4b07b06..62b183c15 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -264,7 +264,7 @@ public void logUnitTests(List unitTests) { this.qualityResult.setUnitTests(unitTests); } - public void logCoveragePerTest(Map>> coveragePerTest) { + public void logCoveragePerTest(Map> coveragePerTest) { this.qualityResult.setCoveragePerTest(coveragePerTest); } From ccc0303a9afcbe457fb87bd6ec3bdafd35769f7d Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Wed, 4 Feb 2026 17:35:45 +0100 Subject: [PATCH 12/23] Improving method to check for cohesion --- .../step/CollectCoverageInformationStep.java | 4 +-- .../execution/step/RunJUnitTestsStep.java | 4 +-- .../cse1110/andy/result/QualityResult.java | 28 +++++++++---------- .../cse1110/andy/result/ResultBuilder.java | 2 +- .../integration/quality/CohesionTest.java | 18 +++++++++--- 5 files changed, 32 insertions(+), 24 deletions(-) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java index ca1a0d24c..bc45d7a0b 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java @@ -87,9 +87,7 @@ public void execute(Context ctx, ResultBuilder result) { /* Log the lines covered by each test by scanning the method line by line. */ - List tests = result.getQualityResult().getUnitTests().stream() - .map(TestIdentifier::getDisplayName) - .toList(); + List tests = result.getQualityResult().getUnitTests(); Map> coveragePerTest = linesCoveredPerTest(ctx, testClass, tests); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java index 38ac55abd..61fb10df6 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java @@ -81,12 +81,12 @@ public void execute(Context ctx, ResultBuilder result) { launcher = LauncherFactory.create(); TestPlan plan = launcher.discover(request); - List unitTests = new ArrayList<>(); + List unitTests = new ArrayList<>(); plan.getRoots().forEach(root -> plan.getDescendants(root).stream() .filter(TestIdentifier::isTest) - .forEach(unitTests::add)); + .forEach(ti -> unitTests.add(ti.getDisplayName()))); result.logUnitTests(unitTests); } catch (Exception e) { diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index b01c0ba78..056c94bf2 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -8,7 +8,7 @@ public class QualityResult { private int score; // between 0 and 1 private int numUnitTests; - private List unitTests; + private List unitTests; private LinkedList metaTestReports; private Map> coveragePerTest; @@ -47,11 +47,11 @@ public void setNumUnitTests(int numUnitTests) { this.numUnitTests = numUnitTests; } - public List getUnitTests() { + public List getUnitTests() { return unitTests; } - public void setUnitTests(List unitTests) { + public void setUnitTests(List unitTests) { this.unitTests = unitTests; } @@ -89,22 +89,18 @@ public long countTests() { * @return the number of such tests */ public long countCohesiveTests() { - Set cohesiveTests = new HashSet<>(); // list with tests that were triggered once - Set otherTests = new HashSet<>(); // list with tests that were triggered more than once + + Map numTriggers = new HashMap<>(); // list with tests that were triggered once + unitTests.forEach(ut -> numTriggers.put(ut, 0)); + for (MetaTestReport metaTestReport : metaTestReports) { for (TestFailureInfo testFailureInfo : metaTestReport.getTestsTriggered()) { String testName = testFailureInfo.getTestCase(); - if (otherTests.contains(testName)) { - continue; - } else if (cohesiveTests.contains(testName)) { - cohesiveTests.remove(testName); - otherTests.add(testName); - } else { - cohesiveTests.add(testName); - } + numTriggers.put(testName, numTriggers.get(testName) + 1); } } - return cohesiveTests.size(); + + return numTriggers.values().stream().filter(nt -> nt == 1).count(); } /** @@ -112,8 +108,11 @@ public long countCohesiveTests() { * @return the number of such tests */ public long countIsolatedTests() { + long count = countTests(); + Set unisolatedTests = new HashSet<>(); + for (MetaTestReport metaTestReport : metaTestReports) { if (metaTestReport.getTestsTriggered().size() == 1) continue; for (TestFailureInfo testFailureInfo : metaTestReport.getTestsTriggered()) { @@ -124,6 +123,7 @@ public long countIsolatedTests() { } } } + return count; } } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index 62b183c15..bbf2b5e76 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -260,7 +260,7 @@ public QualityResult getQualityResult() { return qualityResult; } - public void logUnitTests(List unitTests) { + public void logUnitTests(List unitTests) { this.qualityResult.setUnitTests(unitTests); } diff --git a/andy/src/test/java/integration/quality/CohesionTest.java b/andy/src/test/java/integration/quality/CohesionTest.java index 6657f64c2..69530d290 100644 --- a/andy/src/test/java/integration/quality/CohesionTest.java +++ b/andy/src/test/java/integration/quality/CohesionTest.java @@ -4,8 +4,11 @@ import nl.tudelft.cse1110.andy.result.QualityResult; import nl.tudelft.cse1110.andy.result.TestFailureInfo; import org.junit.jupiter.api.Test; +import org.junit.platform.launcher.TestIdentifier; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -14,6 +17,7 @@ public class CohesionTest { @Test public void testTriggersOneMetaTestTest() { QualityResult qualityResult = new QualityResult(1); + qualityResult.setUnitTests(List.of("test")); TestFailureInfo failure = new TestFailureInfo("test", "error message"); MetaTestReport metaTestReport = new MetaTestReport(1, 0, 1, List.of(failure)); @@ -25,6 +29,7 @@ public void testTriggersOneMetaTestTest() { @Test public void testTriggersNoMetaTestTest() { QualityResult qualityResult = new QualityResult(1); + qualityResult.setUnitTests(List.of("test")); MetaTestReport metaTestReport = new MetaTestReport(1, 1, 1, List.of()); qualityResult.considerMetaTest(metaTestReport); @@ -35,6 +40,7 @@ public void testTriggersNoMetaTestTest() { @Test public void sameTestTriggersTwoMetaTestsTest() { QualityResult qualityResult = new QualityResult(1); + qualityResult.setUnitTests(List.of("test")); TestFailureInfo failure1 = new TestFailureInfo("test", "error message"); TestFailureInfo failure2 = new TestFailureInfo("test", "some other error message"); @@ -49,9 +55,10 @@ public void sameTestTriggersTwoMetaTestsTest() { @Test public void differentTestsTriggerDifferentMetaTestsTest() { QualityResult qualityResult = new QualityResult(2); + qualityResult.setUnitTests(List.of("test1", "test2")); - TestFailureInfo failure1 = new TestFailureInfo("test 1", "error message"); - TestFailureInfo failure2 = new TestFailureInfo("test 2", "some other error message"); + TestFailureInfo failure1 = new TestFailureInfo("test1", "error message"); + TestFailureInfo failure2 = new TestFailureInfo("test2", "some other error message"); MetaTestReport metaTestReport1 = new MetaTestReport(2, 1, 2, List.of(failure1)); MetaTestReport metaTestReport2 = new MetaTestReport(2, 1, 2, List.of(failure2)); qualityResult.considerMetaTest(metaTestReport1); @@ -63,9 +70,10 @@ public void differentTestsTriggerDifferentMetaTestsTest() { @Test public void differentTestsTriggerSameMetaTestsTest() { QualityResult qualityResult = new QualityResult(2); + qualityResult.setUnitTests(List.of("test1", "test2")); - TestFailureInfo failure1 = new TestFailureInfo("test 1", "error message"); - TestFailureInfo failure2 = new TestFailureInfo("test 2", "some other error message"); + TestFailureInfo failure1 = new TestFailureInfo("test1", "error message"); + TestFailureInfo failure2 = new TestFailureInfo("test2", "some other error message"); MetaTestReport metaTestReport = new MetaTestReport(2, 0, 2, List.of(failure1, failure2)); qualityResult.considerMetaTest(metaTestReport); @@ -75,6 +83,7 @@ public void differentTestsTriggerSameMetaTestsTest() { @Test public void noTestsTest() { QualityResult qualityResult = new QualityResult(0); + qualityResult.setUnitTests(List.of()); MetaTestReport metaTestReport = new MetaTestReport(0, 0, 0, List.of()); qualityResult.considerMetaTest(metaTestReport); @@ -85,6 +94,7 @@ public void noTestsTest() { @Test public void noMetaTestsTest() { QualityResult qualityResult = new QualityResult(1); + qualityResult.setUnitTests(List.of("test")); assertEquals(0, qualityResult.countCohesiveTests()); } From 58f4ae64986336cd381ee0d92895329254172a34 Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Wed, 4 Feb 2026 18:06:19 +0100 Subject: [PATCH 13/23] QualityResult keeps track of mutations killed by each test --- .../andy/execution/step/RunPitestStep.java | 92 ++++++++++++++++++- .../cse1110/andy/result/QualityResult.java | 10 ++ .../cse1110/andy/result/ResultBuilder.java | 4 + 3 files changed, 101 insertions(+), 5 deletions(-) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPitestStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPitestStep.java index 06c542bb9..fa6fe64fa 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPitestStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPitestStep.java @@ -7,23 +7,32 @@ import nl.tudelft.cse1110.andy.result.ResultBuilder; import nl.tudelft.cse1110.andy.utils.ClassUtils; import nl.tudelft.cse1110.andy.utils.FilesUtils; +import org.htmlunit.cyberneko.xerces.dom.ElementImpl; +import org.pitest.mutationtest.DetectionStatus; +import org.pitest.mutationtest.MutationResult; +import org.pitest.mutationtest.MutationStatusTestPair; import org.pitest.mutationtest.commandline.OptionsParser; import org.pitest.mutationtest.commandline.ParseResult; import org.pitest.mutationtest.commandline.PluginFilter; import org.pitest.mutationtest.config.PluginServices; import org.pitest.mutationtest.config.ReportOptions; +import org.pitest.mutationtest.engine.MutationIdentifier; import org.pitest.mutationtest.tooling.AnalysisResult; import org.pitest.mutationtest.tooling.CombinedStatistics; import org.pitest.mutationtest.tooling.EntryPoint; import org.pitest.util.Unchecked; +import org.w3c.dom.Document; +import org.w3c.dom.NodeList; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.PrintStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -59,8 +68,79 @@ public void execute(Context ctx, ResultBuilder result) { result.logPitest(stats); } + + /* + Log the mutations killed by each test. + */ + List tests = result.getQualityResult().getUnitTests(); + + Map> mutationsKilledPerTest = mutationsKilledPerTest(ctx, tests); + + result.logMutationsKilledPerTest(mutationsKilledPerTest); } + private Map> mutationsKilledPerTest( + Context ctx, + List tests) { + + Map> result = new HashMap<>(); + for (String test : tests) result.put(test, new HashSet<>()); + + Path mutationsFile = Paths.get( + ctx.getDirectoryConfiguration().getWorkingDir(), + "pitest", + "mutations.xml" + ); + + if (!Files.exists(mutationsFile)) return result; + + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document doc = builder.parse(mutationsFile.toFile()); + + NodeList mutationNodes = doc.getElementsByTagName("mutation"); + + for (int i = 0; i < mutationNodes.getLength(); i++) { + ElementImpl mutation = (ElementImpl) mutationNodes.item(i); + + String status = mutation.getAttribute("status"); + if (!"KILLED".equals(status)) { + continue; + } + + int mutationId = Integer.parseInt( + mutation.getElementsByTagName("index") + .item(0) + .getTextContent() + ); + + NodeList killingTests = mutation.getElementsByTagName("killingTest"); + + if (killingTests.getLength() == 0) continue; + + String rawTestName = killingTests.item(0).getTextContent(); + + String testName = normalizeTestName(rawTestName); + + result.computeIfAbsent(testName, t -> new HashSet<>()).add(mutationId); + } + } catch (Exception e) { + throw new RuntimeException("Failed to parse mutations.xml", e); + } + + return result; + } + + private String normalizeTestName(String pitTestName) { + int paren = pitTestName.indexOf('('); + if (paren != -1) { + pitTestName = pitTestName.substring(0, paren); + } + return pitTestName.replace('.', '#'); + } + + private String createDirectoryForPitest(Context ctx) { String outputPitestDir = FilesUtils.concatenateDirectories(ctx.getDirectoryConfiguration().getOutputDir(), "pitest"); FilesUtils.createDirIfNeeded(outputPitestDir); @@ -86,7 +166,7 @@ private String[] buildArgs(Context ctx, String pitestOutputDir) { args.add(dirCfg.getWorkingDir()); args.add("--verbose"); - args.add("false"); + args.add("true"); // to allow detecting which mutations are covered by each test args.add("--classPath"); List librariesToInclude = compiledClassesPlusLibraries(ctx, dirCfg); @@ -95,6 +175,8 @@ private String[] buildArgs(Context ctx, String pitestOutputDir) { args.add("--mutators"); args.add(commaSeparated(runCfg.listOfMutants())); + args.add("--exportLineCoverage"); + return args.stream().toArray(String[]::new); } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index 056c94bf2..e2992dd4a 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -9,7 +9,10 @@ public class QualityResult { private int score; // between 0 and 1 private int numUnitTests; private List unitTests; + private LinkedList metaTestReports; + private Map> mutationsKilledPerTest; + private Map> coveragePerTest; public QualityResult(int numUnitTests) { @@ -63,6 +66,13 @@ public void setCoveragePerTest(Map> coveragePerTest) { this.coveragePerTest = coveragePerTest; } + public Map> getMutationsKilledPerTest() { + return mutationsKilledPerTest; + } + + public void setMutationsKilledPerTest(Map> mutationsKilledPerTest) { + } + @Override public String toString() { return "QualityResult{" + diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index bbf2b5e76..60d36fb1f 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -268,6 +268,10 @@ public void logCoveragePerTest(Map> coveragePerTest) { this.qualityResult.setCoveragePerTest(coveragePerTest); } + public void logMutationsKilledPerTest(Map> mutationsKilledPerTest) { + this.qualityResult.setMutationsKilledPerTest(mutationsKilledPerTest); + } + public void logMetaTest(MetaTestReport metaTestReport) { this.qualityResult.considerMetaTest(metaTestReport); } From 76c3f18ff0e7d198b82868709a53d3230498f943 Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Wed, 4 Feb 2026 19:21:01 +0100 Subject: [PATCH 14/23] Ability to check how many meaningful tests there are --- .../execution/metatest/MetaTestReport.java | 10 +++ .../andy/execution/step/RunMetaTestsStep.java | 2 +- .../cse1110/andy/result/QualityResult.java | 68 +++++++++++++++---- .../cse1110/andy/result/ResultBuilder.java | 3 +- 4 files changed, 67 insertions(+), 16 deletions(-) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java index 7ca91ce70..bf3ceb914 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/MetaTestReport.java @@ -6,6 +6,8 @@ public class MetaTestReport { + private String name; + private int testsRan; private int testsFound; private int testsSucceeded; @@ -19,6 +21,14 @@ public MetaTestReport(int testsRan, int testsSucceeded, int testsFound, List unitTests; private LinkedList metaTestReports; - private Map> mutationsKilledPerTest; + private Map> testToMetaTests; // a useful mapping from the test cases to the meta-tests they cover private Map> coveragePerTest; + private Map> mutationsKilledPerTest; public QualityResult(int numUnitTests) { // dummy: this.score = 0; this.numUnitTests = numUnitTests; metaTestReports = new LinkedList<>(); + testToMetaTests = new HashMap<>(); } public static QualityResult build(int score) { @@ -82,6 +83,14 @@ public String toString() { public void considerMetaTest(MetaTestReport metaTestReport) { this.metaTestReports.addFirst(metaTestReport); + + for (TestFailureInfo failure : metaTestReport.getTestsTriggered()) { + String test = failure.getTestCase(); + if (testToMetaTests.get(test) == null) { + testToMetaTests.put(test, new HashSet<>()); + } + testToMetaTests.get(test).add(metaTestReport.getName()); + } } public int computeScore() { @@ -99,18 +108,7 @@ public long countTests() { * @return the number of such tests */ public long countCohesiveTests() { - - Map numTriggers = new HashMap<>(); // list with tests that were triggered once - unitTests.forEach(ut -> numTriggers.put(ut, 0)); - - for (MetaTestReport metaTestReport : metaTestReports) { - for (TestFailureInfo testFailureInfo : metaTestReport.getTestsTriggered()) { - String testName = testFailureInfo.getTestCase(); - numTriggers.put(testName, numTriggers.get(testName) + 1); - } - } - - return numTriggers.values().stream().filter(nt -> nt == 1).count(); + return testToMetaTests.values().stream().filter(nt -> nt.size() == 1).count(); } /** @@ -136,4 +134,46 @@ public long countIsolatedTests() { return count; } + + /** + * Count the number of tests that increase one of: + * 1) number of meta tests triggered + * 2) lines covered + * 3) mutations killed + * @return the number of such tests + */ + private long countMeaningfulTests() { + + Set meaningfulTests = new HashSet<>(); + + meaningfulTests.addAll(meaningfulCoverage(testToMetaTests)); // 1) + + meaningfulTests.addAll(meaningfulCoverage(coveragePerTest)); // 2) + + meaningfulTests.addAll(meaningfulCoverage(mutationsKilledPerTest)); // 3) + + return meaningfulTests.size(); + } + + private Set meaningfulCoverage(Map> map) { + + Set meaningfulTests = new HashSet<>(); + + for (String test : map.keySet()) { + + Set meaningfulLines = new HashSet<>(map.get(test)); + + for (String otherTest : map.keySet()) { + if (test.equals(otherTest)) continue; + Set linesWithOtherTest = map.get(otherTest); + meaningfulLines.removeAll(linesWithOtherTest); + } + + if (!meaningfulLines.isEmpty() && !meaningfulTests.contains(test)) { + meaningfulTests.add(test); + } + } + + return meaningfulTests; + } } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index 60d36fb1f..1ce2529ef 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -272,7 +272,8 @@ public void logMutationsKilledPerTest(Map> mutationsKilledP this.qualityResult.setMutationsKilledPerTest(mutationsKilledPerTest); } - public void logMetaTest(MetaTestReport metaTestReport) { + public void logMetaTest(String name, MetaTestReport metaTestReport) { + metaTestReport.setName(name); this.qualityResult.considerMetaTest(metaTestReport); } From 2aef104e8fb2a969c6152e9bc5f84f254278d0c3 Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Wed, 4 Feb 2026 20:12:09 +0100 Subject: [PATCH 15/23] Writer has the result of meanignful tests, but there are some bugs to be fixed --- .../andy/execution/step/RunJUnitTestsStep.java | 15 +++++++++++---- .../execution/step/RunPenaltyMetaTestsStep.java | 2 +- .../cse1110/andy/result/QualityResult.java | 11 ++++++----- .../writer/standard/StandardResultWriter.java | 6 ++++-- .../andy/writer/weblab/WebLabResultWriter.java | 1 + 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java index 61fb10df6..39bca95f9 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java @@ -83,10 +83,17 @@ public void execute(Context ctx, ResultBuilder result) { List unitTests = new ArrayList<>(); - plan.getRoots().forEach(root -> - plan.getDescendants(root).stream() - .filter(TestIdentifier::isTest) - .forEach(ti -> unitTests.add(ti.getDisplayName()))); + int s = plan.getRoots().size(); + int d = plan.getDescendants(plan.getRoots().iterator().next()).size(); + +// plan.getRoots().forEach(root -> +// plan.getDescendants(root).stream() +// .filter(TestIdentifier::isTest) +// .forEach(ti -> unitTests.add(ti.getDisplayName()))); + + plan.getRoots().forEach(root -> { + unitTests.add(root.getDisplayName()); + }); result.logUnitTests(unitTests); } catch (Exception e) { diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java index 14cd8ace2..f12347871 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java @@ -32,7 +32,7 @@ public void execute(Context ctx, ResultBuilder result) { for (MetaTest metaTest : metaTests) { MetaTestReport report = metaTest.execute(ctx, dirCfg, runCfg); - result.logMetaTest(report); + result.logMetaTest(metaTest.getName(), report); boolean passesTheMetaTest = report.passesTheMetaTest(); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index 23d2da8b9..7ddf4e524 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -72,6 +72,7 @@ public Map> getMutationsKilledPerTest() { } public void setMutationsKilledPerTest(Map> mutationsKilledPerTest) { + this.mutationsKilledPerTest = mutationsKilledPerTest; } @Override @@ -142,7 +143,7 @@ public long countIsolatedTests() { * 3) mutations killed * @return the number of such tests */ - private long countMeaningfulTests() { + public long countMeaningfulTests() { Set meaningfulTests = new HashSet<>(); @@ -161,15 +162,15 @@ private Set meaningfulCoverage(Map> map) { for (String test : map.keySet()) { - Set meaningfulLines = new HashSet<>(map.get(test)); + Set meaningfulContributions = new HashSet<>(map.get(test)); for (String otherTest : map.keySet()) { if (test.equals(otherTest)) continue; - Set linesWithOtherTest = map.get(otherTest); - meaningfulLines.removeAll(linesWithOtherTest); + Set contributionsByOtherTest = new HashSet<>(map.get(otherTest)); + meaningfulContributions.removeAll(contributionsByOtherTest); } - if (!meaningfulLines.isEmpty() && !meaningfulTests.contains(test)) { + if (!meaningfulContributions.isEmpty()) { meaningfulTests.add(test); } } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java b/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java index 18a9a4944..584952a1c 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java @@ -290,8 +290,10 @@ private void printQualityResults(Context ctx, QualityResult qualityResult) { l(String.format("Score: %d", qualityResult.computeScore())); if (allHints) { - l(String.format("Cohesive tests: %d/%d", qualityResult.countCohesiveTests(), qualityResult.countTests())); - l(String.format("Independent tests: %d/%d", qualityResult.countIsolatedTests(), qualityResult.countTests())); + long allTests = qualityResult.countTests(); + l(String.format("Cohesive tests: %d/%d", qualityResult.countCohesiveTests(), allTests)); + l(String.format("Independent tests: %d/%d", qualityResult.countIsolatedTests(), allTests)); + l(String.format("Meaningful tests: %d/%d", qualityResult.countMeaningfulTests(), allTests)); } } diff --git a/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java b/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java index 2b06f6bcc..c920d0a4a 100644 --- a/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java +++ b/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java @@ -101,6 +101,7 @@ private static void appendMetaScoreElements(Result result, Document doc, Element appendMetaScore(doc, metaElement, "Quality score", result.getQualityResult().computeScore()); appendMetaScore(doc, metaElement, "Cohesive tests", (int) result.getQualityResult().countCohesiveTests()); appendMetaScore(doc, metaElement, "Independent tests", (int) result.getQualityResult().countIsolatedTests()); + appendMetaScore(doc, metaElement, "Meaningful tests", (int) result.getQualityResult().countMeaningfulTests()); // Existing code checks and meta tests result.getCodeChecks().getCheckResults().forEach(check -> appendMetaScore(doc, metaElement, check.getDescription(), check.passed() ? 1 : 0)); From 7f77fc7a40d8a0ddbf24aede0e7326c85d050f2e Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Sat, 14 Mar 2026 17:37:48 +0100 Subject: [PATCH 16/23] Fixing contributing tests --- .../step/CollectCoverageInformationStep.java | 41 ++-- .../execution/step/RunJUnitTestsStep.java | 52 +++-- .../andy/execution/step/RunPitestStep.java | 74 ++++--- .../cse1110/andy/result/QualityResult.java | 44 ++-- .../cse1110/andy/result/ResultBuilder.java | 40 +++- .../writer/standard/StandardResultWriter.java | 3 +- .../integration/quality/CohesionTest.java | 202 +++++++++--------- .../NumberUtilsAddOfficialSolution.java | 3 +- .../writer/weblab/WebLabResultWriter.java | 2 +- 9 files changed, 247 insertions(+), 214 deletions(-) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java index bc45d7a0b..6096fccc0 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java @@ -87,7 +87,7 @@ public void execute(Context ctx, ResultBuilder result) { /* Log the lines covered by each test by scanning the method line by line. */ - List tests = result.getQualityResult().getUnitTests(); + Map tests = result.getQualityResult().getUnitTests(); Map> coveragePerTest = linesCoveredPerTest(ctx, testClass, tests); @@ -101,45 +101,52 @@ public void execute(Context ctx, ResultBuilder result) { } } - private Map> linesCoveredPerTest(Context ctx, String testClass, List tests) throws Exception { - + private Map> linesCoveredPerTest(Context ctx, String testClass, Map tests) throws Exception { DirectoryConfiguration dirCfg = ctx.getDirectoryConfiguration(); RunConfiguration runCfg = ctx.getRunConfiguration(); - Map> coveragePerTest = new HashMap<>(); - for (String testName : tests) { + for (Map.Entry entry : tests.entrySet()) { + String uniqueId = entry.getKey(); + String displayName = entry.getValue(); - // 1. Fresh JaCoCo data for this test + // 1. Fresh runtime + data for this test + IRuntime runtime = new LoggerRuntime(); RuntimeData data = new RuntimeData(); - LoggerRuntime runtime = new LoggerRuntime(); // or whatever runtime you use runtime.startup(data); - // 2. Run exactly one test - runSingleTest(testName); + // 2. Re-instrument into a fresh classloader + Instrumenter instr = new Instrumenter(runtime); + ClassLoader current = Thread.currentThread().getContextClassLoader().getParent(); // use parent to avoid already-instrumented classes + FromBytesClassLoader freshLoader = new FromBytesClassLoader(current); + instrumentAllInDirectory(instr, new File(dirCfg.getWorkingDir()), freshLoader, ""); + + // 3. Swap in the fresh classloader and run just this one test + Thread.currentThread().setContextClassLoader(freshLoader); + runSingleTest(uniqueId); - // 3. Collect execution data + // 4. Collect coverage ExecutionDataStore executionData = new ExecutionDataStore(); SessionInfoStore sessionInfos = new SessionInfoStore(); data.collect(executionData, sessionInfos, false); runtime.shutdown(); - // 4. Analyze coverage + // 5. Analyze CoverageBuilder coverageBuilder = new CoverageBuilder(); Analyzer analyzer = new Analyzer(executionData, coverageBuilder); - for (String classUnderTest : runCfg.classesUnderTest()) { - try (InputStream in = - getClassAsInputStream(dirCfg.getWorkingDir(), classUnderTest)) { + try (InputStream in = getClassAsInputStream(dirCfg.getWorkingDir(), classUnderTest)) { analyzer.analyzeClass(in, classUnderTest); } } - // 5. Extract covered lines Set coveredLines = extractCoveredLines(coverageBuilder, testClass); - coveragePerTest.put(testName, coveredLines); + coveragePerTest.put(displayName, coveredLines); } + + // Restore original instrumented classloader for the rest of the pipeline + Thread.currentThread().setContextClassLoader(ctx.getClassloaderWithStudentsCode()); return coveragePerTest; } @@ -160,7 +167,7 @@ private Set extractCoveredLines( Set result = new HashSet<>(); for (IClassCoverage cc : coverageBuilder.getClasses()) { - String className = cc.getName().replace('/', '.'); + String className = cc.getName().replace('/', '.').concat("Tests"); if (!className.equals(testClass)) { continue; diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java index 39bca95f9..9c45008b1 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java @@ -5,6 +5,7 @@ import nl.tudelft.cse1110.andy.result.ResultBuilder; import nl.tudelft.cse1110.andy.utils.ClassUtils; import nl.tudelft.cse1110.andy.utils.FilesUtils; +import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.engine.discovery.DiscoverySelectors; import org.junit.platform.engine.reporting.ReportEntry; import org.junit.platform.launcher.*; @@ -18,9 +19,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; +import java.util.*; import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; @@ -60,41 +59,38 @@ public void execute(Context ctx, ResultBuilder result) { .configurationParameter("jqwik.shrinking.default", "BOUNDED") .configurationParameter("jqwik.database", FilesUtils.createTemporaryDirectory("jqwik").resolve("jqwik-db").toString()) .build(); - launcher.execute(request); - - TestExecutionSummary summary = listener.getSummary(); - /* Restore the sysout back, and put it in the result in case there's something */ - System.setOut(console); - - /* Log the junit result */ - result.logJUnitRun(summary, output.toString()); + TestPlan plan = launcher.discover(request); /* - Log the unit tests available for the quality metrics + Find the unit tests available for the quality metrics */ - request = - LauncherDiscoveryRequestBuilder.request() - .selectors(DiscoverySelectors.selectPackage("org.example")) - .build(); - launcher = LauncherFactory.create(); - TestPlan plan = launcher.discover(request); + Map unitTests = new HashMap<>(); // uniqueId -> displayName + + launcher.registerTestExecutionListeners(new TestExecutionListener() { + @Override + public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { + if (testIdentifier.isTest()) { + String displayName = plan.getParent(testIdentifier) + .map(parent -> parent.getDisplayName() + " " + testIdentifier.getDisplayName()) + .orElse(testIdentifier.getDisplayName()); + unitTests.put(testIdentifier.getUniqueId(), displayName); + } + } + }); - List unitTests = new ArrayList<>(); + launcher.execute(request); - int s = plan.getRoots().size(); - int d = plan.getDescendants(plan.getRoots().iterator().next()).size(); + TestExecutionSummary summary = listener.getSummary(); -// plan.getRoots().forEach(root -> -// plan.getDescendants(root).stream() -// .filter(TestIdentifier::isTest) -// .forEach(ti -> unitTests.add(ti.getDisplayName()))); + /* Restore the sysout back, and put it in the result in case there's something */ + System.setOut(console); - plan.getRoots().forEach(root -> { - unitTests.add(root.getDisplayName()); - }); + /* Log the junit result */ + result.logJUnitRun(summary, output.toString()); + /* Log the unit tests */ result.logUnitTests(unitTests); } catch (Exception e) { result.genericFailure(this, e); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPitestStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPitestStep.java index fa6fe64fa..cca9c9fbf 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPitestStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPitestStep.java @@ -22,19 +22,20 @@ import org.pitest.mutationtest.tooling.EntryPoint; import org.pitest.util.Unchecked; import org.w3c.dom.Document; +import org.w3c.dom.Element; import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.PrintStream; +import java.io.*; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.stream.Stream; public class RunPitestStep implements ExecutionStep { @@ -64,32 +65,38 @@ public void execute(Context ctx, ResultBuilder result) { // run! final CombinedStatistics stats = runReport(ctx, data, plugins); + String mutationsXml = null; + Path mutationsFile = Paths.get(outputPitestDir, "mutations.xml"); + if (Files.exists(mutationsFile)) { + try { + mutationsXml = Files.readString(mutationsFile); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + System.setOut(console); result.logPitest(stats); - } - /* - Log the mutations killed by each test. - */ - List tests = result.getQualityResult().getUnitTests(); + /* + Log the mutations killed by each test. + */ + Map tests = result.getQualityResult().getUnitTests(); - Map> mutationsKilledPerTest = mutationsKilledPerTest(ctx, tests); + Map> mutationsKilledPerTest = mutationsKilledPerTest(ctx, tests, mutationsXml); - result.logMutationsKilledPerTest(mutationsKilledPerTest); + result.logMutationsKilledPerTest(mutationsKilledPerTest); + } } - private Map> mutationsKilledPerTest( - Context ctx, - List tests) { + private Map> mutationsKilledPerTest(Context ctx, Map tests, String mutationsXml) { Map> result = new HashMap<>(); - for (String test : tests) result.put(test, new HashSet<>()); + for (String test : tests.keySet()) result.put(test, new HashSet<>()); Path mutationsFile = Paths.get( - ctx.getDirectoryConfiguration().getWorkingDir(), - "pitest", - "mutations.xml" + ctx.getDirectoryConfiguration().getOutputDir(), "pitest", "mutations.xml" ); if (!Files.exists(mutationsFile)) return result; @@ -97,22 +104,20 @@ private Map> mutationsKilledPerTest( try { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder builder = factory.newDocumentBuilder(); - Document doc = builder.parse(mutationsFile.toFile()); + if (mutationsXml == null) return result; + Document doc = builder.parse(new InputSource(new StringReader(mutationsXml))); NodeList mutationNodes = doc.getElementsByTagName("mutation"); for (int i = 0; i < mutationNodes.getLength(); i++) { - ElementImpl mutation = (ElementImpl) mutationNodes.item(i); + Element mutation = (Element) mutationNodes.item(i); String status = mutation.getAttribute("status"); - if (!"KILLED".equals(status)) { - continue; - } + + if (!"KILLED".equals(status)) continue; int mutationId = Integer.parseInt( - mutation.getElementsByTagName("index") - .item(0) - .getTextContent() + mutation.getElementsByTagName("index").item(0).getTextContent() ); NodeList killingTests = mutation.getElementsByTagName("killingTest"); @@ -123,7 +128,11 @@ private Map> mutationsKilledPerTest( String testName = normalizeTestName(rawTestName); - result.computeIfAbsent(testName, t -> new HashSet<>()).add(mutationId); + for (String key : result.keySet()) { + if (key.contains(testName)) { + result.get(key).add(mutationId); + } + } } } catch (Exception e) { throw new RuntimeException("Failed to parse mutations.xml", e); @@ -133,11 +142,12 @@ private Map> mutationsKilledPerTest( } private String normalizeTestName(String pitTestName) { - int paren = pitTestName.indexOf('('); - if (paren != -1) { - pitTestName = pitTestName.substring(0, paren); + // PIT prepends "ClassName." before the JUnit unique ID - strip it + int bracketIndex = pitTestName.indexOf('['); + if (bracketIndex != -1) { + return pitTestName.substring(bracketIndex); } - return pitTestName.replace('.', '#'); + return pitTestName; } @@ -166,7 +176,6 @@ private String[] buildArgs(Context ctx, String pitestOutputDir) { args.add(dirCfg.getWorkingDir()); args.add("--verbose"); - args.add("true"); // to allow detecting which mutations are covered by each test args.add("--classPath"); List librariesToInclude = compiledClassesPlusLibraries(ctx, dirCfg); @@ -175,7 +184,8 @@ private String[] buildArgs(Context ctx, String pitestOutputDir) { args.add("--mutators"); args.add(commaSeparated(runCfg.listOfMutants())); - args.add("--exportLineCoverage"); + args.add("--outputFormats"); + args.add("XML"); return args.stream().toArray(String[]::new); } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index 7ddf4e524..488ec8a08 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -7,13 +7,13 @@ public class QualityResult { private int score; // between 0 and 1 private int numUnitTests; - private List unitTests; + private Map unitTests; // uniqueId (testId below) -> displayName private LinkedList metaTestReports; private Map> testToMetaTests; // a useful mapping from the test cases to the meta-tests they cover - private Map> coveragePerTest; - private Map> mutationsKilledPerTest; + private Map> coveragePerTest; // testId -> linesCovered + private Map> mutationsKilledPerTest; // testId -> mutationId public QualityResult(int numUnitTests) { // dummy: @@ -51,11 +51,11 @@ public void setNumUnitTests(int numUnitTests) { this.numUnitTests = numUnitTests; } - public List getUnitTests() { + public Map getUnitTests() { return unitTests; } - public void setUnitTests(List unitTests) { + public void setUnitTests(Map unitTests) { this.unitTests = unitTests; } @@ -143,38 +143,26 @@ public long countIsolatedTests() { * 3) mutations killed * @return the number of such tests */ - public long countMeaningfulTests() { + public long countContributingTests() { - Set meaningfulTests = new HashSet<>(); + Set contributingTests = new HashSet<>(); - meaningfulTests.addAll(meaningfulCoverage(testToMetaTests)); // 1) + contributingTests.addAll(contribution(testToMetaTests)); // 1) - meaningfulTests.addAll(meaningfulCoverage(coveragePerTest)); // 2) + contributingTests.addAll(contribution(coveragePerTest)); // 2) - meaningfulTests.addAll(meaningfulCoverage(mutationsKilledPerTest)); // 3) + contributingTests.addAll(contribution(mutationsKilledPerTest)); // 3) - return meaningfulTests.size(); + return contributingTests.size(); } - private Set meaningfulCoverage(Map> map) { - - Set meaningfulTests = new HashSet<>(); - + private Set contribution(Map> map) { + Set contributingTests = new HashSet<>(); for (String test : map.keySet()) { - - Set meaningfulContributions = new HashSet<>(map.get(test)); - - for (String otherTest : map.keySet()) { - if (test.equals(otherTest)) continue; - Set contributionsByOtherTest = new HashSet<>(map.get(otherTest)); - meaningfulContributions.removeAll(contributionsByOtherTest); - } - - if (!meaningfulContributions.isEmpty()) { - meaningfulTests.add(test); + if (!map.get(test).isEmpty()) { + contributingTests.add(test); } } - - return meaningfulTests; + return contributingTests; } } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index 1ce2529ef..ff2788536 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -260,20 +260,52 @@ public QualityResult getQualityResult() { return qualityResult; } - public void logUnitTests(List unitTests) { + public void logUnitTests(Map unitTests) { this.qualityResult.setUnitTests(unitTests); } public void logCoveragePerTest(Map> coveragePerTest) { - this.qualityResult.setCoveragePerTest(coveragePerTest); + Map> coveragePerTestNormalized = new HashMap<>(); + for (String testName : coveragePerTest.keySet()) { + if (testName.contains("[test-template:")) { + + Set lines = coveragePerTest.get(testName); + + String method = testName.replaceAll("\\(.*", "").trim(); + String invocation = testName.replaceAll(".*\\[(\\d+)\\].*", "#$1").trim(); + String normalizedName = method + " " + invocation; + + coveragePerTestNormalized.put(normalizedName, lines); + } + } + this.qualityResult.setCoveragePerTest(coveragePerTestNormalized); } public void logMutationsKilledPerTest(Map> mutationsKilledPerTest) { - this.qualityResult.setMutationsKilledPerTest(mutationsKilledPerTest); + Map> mutationsKilledPerTestNormalized = new HashMap<>(); + for (String testName : mutationsKilledPerTest.keySet()) { + if (testName.contains("[test-template:")) { + + Set mutations = mutationsKilledPerTest.get(testName); + + String method = testName.replaceAll(".*\\[test-template:([^(]+).*", "$1").trim(); + String invocation = testName.replaceAll(".*\\[test-template-invocation:#(\\d+)\\].*", "#$1").trim(); + String normalizedName = method + " " + invocation; + + mutationsKilledPerTestNormalized.put(normalizedName, mutations); + } + } + this.qualityResult.setMutationsKilledPerTest(mutationsKilledPerTestNormalized); } public void logMetaTest(String name, MetaTestReport metaTestReport) { - metaTestReport.setName(name); + String normalizedName = name; + if (name.matches(".* \\(\\d+\\)")) { + String method = name.replaceAll(" \\(\\d+\\)$", "").trim(); + String invocation = name.replaceAll(".* \\((\\d+)\\)$", "#$1").trim(); + normalizedName = method + " " + invocation; + } + metaTestReport.setName(normalizedName); this.qualityResult.considerMetaTest(metaTestReport); } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java b/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java index 584952a1c..d85d99db6 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java @@ -1,7 +1,6 @@ package nl.tudelft.cse1110.andy.writer.standard; import nl.tudelft.cse1110.andy.execution.Context.Context; -import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; import nl.tudelft.cse1110.andy.execution.mode.Action; import nl.tudelft.cse1110.andy.execution.mode.Mode; import nl.tudelft.cse1110.andy.execution.mode.ModeActionSelector; @@ -293,7 +292,7 @@ private void printQualityResults(Context ctx, QualityResult qualityResult) { long allTests = qualityResult.countTests(); l(String.format("Cohesive tests: %d/%d", qualityResult.countCohesiveTests(), allTests)); l(String.format("Independent tests: %d/%d", qualityResult.countIsolatedTests(), allTests)); - l(String.format("Meaningful tests: %d/%d", qualityResult.countMeaningfulTests(), allTests)); + l(String.format("Contributing tests: %d/%d", qualityResult.countContributingTests(), allTests)); } } diff --git a/andy/src/test/java/integration/quality/CohesionTest.java b/andy/src/test/java/integration/quality/CohesionTest.java index 69530d290..6458466d2 100644 --- a/andy/src/test/java/integration/quality/CohesionTest.java +++ b/andy/src/test/java/integration/quality/CohesionTest.java @@ -1,101 +1,101 @@ -package integration.quality; - -import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; -import nl.tudelft.cse1110.andy.result.QualityResult; -import nl.tudelft.cse1110.andy.result.TestFailureInfo; -import org.junit.jupiter.api.Test; -import org.junit.platform.launcher.TestIdentifier; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class CohesionTest { - - @Test - public void testTriggersOneMetaTestTest() { - QualityResult qualityResult = new QualityResult(1); - qualityResult.setUnitTests(List.of("test")); - - TestFailureInfo failure = new TestFailureInfo("test", "error message"); - MetaTestReport metaTestReport = new MetaTestReport(1, 0, 1, List.of(failure)); - qualityResult.considerMetaTest(metaTestReport); - - assertEquals(1, qualityResult.countCohesiveTests()); - } - - @Test - public void testTriggersNoMetaTestTest() { - QualityResult qualityResult = new QualityResult(1); - qualityResult.setUnitTests(List.of("test")); - - MetaTestReport metaTestReport = new MetaTestReport(1, 1, 1, List.of()); - qualityResult.considerMetaTest(metaTestReport); - - assertEquals(0, qualityResult.countCohesiveTests()); - } - - @Test - public void sameTestTriggersTwoMetaTestsTest() { - QualityResult qualityResult = new QualityResult(1); - qualityResult.setUnitTests(List.of("test")); - - TestFailureInfo failure1 = new TestFailureInfo("test", "error message"); - TestFailureInfo failure2 = new TestFailureInfo("test", "some other error message"); - MetaTestReport metaTestReport1 = new MetaTestReport(1, 1, 1, List.of(failure1)); - MetaTestReport metaTestReport2 = new MetaTestReport(1, 1, 1, List.of(failure2)); - qualityResult.considerMetaTest(metaTestReport1); - qualityResult.considerMetaTest(metaTestReport2); - - assertEquals(0, qualityResult.countCohesiveTests()); - } - - @Test - public void differentTestsTriggerDifferentMetaTestsTest() { - QualityResult qualityResult = new QualityResult(2); - qualityResult.setUnitTests(List.of("test1", "test2")); - - TestFailureInfo failure1 = new TestFailureInfo("test1", "error message"); - TestFailureInfo failure2 = new TestFailureInfo("test2", "some other error message"); - MetaTestReport metaTestReport1 = new MetaTestReport(2, 1, 2, List.of(failure1)); - MetaTestReport metaTestReport2 = new MetaTestReport(2, 1, 2, List.of(failure2)); - qualityResult.considerMetaTest(metaTestReport1); - qualityResult.considerMetaTest(metaTestReport2); - - assertEquals(2, qualityResult.countCohesiveTests()); - } - - @Test - public void differentTestsTriggerSameMetaTestsTest() { - QualityResult qualityResult = new QualityResult(2); - qualityResult.setUnitTests(List.of("test1", "test2")); - - TestFailureInfo failure1 = new TestFailureInfo("test1", "error message"); - TestFailureInfo failure2 = new TestFailureInfo("test2", "some other error message"); - MetaTestReport metaTestReport = new MetaTestReport(2, 0, 2, List.of(failure1, failure2)); - qualityResult.considerMetaTest(metaTestReport); - - assertEquals(2, qualityResult.countCohesiveTests()); - } - - @Test - public void noTestsTest() { - QualityResult qualityResult = new QualityResult(0); - qualityResult.setUnitTests(List.of()); - - MetaTestReport metaTestReport = new MetaTestReport(0, 0, 0, List.of()); - qualityResult.considerMetaTest(metaTestReport); - - assertEquals(0, qualityResult.countCohesiveTests()); - } - - @Test - public void noMetaTestsTest() { - QualityResult qualityResult = new QualityResult(1); - qualityResult.setUnitTests(List.of("test")); - - assertEquals(0, qualityResult.countCohesiveTests()); - } -} +//package integration.quality; +// +//import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; +//import nl.tudelft.cse1110.andy.result.QualityResult; +//import nl.tudelft.cse1110.andy.result.TestFailureInfo; +//import org.junit.jupiter.api.Test; +//import org.junit.platform.launcher.TestIdentifier; +// +//import java.util.HashMap; +//import java.util.List; +//import java.util.Map; +// +//import static org.junit.jupiter.api.Assertions.assertEquals; +// +//public class CohesionTest { +// +// @Test +// public void testTriggersOneMetaTestTest() { +// QualityResult qualityResult = new QualityResult(1); +// qualityResult.setUnitTests(List.of("test")); +// +// TestFailureInfo failure = new TestFailureInfo("test", "error message"); +// MetaTestReport metaTestReport = new MetaTestReport(1, 0, 1, List.of(failure)); +// qualityResult.considerMetaTest(metaTestReport); +// +// assertEquals(1, qualityResult.countCohesiveTests()); +// } +// +// @Test +// public void testTriggersNoMetaTestTest() { +// QualityResult qualityResult = new QualityResult(1); +// qualityResult.setUnitTests(List.of("test")); +// +// MetaTestReport metaTestReport = new MetaTestReport(1, 1, 1, List.of()); +// qualityResult.considerMetaTest(metaTestReport); +// +// assertEquals(0, qualityResult.countCohesiveTests()); +// } +// +// @Test +// public void sameTestTriggersTwoMetaTestsTest() { +// QualityResult qualityResult = new QualityResult(1); +// qualityResult.setUnitTests(List.of("test")); +// +// TestFailureInfo failure1 = new TestFailureInfo("test", "error message"); +// TestFailureInfo failure2 = new TestFailureInfo("test", "some other error message"); +// MetaTestReport metaTestReport1 = new MetaTestReport(1, 1, 1, List.of(failure1)); +// MetaTestReport metaTestReport2 = new MetaTestReport(1, 1, 1, List.of(failure2)); +// qualityResult.considerMetaTest(metaTestReport1); +// qualityResult.considerMetaTest(metaTestReport2); +// +// assertEquals(0, qualityResult.countCohesiveTests()); +// } +// +// @Test +// public void differentTestsTriggerDifferentMetaTestsTest() { +// QualityResult qualityResult = new QualityResult(2); +// qualityResult.setUnitTests(List.of("test1", "test2")); +// +// TestFailureInfo failure1 = new TestFailureInfo("test1", "error message"); +// TestFailureInfo failure2 = new TestFailureInfo("test2", "some other error message"); +// MetaTestReport metaTestReport1 = new MetaTestReport(2, 1, 2, List.of(failure1)); +// MetaTestReport metaTestReport2 = new MetaTestReport(2, 1, 2, List.of(failure2)); +// qualityResult.considerMetaTest(metaTestReport1); +// qualityResult.considerMetaTest(metaTestReport2); +// +// assertEquals(2, qualityResult.countCohesiveTests()); +// } +// +// @Test +// public void differentTestsTriggerSameMetaTestsTest() { +// QualityResult qualityResult = new QualityResult(2); +// qualityResult.setUnitTests(List.of("test1", "test2")); +// +// TestFailureInfo failure1 = new TestFailureInfo("test1", "error message"); +// TestFailureInfo failure2 = new TestFailureInfo("test2", "some other error message"); +// MetaTestReport metaTestReport = new MetaTestReport(2, 0, 2, List.of(failure1, failure2)); +// qualityResult.considerMetaTest(metaTestReport); +// +// assertEquals(2, qualityResult.countCohesiveTests()); +// } +// +// @Test +// public void noTestsTest() { +// QualityResult qualityResult = new QualityResult(0); +// qualityResult.setUnitTests(List.of()); +// +// MetaTestReport metaTestReport = new MetaTestReport(0, 0, 0, List.of()); +// qualityResult.considerMetaTest(metaTestReport); +// +// assertEquals(0, qualityResult.countCohesiveTests()); +// } +// +// @Test +// public void noMetaTestsTest() { +// QualityResult qualityResult = new QualityResult(1); +// qualityResult.setUnitTests(List.of("test")); +// +// assertEquals(0, qualityResult.countCohesiveTests()); +// } +//} diff --git a/andy/src/test/resources/grader/fixtures/Solution/NumberUtilsAddOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/NumberUtilsAddOfficialSolution.java index 5e47a1673..20485b136 100644 --- a/andy/src/test/resources/grader/fixtures/Solution/NumberUtilsAddOfficialSolution.java +++ b/andy/src/test/resources/grader/fixtures/Solution/NumberUtilsAddOfficialSolution.java @@ -138,7 +138,8 @@ void shouldThrowExceptionWhenDigitsAreOutOfRange(List left, List digitsOutOfRange() { return Stream.of(Arguments.of(numbers(1, -1, 1), numbers(1, 1, 1)), - Arguments.of(numbers(1, 1, 1), numbers(1, -1, 1)), Arguments.of(numbers(1, 11, 1), numbers(1, 1, 1)), + Arguments.of(numbers(1, 1, 1), numbers(1, -1, 1)), + Arguments.of(numbers(1, 11, 1), numbers(1, 1, 1)), Arguments.of(numbers(1, 1, 1), numbers(1, 11, 1))); } diff --git a/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java b/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java index c920d0a4a..f770a1b9b 100644 --- a/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java +++ b/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java @@ -101,7 +101,7 @@ private static void appendMetaScoreElements(Result result, Document doc, Element appendMetaScore(doc, metaElement, "Quality score", result.getQualityResult().computeScore()); appendMetaScore(doc, metaElement, "Cohesive tests", (int) result.getQualityResult().countCohesiveTests()); appendMetaScore(doc, metaElement, "Independent tests", (int) result.getQualityResult().countIsolatedTests()); - appendMetaScore(doc, metaElement, "Meaningful tests", (int) result.getQualityResult().countMeaningfulTests()); + appendMetaScore(doc, metaElement, "Contributing tests", (int) result.getQualityResult().countContributingTests()); // Existing code checks and meta tests result.getCodeChecks().getCheckResults().forEach(check -> appendMetaScore(doc, metaElement, check.getDescription(), check.passed() ? 1 : 0)); From 72f5c365c5d7b587aa4016a54924ef2de2e43dd1 Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Sat, 14 Mar 2026 17:41:31 +0100 Subject: [PATCH 17/23] Cleaning up code --- .../andy/execution/metatest/library/LibraryMetaTest.java | 1 - .../andy/execution/step/CollectCoverageInformationStep.java | 2 -- .../andy/execution/step/InstrumentCodeForCoverageStep.java | 2 -- .../cse1110/andy/execution/step/RunJUnitTestsStep.java | 2 +- .../tudelft/cse1110/andy/execution/step/RunPitestStep.java | 6 ------ 5 files changed, 1 insertion(+), 12 deletions(-) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java index c6632c241..e7cc9c964 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java @@ -13,7 +13,6 @@ import nl.tudelft.cse1110.andy.result.CompilationErrorInfo; import nl.tudelft.cse1110.andy.result.CompilationResult; import nl.tudelft.cse1110.andy.result.Result; -import nl.tudelft.cse1110.andy.result.TestFailureInfo; import nl.tudelft.cse1110.andy.utils.CodeSnippetUtils; import nl.tudelft.cse1110.andy.utils.FilesUtils; diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java index 6096fccc0..7ff1195b7 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java @@ -9,7 +9,6 @@ import nl.tudelft.cse1110.andy.utils.FilesUtils; import nl.tudelft.cse1110.andy.utils.FromBytesClassLoader; import org.jacoco.core.analysis.*; -import org.jacoco.core.data.ExecutionData; import org.jacoco.core.data.ExecutionDataStore; import org.jacoco.core.data.SessionInfoStore; import org.jacoco.core.instr.Instrumenter; @@ -23,7 +22,6 @@ import org.junit.platform.engine.discovery.DiscoverySelectors; import org.junit.platform.launcher.Launcher; import org.junit.platform.launcher.LauncherDiscoveryRequest; -import org.junit.platform.launcher.TestIdentifier; import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; import org.junit.platform.launcher.core.LauncherFactory; diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/InstrumentCodeForCoverageStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/InstrumentCodeForCoverageStep.java index b986e40b9..2df31033b 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/InstrumentCodeForCoverageStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/InstrumentCodeForCoverageStep.java @@ -15,8 +15,6 @@ import java.io.IOException; import java.io.InputStream; -import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; - public class InstrumentCodeForCoverageStep implements ExecutionStep { @Override diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java index 9c45008b1..2a1376fd3 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunJUnitTestsStep.java @@ -6,7 +6,6 @@ import nl.tudelft.cse1110.andy.utils.ClassUtils; import nl.tudelft.cse1110.andy.utils.FilesUtils; import org.junit.platform.engine.TestExecutionResult; -import org.junit.platform.engine.discovery.DiscoverySelectors; import org.junit.platform.engine.reporting.ReportEntry; import org.junit.platform.launcher.*; import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; @@ -25,6 +24,7 @@ public class RunJUnitTestsStep implements ExecutionStep { + @SuppressWarnings("checkstyle:MethodLength") @Override public void execute(Context ctx, ResultBuilder result) { try { diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPitestStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPitestStep.java index cca9c9fbf..64a9d291a 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPitestStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPitestStep.java @@ -7,16 +7,11 @@ import nl.tudelft.cse1110.andy.result.ResultBuilder; import nl.tudelft.cse1110.andy.utils.ClassUtils; import nl.tudelft.cse1110.andy.utils.FilesUtils; -import org.htmlunit.cyberneko.xerces.dom.ElementImpl; -import org.pitest.mutationtest.DetectionStatus; -import org.pitest.mutationtest.MutationResult; -import org.pitest.mutationtest.MutationStatusTestPair; import org.pitest.mutationtest.commandline.OptionsParser; import org.pitest.mutationtest.commandline.ParseResult; import org.pitest.mutationtest.commandline.PluginFilter; import org.pitest.mutationtest.config.PluginServices; import org.pitest.mutationtest.config.ReportOptions; -import org.pitest.mutationtest.engine.MutationIdentifier; import org.pitest.mutationtest.tooling.AnalysisResult; import org.pitest.mutationtest.tooling.CombinedStatistics; import org.pitest.mutationtest.tooling.EntryPoint; @@ -35,7 +30,6 @@ import java.util.*; import java.util.logging.Logger; import java.util.stream.Collectors; -import java.util.stream.Stream; public class RunPitestStep implements ExecutionStep { From 007c527479e25d168ecb25d9b303d6ea9f3dce7b Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Sat, 14 Mar 2026 19:27:19 +0100 Subject: [PATCH 18/23] Output now includes description of test performance in quality metrics --- .../cse1110/andy/result/QualityResult.java | 147 ++++++++++++++++-- .../cse1110/andy/result/ResultBuilder.java | 30 ++-- .../writer/standard/StandardResultWriter.java | 5 +- 3 files changed, 150 insertions(+), 32 deletions(-) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index 488ec8a08..d07b10980 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -3,6 +3,7 @@ import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; import java.util.*; +import java.util.stream.Collectors; public class QualityResult { private int score; // between 0 and 1 @@ -90,7 +91,15 @@ public void considerMetaTest(MetaTestReport metaTestReport) { if (testToMetaTests.get(test) == null) { testToMetaTests.put(test, new HashSet<>()); } - testToMetaTests.get(test).add(metaTestReport.getName()); + + String testName = metaTestReport.getName(); + + if (testName.matches(".* \\(\\d+\\)")) { + String method = testName.replaceAll(" \\(\\d+\\)$", "").trim(); + String invocation = testName.replaceAll(".* \\((\\d+)\\)$", "#$1").trim(); + testName = method + " " + invocation; + } + testToMetaTests.get(test).add(testName); } } @@ -112,30 +121,45 @@ public long countCohesiveTests() { return testToMetaTests.values().stream().filter(nt -> nt.size() == 1).count(); } + @SuppressWarnings("checkstyle:DeclarationOrder") + private Map> nonisolatedTests = new HashMap<>(); + /** * Count the number of tests that do not trigger meta-tests already covered by other tests * @return the number of such tests */ public long countIsolatedTests() { - long count = countTests(); - - Set unisolatedTests = new HashSet<>(); - for (MetaTestReport metaTestReport : metaTestReports) { if (metaTestReport.getTestsTriggered().size() == 1) continue; - for (TestFailureInfo testFailureInfo : metaTestReport.getTestsTriggered()) { - String testName = testFailureInfo.getTestCase(); - if (!unisolatedTests.contains(testName)) { - unisolatedTests.add(testName); - count--; - } + + // All tests that trigger this meta-test collide with each other + List collidingTests = metaTestReport.getTestsTriggered().stream() + .map(TestFailureInfo::getTestCase) + .toList(); + + for (String test : collidingTests) { + nonisolatedTests.computeIfAbsent(test, t -> { + return new HashSet<>(); + }) + .addAll(collidingTests.stream() + .filter(other -> !other.equals(test)) + .collect(Collectors.toSet())); } } + int count = unitTests.size(); + + for (Set collisions : nonisolatedTests.values()) { + if (!collisions.isEmpty()) count--; + } + return count; } + @SuppressWarnings("checkstyle:DeclarationOrder") + Map> contributingTests = new HashMap<>(); + /** * Count the number of tests that increase one of: * 1) number of meta tests triggered @@ -145,13 +169,26 @@ public long countIsolatedTests() { */ public long countContributingTests() { - Set contributingTests = new HashSet<>(); - - contributingTests.addAll(contribution(testToMetaTests)); // 1) + // 1) + Set contributingMetaTests = contribution(testToMetaTests); + for (String test : contributingMetaTests) { + contributingTests.computeIfAbsent(test, t -> new ArrayList<>()); + contributingTests.get(test).add(1); + } - contributingTests.addAll(contribution(coveragePerTest)); // 2) + // 2) + Set contributingCoverage = contribution(coveragePerTest); + for (String test : contributingCoverage) { + contributingTests.computeIfAbsent(test, t -> new ArrayList<>()); + contributingTests.get(test).add(2); + } - contributingTests.addAll(contribution(mutationsKilledPerTest)); // 3) + // 3) + Set contributingMutation = contribution(mutationsKilledPerTest); + for (String test : contributingMutation) { + contributingTests.computeIfAbsent(test, t -> new ArrayList<>()); + contributingTests.get(test).add(3); + } return contributingTests.size(); } @@ -165,4 +202,82 @@ private Set contribution(Map> map) { } return contributingTests; } + + /** + * Used to get an overview of cohesive tests in the output + * @return a list of cohesive and non-cohesive tests + */ + public String listCohesiveTests() { + StringBuilder sb = new StringBuilder("Tests that only cover a single meta-test: \n"); + + for (String testName : testToMetaTests.keySet()) { + if (testToMetaTests.get(testName) == null || + testToMetaTests.get(testName).size() != 1) { + sb.append(" > " + testName + " ✕\n"); + } else { + sb.append(" > " + testName + " ✓\n"); + } + } + + return sb.toString(); + } + + /** + * Used to get an overview of isolated tests in the output + * @return a list of isolated and non-isolated tests + */ + public String listIsolatedTests() { + + StringBuilder sb = new StringBuilder("Tests that do not trigger meta-tests already covered by other tests: \n"); + + for (String testName : unitTests.values()) { + if (nonisolatedTests.containsKey(testName)) { + sb.append(" > " + testName + " ✕ - "); + Set collisions = nonisolatedTests.get(testName); + for (String collision : collisions) { + sb.append(collision + "; "); + } + sb.append("\n"); + } else { + sb.append(" > " + testName + " ✓\n"); + } + } + + return sb.toString(); + } + + /** + * Used to get an overview of contributing tests in the output + * @return a list of contributing and non-contributing tests + */ + public String listContributingTests() { + + StringBuilder sb = new StringBuilder("Tests that increase a metric: \n"); + + for (String testName : unitTests.values()) { + if (contributingTests.containsKey(testName)) { + sb.append(" > " + testName + " ✓ - "); + List contributions = contributingTests.get(testName); + for (int contribution : contributions) { + switch (contribution) { + case 1: + sb.append("meta-tests; "); + break; + case 2: + sb.append("coverage; "); + break; + case 3: + sb.append("mutation; "); + break; + default: + } + } + sb.append("\n"); + } else { + sb.append(" > " + testName + " ✕\n"); + } + } + + return sb.toString(); + } } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index ff2788536..9c7b8b636 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -261,24 +261,24 @@ public QualityResult getQualityResult() { } public void logUnitTests(Map unitTests) { - this.qualityResult.setUnitTests(unitTests); - } - - public void logCoveragePerTest(Map> coveragePerTest) { - Map> coveragePerTestNormalized = new HashMap<>(); - for (String testName : coveragePerTest.keySet()) { - if (testName.contains("[test-template:")) { - - Set lines = coveragePerTest.get(testName); + Map unitTestsNormalized = new HashMap<>(); + for (String uniqueId : unitTests.keySet()) { - String method = testName.replaceAll("\\(.*", "").trim(); - String invocation = testName.replaceAll(".*\\[(\\d+)\\].*", "#$1").trim(); - String normalizedName = method + " " + invocation; + String displayName = unitTests.get(uniqueId); - coveragePerTestNormalized.put(normalizedName, lines); + if (displayName.matches(".*\\(.*\\).*\\[\\d+\\].*")) { + String method = displayName.replaceAll("\\(.*", "").trim(); + String invocation = displayName.replaceAll(".*?\\[(\\d+)\\].*", "($1)").trim(); + displayName = method + " " + invocation; } + + unitTestsNormalized.put(uniqueId, displayName); } - this.qualityResult.setCoveragePerTest(coveragePerTestNormalized); + this.qualityResult.setUnitTests(unitTestsNormalized); + } + + public void logCoveragePerTest(Map> coveragePerTest) { + this.qualityResult.setCoveragePerTest(coveragePerTest); } public void logMutationsKilledPerTest(Map> mutationsKilledPerTest) { @@ -289,7 +289,7 @@ public void logMutationsKilledPerTest(Map> mutationsKilledP Set mutations = mutationsKilledPerTest.get(testName); String method = testName.replaceAll(".*\\[test-template:([^(]+).*", "$1").trim(); - String invocation = testName.replaceAll(".*\\[test-template-invocation:#(\\d+)\\].*", "#$1").trim(); + String invocation = testName.replaceAll(".*\\[test-template-invocation:#(\\d+)\\].*", "($1)").trim(); String normalizedName = method + " " + invocation; mutationsKilledPerTestNormalized.put(normalizedName, mutations); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java b/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java index d85d99db6..e3035b680 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java @@ -286,13 +286,16 @@ private void printQualityResults(Context ctx, QualityResult qualityResult) { return; l("\n--- Quality Results"); - l(String.format("Score: %d", qualityResult.computeScore())); + l(String.format("Score: %d\n", qualityResult.computeScore())); if (allHints) { long allTests = qualityResult.countTests(); l(String.format("Cohesive tests: %d/%d", qualityResult.countCohesiveTests(), allTests)); + l(qualityResult.listCohesiveTests()); l(String.format("Independent tests: %d/%d", qualityResult.countIsolatedTests(), allTests)); + l(qualityResult.listIsolatedTests()); l(String.format("Contributing tests: %d/%d", qualityResult.countContributingTests(), allTests)); + l(qualityResult.listContributingTests()); } } From d8fbe77e754469601e9ab489c20b42bd816637ae Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Sun, 15 Mar 2026 16:31:21 +0100 Subject: [PATCH 19/23] Bug fixes --- .../step/CollectCoverageInformationStep.java | 17 ++-- .../cse1110/andy/result/QualityResult.java | 14 +-- .../cse1110/andy/result/ResultBuilder.java | 42 ++++---- .../quality/OverviewQualityResults.java | 3 +- .../Config/ContainsAnyConfiguration.java | 99 +++++++++++++++++++ .../fixtures/Library/ContainsAnyLibrary.java | 40 ++++++++ .../Solution/ContainsAnyOfficialSolution.java | 58 +++++++++++ 7 files changed, 235 insertions(+), 38 deletions(-) create mode 100644 andy/src/test/resources/grader/fixtures/Config/ContainsAnyConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/ContainsAnyLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/ContainsAnyOfficialSolution.java diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java index 7ff1195b7..d8e843b81 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/CollectCoverageInformationStep.java @@ -106,7 +106,6 @@ private Map> linesCoveredPerTest(Context ctx, String testCl for (Map.Entry entry : tests.entrySet()) { String uniqueId = entry.getKey(); - String displayName = entry.getValue(); // 1. Fresh runtime + data for this test IRuntime runtime = new LoggerRuntime(); @@ -138,9 +137,9 @@ private Map> linesCoveredPerTest(Context ctx, String testCl } } - Set coveredLines = extractCoveredLines(coverageBuilder, testClass); + Set coveredLines = extractCoveredLines(coverageBuilder); - coveragePerTest.put(displayName, coveredLines); + coveragePerTest.put(uniqueId, coveredLines); } // Restore original instrumented classloader for the rest of the pipeline @@ -159,17 +158,15 @@ private void runSingleTest(String testId) { } private Set extractCoveredLines( - CoverageBuilder coverageBuilder, - String testClass) { + CoverageBuilder coverageBuilder) { Set result = new HashSet<>(); for (IClassCoverage cc : coverageBuilder.getClasses()) { - String className = cc.getName().replace('/', '.').concat("Tests"); - - if (!className.equals(testClass)) { - continue; - } +// THIS CHECK IS OBSOLETE +// if (!className.equals(testClass)) { +// continue; +// } for (int line = cc.getFirstLine(); line <= cc.getLastLine(); line++) { ILine l = cc.getLine(line); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index d07b10980..814f6ad6a 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -11,10 +11,10 @@ public class QualityResult { private Map unitTests; // uniqueId (testId below) -> displayName private LinkedList metaTestReports; - private Map> testToMetaTests; // a useful mapping from the test cases to the meta-tests they cover + private Map> testToMetaTests; // displayName -> meta-tests - private Map> coveragePerTest; // testId -> linesCovered - private Map> mutationsKilledPerTest; // testId -> mutationId + private Map> coveragePerTest; // displayName -> linesCovered + private Map> mutationsKilledPerTest; // displayName -> mutationId public QualityResult(int numUnitTests) { // dummy: @@ -22,6 +22,8 @@ public QualityResult(int numUnitTests) { this.numUnitTests = numUnitTests; metaTestReports = new LinkedList<>(); testToMetaTests = new HashMap<>(); + coveragePerTest = new HashMap<>(); + mutationsKilledPerTest = new HashMap<>(); } public static QualityResult build(int score) { @@ -96,7 +98,7 @@ public void considerMetaTest(MetaTestReport metaTestReport) { if (testName.matches(".* \\(\\d+\\)")) { String method = testName.replaceAll(" \\(\\d+\\)$", "").trim(); - String invocation = testName.replaceAll(".* \\((\\d+)\\)$", "#$1").trim(); + String invocation = testName.replaceAll(".* \\((\\d+)\\)$", "($1)").trim(); testName = method + " " + invocation; } testToMetaTests.get(test).add(testName); @@ -230,7 +232,7 @@ public String listIsolatedTests() { StringBuilder sb = new StringBuilder("Tests that do not trigger meta-tests already covered by other tests: \n"); - for (String testName : unitTests.values()) { + for (String testName : testToMetaTests.keySet()) { if (nonisolatedTests.containsKey(testName)) { sb.append(" > " + testName + " ✕ - "); Set collisions = nonisolatedTests.get(testName); @@ -254,7 +256,7 @@ public String listContributingTests() { StringBuilder sb = new StringBuilder("Tests that increase a metric: \n"); - for (String testName : unitTests.values()) { + for (String testName : testToMetaTests.keySet()) { if (contributingTests.containsKey(testName)) { sb.append(" > " + testName + " ✓ - "); List contributions = contributingTests.get(testName); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index 9c7b8b636..5bfcc39fb 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -263,37 +263,25 @@ public QualityResult getQualityResult() { public void logUnitTests(Map unitTests) { Map unitTestsNormalized = new HashMap<>(); for (String uniqueId : unitTests.keySet()) { - - String displayName = unitTests.get(uniqueId); - - if (displayName.matches(".*\\(.*\\).*\\[\\d+\\].*")) { - String method = displayName.replaceAll("\\(.*", "").trim(); - String invocation = displayName.replaceAll(".*?\\[(\\d+)\\].*", "($1)").trim(); - displayName = method + " " + invocation; - } - - unitTestsNormalized.put(uniqueId, displayName); + unitTestsNormalized.put(uniqueId, normalizeName(unitTests.get(uniqueId))); } this.qualityResult.setUnitTests(unitTestsNormalized); } public void logCoveragePerTest(Map> coveragePerTest) { - this.qualityResult.setCoveragePerTest(coveragePerTest); + Map> coveragePerTestNormalized = new HashMap<>(); + for (String testName : coveragePerTest.keySet()) { + Set lines = coveragePerTest.get(testName); + coveragePerTestNormalized.put(normalizeName(testName), lines); + } + this.qualityResult.setCoveragePerTest(coveragePerTestNormalized); } public void logMutationsKilledPerTest(Map> mutationsKilledPerTest) { Map> mutationsKilledPerTestNormalized = new HashMap<>(); for (String testName : mutationsKilledPerTest.keySet()) { - if (testName.contains("[test-template:")) { - - Set mutations = mutationsKilledPerTest.get(testName); - - String method = testName.replaceAll(".*\\[test-template:([^(]+).*", "$1").trim(); - String invocation = testName.replaceAll(".*\\[test-template-invocation:#(\\d+)\\].*", "($1)").trim(); - String normalizedName = method + " " + invocation; - - mutationsKilledPerTestNormalized.put(normalizedName, mutations); - } + Set mutations = mutationsKilledPerTest.get(testName); + mutationsKilledPerTestNormalized.put(normalizeName(testName), mutations); } this.qualityResult.setMutationsKilledPerTest(mutationsKilledPerTestNormalized); } @@ -309,6 +297,18 @@ public void logMetaTest(String name, MetaTestReport metaTestReport) { this.qualityResult.considerMetaTest(metaTestReport); } + private String normalizeName(String name) { + String normalizedName = name; + + // parameterized unit test + if (name.contains("test-template:")) { + normalizedName = name.split("test-template:")[1].split("\\(")[0] + + " (" + name.split("test-template-invocation:#")[1].split("]")[0] + ")"; + } + + return normalizedName; + } + /* * Generic failures */ diff --git a/andy/src/test/java/integration/quality/OverviewQualityResults.java b/andy/src/test/java/integration/quality/OverviewQualityResults.java index ce005468a..baea24d80 100644 --- a/andy/src/test/java/integration/quality/OverviewQualityResults.java +++ b/andy/src/test/java/integration/quality/OverviewQualityResults.java @@ -87,7 +87,8 @@ void metaTestQualityIsAcceptable( static Stream testSuites() { return Stream.of( - Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration") + // Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration") + Arguments.of("ContainsAnyLibrary", "ContainsAnyOfficialSolution", "ContainsAnyConfiguration") ); } } diff --git a/andy/src/test/resources/grader/fixtures/Config/ContainsAnyConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/ContainsAnyConfiguration.java new file mode 100644 index 000000000..356d1099c --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/ContainsAnyConfiguration.java @@ -0,0 +1,99 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.4f); + put("mutation", 0.0f); + put("meta", 0.6f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.CollectionUtils"); + } + + @Override + public boolean skipPitest() { + return true; + } + + @Override + public List metaTests() { + return List.of( + MetaTest.insertAt("finds bug where method does not work when coll1 is empty", 25, + "if (coll1 != null && coll1.isEmpty()) return true;"), + MetaTest.insertAt("finds bug where method does not work when coll2 is empty", 25, + "if (coll2 != null && coll2.isEmpty()) return true;"), + MetaTest.insertAt("finds bug where method does not work when both coll1 and coll2 are empty", 25, + "if (coll1 != null && coll1.isEmpty() && coll2 != null && coll2.isEmpty()) return true;"), + MetaTest.withStringReplacement("finds bug where method does not work when there are no intersections", + "return false;", + "return true;"), + MetaTest.withLineReplacement("finds bug where method does not work when there are multiple intersections", + 25, 38, + """ + int numIntersections = 0; + if (coll1.size() < coll2.size()) { + for (Object aColl1 : coll1) { + if (coll2.contains(aColl1)) { + numIntersections++; + } + } + } else { + for (final Object aColl2 : coll2) { + if (coll1.contains(aColl2)) { + numIntersections++; + } + } + } + return numIntersections == 1; + """), + MetaTest.withLineReplacement("finds bug where method does not work when there is only a single intersection", + 25, 38, + """ + int numIntersections = 0; + if (coll1.size() < coll2.size()) { + for (Object aColl1 : coll1) { + if (coll2.contains(aColl1)) { + numIntersections++; + } + } + } else { + for (final Object aColl2 : coll2) { + if (coll1.contains(aColl2)) { + numIntersections++; + } + } + } + return numIntersections > 1; + """), + MetaTest.withLineReplacement("finds bug where method only checks elements at the same position", + 25, 37, + """ + var it1 = coll1.iterator(); + var it2 = coll2.iterator(); + while (it1.hasNext() && it2.hasNext()) { + Object aColl1 = it1.next(); + Object aColl2 = it2.next(); + if (aColl1.equals(aColl2)) { + return true; + } + } + """), + MetaTest.insertAt("finds bug where method does not work with collections of different sizes", + 25, + "if (coll1.size() != coll2.size()) return false;") + ); + } +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Library/ContainsAnyLibrary.java b/andy/src/test/resources/grader/fixtures/Library/ContainsAnyLibrary.java new file mode 100644 index 000000000..5e2f530a9 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/ContainsAnyLibrary.java @@ -0,0 +1,40 @@ +package delft; + +import java.util.Collection; + +class CollectionUtils { + + /** CollectionUtils should not normally be instantiated. */ + private CollectionUtils() { + } + + /** + * Returns true iff at least one element is in both collections. + * + *

+ * + * @param coll1 + * the first collection, must not be null + * @param coll2 + * the second collection, must not be null + * @return true iff the intersection of the collections is + * non-empty + * @since 2.1 + */ + public static boolean containsAny(final Collection coll1, final Collection coll2) { + if (coll1.size() < coll2.size()) { + for (final Object aColl1 : coll1) { + if (coll2.contains(aColl1)) { + return true; + } + } + } else { + for (final Object aColl2 : coll2) { + if (coll1.contains(aColl2)) { + return true; + } + } + } + return false; + } +} diff --git a/andy/src/test/resources/grader/fixtures/Solution/ContainsAnyOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/ContainsAnyOfficialSolution.java new file mode 100644 index 000000000..74266075d --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/ContainsAnyOfficialSolution.java @@ -0,0 +1,58 @@ +package delft; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.*; +import java.util.stream.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.*; +import org.junit.jupiter.params.provider.*; + +class CollectionUtilsTest { + + @ParameterizedTest(name = "{0}") + @MethodSource("generator") + void containsAny(String description, Collection coll1, Collection coll2, boolean expectedResult) { + boolean result = CollectionUtils.containsAny(coll1, coll2); + assertThat(result).isEqualTo(expectedResult); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("nullGenerator") + void nullList(String description, Collection coll1, Collection coll2) { + assertThatThrownBy(() -> CollectionUtils.containsAny(coll1, coll2)).isInstanceOf(Exception.class); + } + + private static Stream generator() { + // many elements in both + Arguments tc1 = Arguments.of("many elements in c1, c2, single intersection", Arrays.asList(1, 2, 3), + Arrays.asList(4, 1, 5), true); + Arguments tc2 = Arguments.of("many elements in c1, c2, >1 intersection", Arrays.asList(1, 2, 3), + Arrays.asList(1, 4, 2), true); + Arguments tc3 = Arguments.of("many elements in c1, c2, no intersection", Arrays.asList(1, 2, 3), + Arrays.asList(4, 5, 6), false); + // single element in c1 + Arguments tc4 = Arguments.of("single element in c1, many in c2, intersection", Arrays.asList(1), + Arrays.asList(1, 4, 5), true); + Arguments tc5 = Arguments.of("single element in c1, many in c2, no intersection", Arrays.asList(1), + Arrays.asList(4, 5, 6), false); + // single element in c2 + Arguments tc6 = Arguments.of("many elements in c1, single in c2, intersection", Arrays.asList(1, 4, 5), + Arrays.asList(1), true); + Arguments tc7 = Arguments.of("many elements in c1, single in c2, no intersection", Arrays.asList(4, 5, 6), + Arrays.asList(1), false); + // empty lists + Arguments tc8 = Arguments.of("empty c1", new ArrayList(), Arrays.asList(1), false); + Arguments tc9 = Arguments.of("empty c2", Arrays.asList(1), Collections.emptyList(), false); + Arguments tc10 = Arguments.of("empty c1 and c2", Collections.emptyList(), Collections.emptyList(), false); + return Stream.of(tc1, tc2, tc3, tc4, tc5, tc6, tc7, tc8, tc9, tc10); + } + + private static Stream nullGenerator() { + Arguments tc1 = Arguments.of("c1 null", null, Arrays.asList(1, 4, 5)); + Arguments tc2 = Arguments.of("c2 null", Arrays.asList(1, 2, 3), null); + Arguments tc3 = Arguments.of("both null", null, null); + return Stream.of(tc1, tc2, tc3); + } +} From db9e46014271caf3f02643edd5fb1bbd9b9c9111 Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Sun, 15 Mar 2026 16:38:05 +0100 Subject: [PATCH 20/23] Adding more test suites with parameterized unit tests --- .../quality/OverviewQualityResults.java | 4 +- .../Config/IsSortedConfiguration.java | 88 +++++++++++++++++ .../Config/lastIndexOfConfiguration.java | 94 +++++++++++++++++++ .../fixtures/Library/IsSortedLibrary.java | 33 +++++++ .../fixtures/Library/lastIndexOfLibrary.java | 52 ++++++++++ .../Solution/IsSortedOfficialSolution.java | 32 +++++++ .../Solution/LastIndexOfOfficialSolution.java | 38 ++++++++ 7 files changed, 340 insertions(+), 1 deletion(-) create mode 100644 andy/src/test/resources/grader/fixtures/Config/IsSortedConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/lastIndexOfConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/IsSortedLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/lastIndexOfLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/IsSortedOfficialSolution.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/LastIndexOfOfficialSolution.java diff --git a/andy/src/test/java/integration/quality/OverviewQualityResults.java b/andy/src/test/java/integration/quality/OverviewQualityResults.java index baea24d80..19cd3668b 100644 --- a/andy/src/test/java/integration/quality/OverviewQualityResults.java +++ b/andy/src/test/java/integration/quality/OverviewQualityResults.java @@ -88,7 +88,9 @@ void metaTestQualityIsAcceptable( static Stream testSuites() { return Stream.of( // Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration") - Arguments.of("ContainsAnyLibrary", "ContainsAnyOfficialSolution", "ContainsAnyConfiguration") + // Arguments.of("ContainsAnyLibrary", "ContainsAnyOfficialSolution", "ContainsAnyConfiguration") + // Arguments.of("LastIndexOfLibrary", "LastIndexOfOfficialSolution", "LastIndexOfConfiguration") + Arguments.of("IsSortedLibrary", "IsSortedOfficialSolution", "IsSortedConfiguration") ); } } diff --git a/andy/src/test/resources/grader/fixtures/Config/IsSortedConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/IsSortedConfiguration.java new file mode 100644 index 000000000..373d9cd97 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/IsSortedConfiguration.java @@ -0,0 +1,88 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.2f); + put("meta", 0.7f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.ArrayUtils"); + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withLineReplacement("finds bug where false is returned for null input", + 19, 21, + """ + if (array == null) { + return false; + } + if (array.length < 2) { + return true; + } + """), + MetaTest.withStringReplacement("finds bug where check for length < 2 is missing", + " || array.length < 2", + ""), + MetaTest.withStringReplacement("finds bug where last element is ignored", + "i < n", + "i < n - 1"), + MetaTest.withLineReplacement("finds bug where only the sorting of the last two elements is considered", + 24, 31, + """ + boolean result = true; + for (int i = 1; i < n; i++) { + final int current = array[i]; + result = (previous <= current); + previous = current; + } + return result;"""), + MetaTest.withLineReplacement("finds bug where method throws exception if array has one element", + 19, 31, + """ + if (array == null || array.length == 0) { + return true; + } + int previous = array[0]; + int current = array[1]; + final int n = array.length; + for (int i = 2; i < n; i++) { + if (previous > current) { + return false; + } + previous = current; + current = array[i]; + } + if (previous > current) { + return false; + } + return true;"""), + MetaTest.withStringReplacement("finds bug where condition is changed to previous >= current", + "previous > current", + "previous >= current"), + MetaTest.withLineReplacement("finds bug where true is returned even when the input is unsorted", + 27, 27, + "return true;"), + MetaTest.withLineReplacement("finds bug where false is returned for a sorted list of length >= 2", + 31, 31, + "return false;" + ) + ); + } + +} diff --git a/andy/src/test/resources/grader/fixtures/Config/lastIndexOfConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/lastIndexOfConfiguration.java new file mode 100644 index 000000000..c7a75952b --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/lastIndexOfConfiguration.java @@ -0,0 +1,94 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.3f); + put("mutation", 0.0f); + put("meta", 0.7f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.ArrayUtils"); + } + + @Override + public boolean skipPitest() { + return true; + } + + @Override + public List metaTests() { + return List.of( + MetaTest.insertAt("finds bug where negative startIndex throws exception", + 37, + """ + if (startIndex < 0) { + throw new IllegalArgumentException("startIndex must be non-negative."); + } + """), + MetaTest.withLineReplacement("finds bug where a startIndex higher than the array length isn't caught", + 42, 44, + """ + } + """), + MetaTest.insertAt("finds bug where an array containing only the value to find does not return the correct index", + 40, + """ + if(array.length == 1){ + return INDEX_NOT_FOUND; + } + """), + MetaTest.insertAt("finds bug where a single element array, not containing the value to find, does not return index -1", + 40, + """ + if(array.length == 1){ + return array[0]; + } + """), + MetaTest.insertAt("finds bug where the value to find can only be at startIndex", + 49, + """ + else{ + return INDEX_NOT_FOUND; + } + """), + MetaTest.withLineReplacement("finds bug where the value to find cannot be at startIndex", + 45, 45, + """ + for (int i = startIndex - 1; i >= 0; i--) { + """), + MetaTest.withLineReplacement("finds bug where invalid index positions are not considered", + 40, 44, + "" + ), + MetaTest.withStringReplacement("finds bug where the index is of the last value that is not the value to find", + "valueToFind == array[i]", "valueToFind != array[i]"), + MetaTest.withStringReplacement("finds bug where the index of the first occurrence of the value is returned", + "int i = startIndex; i >= 0; i--", "int i = 0; i <= startIndex; i++"), + MetaTest.withLineReplacement("finds bug where the first value in the array is not checked", + 45, 45, + """ + for (int i = startIndex; i >= 1; i--) { + """), + MetaTest.withLineReplacement("finds bug where startIndex is ignored", + 45, 45, + """ + for (int i = array.length; i >= 0; i--) { + """), + MetaTest.withStringReplacement("finds bug where -1 is returned even when value is found", + "return i;", "return INDEX_NOT_FOUND;") + ); + } +} diff --git a/andy/src/test/resources/grader/fixtures/Library/IsSortedLibrary.java b/andy/src/test/resources/grader/fixtures/Library/IsSortedLibrary.java new file mode 100644 index 000000000..12d7c0904 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/IsSortedLibrary.java @@ -0,0 +1,33 @@ +package delft; + +class ArrayUtils { + + /** ArrayUtils should not normally be instantiated. */ + private ArrayUtils() { + } + + /** + * This method checks whether the provided array is sorted according to natural + * ordering. + * + * @param array + * the array to check + * @return whether the array is sorted according to natural ordering + * @since 3.4 + */ + public static boolean isSorted(final int[] array) { + if (array == null || array.length < 2) { + return true; + } + int previous = array[0]; + final int n = array.length; + for (int i = 1; i < n; i++) { + final int current = array[i]; + if (previous > current) { + return false; + } + previous = current; + } + return true; + } +} diff --git a/andy/src/test/resources/grader/fixtures/Library/lastIndexOfLibrary.java b/andy/src/test/resources/grader/fixtures/Library/lastIndexOfLibrary.java new file mode 100644 index 000000000..16697f105 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/lastIndexOfLibrary.java @@ -0,0 +1,52 @@ +package delft; + +class ArrayUtils { + /** ArrayUtils should not normally be instantiated. */ + private ArrayUtils() { + } + + /** + * The index value when an element is not found in a list or array: {@code -1}. + * This value is returned by methods in this class and can also be used in + * comparisons with values returned by various method from + * {@link java.util.List}. + */ + public static final int INDEX_NOT_FOUND = -1; + + /** + * Finds the last index of the given value in the array starting at the given index. + * + *

+ * This method returns {@link #INDEX_NOT_FOUND} ({@code -1}) for a {@code null} + * input array. + * + *

+ * A negative startIndex will return {@link #INDEX_NOT_FOUND} ({@code -1}). A + * startIndex larger than the array length will search from the end of the array. + * + * @param array + * the array to traverse for looking for the object, may be {@code null} + * @param valueToFind + * the value to find + * @param startIndex + * the start index to traverse backwards from + * @return the last index of the value within the array, + * {@link #INDEX_NOT_FOUND} ({@code -1}) if not found or {@code null} array input + */ + public static int lastIndexOf(final int[] array, final int valueToFind, int startIndex) { + if (array == null) { + return INDEX_NOT_FOUND; + } + if (startIndex < 0) { + return INDEX_NOT_FOUND; + } else if (startIndex >= array.length) { + startIndex = array.length - 1; + } + for (int i = startIndex; i >= 0; i--) { + if (valueToFind == array[i]) { + return i; + } + } + return INDEX_NOT_FOUND; + } +} diff --git a/andy/src/test/resources/grader/fixtures/Solution/IsSortedOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/IsSortedOfficialSolution.java new file mode 100644 index 000000000..256aaef0b --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/IsSortedOfficialSolution.java @@ -0,0 +1,32 @@ +package delft; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.*; +import java.util.stream.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.*; +import org.junit.jupiter.params.provider.*; + +class ArrayUtilsTest { + + @ParameterizedTest(name = "{0}") + @MethodSource("generator") + void isSorted(String description, int[] array, boolean expectedResult) { + assertThat(ArrayUtils.isSorted(array)).isEqualTo(expectedResult); + } + + private static Stream generator() { + Arguments tc0 = Arguments.of("empty", new int[]{}, true); + Arguments tc1 = Arguments.of("single element", new int[]{1}, true); + Arguments tc2 = Arguments.of("unsorted", new int[]{1, 3, 2}, false); + Arguments tc3 = Arguments.of("sorted", new int[]{1, 2, 3}, true); + Arguments tc4 = Arguments.of("null array", null, true); + Arguments tc5 = Arguments.of("sorted with doubles", new int[]{1, 1, 2, 2, 3}, true); + Arguments tc6 = Arguments.of("two elements", new int[]{2, 1}, false); + Arguments tc7 = Arguments.of("unsorted followed by sorted", new int[]{2, 1, 3}, false); + return Stream.of(tc0, tc1, tc2, tc3, tc4, tc5, tc6, tc7); + } + +} diff --git a/andy/src/test/resources/grader/fixtures/Solution/LastIndexOfOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/LastIndexOfOfficialSolution.java new file mode 100644 index 000000000..6d5421eb5 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/LastIndexOfOfficialSolution.java @@ -0,0 +1,38 @@ +package delft; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.*; +import java.util.stream.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.*; +import org.junit.jupiter.params.provider.*; + +class ArrayUtilsTest { + + public static final int INDEX_NOT_FOUND = -1; + + @ParameterizedTest + @MethodSource("generator") + void lastIndexOf(String description, int[] array, int valueToFind, int startIndex, int expectedResult) { + assertThat(ArrayUtils.lastIndexOf(array, valueToFind, startIndex)).isEqualTo(expectedResult); + } + + private static Stream generator() { + Arguments tc1 = Arguments.of("T1 null array", null, 1, 2, INDEX_NOT_FOUND); + Arguments tc2 = Arguments.of("T2 negative index", new int[]{0, 1, 2}, 2, -1, INDEX_NOT_FOUND); + Arguments tc3 = Arguments.of("T3 index bigger than array", new int[]{0, 1, 2}, 1, 5, 1); + Arguments tc4 = Arguments.of("T4 empty array", new int[]{}, 0, 1, INDEX_NOT_FOUND); + Arguments tc5 = Arguments.of("T5 length one array with element", new int[]{1}, 1, 0, 0); + Arguments tc6 = Arguments.of("T6 length one array without element", new int[]{1}, 2, 0, INDEX_NOT_FOUND); + Arguments tc7 = Arguments.of("T7 array with element", new int[]{0, 1, 2}, 1, 2, 1); + Arguments tc8 = Arguments.of("T8 array with element many times", new int[]{0, 1, 1, 2}, 1, 2, 2); + Arguments tc9 = Arguments.of("T9 array without element", new int[]{0, 1, 2}, 3, 2, INDEX_NOT_FOUND); + Arguments tc10 = Arguments.of("T10 array with element, start index == 0", new int[]{0, 1, 2}, 0, 0, 0); + Arguments tc11 = Arguments.of("T11 array without element, start index == 0", new int[]{0, 1, 2}, 3, 0, + INDEX_NOT_FOUND); + Arguments tc12 = Arguments.of("T12 element at the index", new int[]{0, 1, 1, 2}, 1, 1, 1); + return Stream.of(tc1, tc2, tc3, tc4, tc5, tc6, tc7, tc8, tc9, tc10, tc11, tc12); + } +} \ No newline at end of file From f2fb04c027f13d0823eb5eef973afee6f8784c41 Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Sun, 15 Mar 2026 17:17:12 +0100 Subject: [PATCH 21/23] Adding test suite with non-parameterized unit tests --- .../cse1110/andy/result/QualityResult.java | 22 +- .../cse1110/andy/result/ResultBuilder.java | 8 +- .../quality/OverviewQualityResults.java | 5 +- .../AutoAssignStudentsConfiguration.java | 74 +++++++ .../Library/AutoAssignStudentsLibrary.java | 203 ++++++++++++++++++ .../AutoAssignStudentsOfficialSolution.java | 145 +++++++++++++ 6 files changed, 442 insertions(+), 15 deletions(-) create mode 100644 andy/src/test/resources/grader/fixtures/Config/AutoAssignStudentsConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/AutoAssignStudentsLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/AutoAssignStudentsOfficialSolution.java diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index 814f6ad6a..d24ddde03 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -90,17 +90,18 @@ public void considerMetaTest(MetaTestReport metaTestReport) { for (TestFailureInfo failure : metaTestReport.getTestsTriggered()) { String test = failure.getTestCase(); - if (testToMetaTests.get(test) == null) { - testToMetaTests.put(test, new HashSet<>()); - } + if (test.endsWith("()")) test = test.substring(0, test.length() - 2); + + testToMetaTests.computeIfAbsent(test, k -> new HashSet<>()); String testName = metaTestReport.getName(); - if (testName.matches(".* \\(\\d+\\)")) { - String method = testName.replaceAll(" \\(\\d+\\)$", "").trim(); - String invocation = testName.replaceAll(".* \\((\\d+)\\)$", "($1)").trim(); - testName = method + " " + invocation; - } +// if (testName.matches(".* \\(\\d+\\)")) { +// String method = testName.replaceAll(" \\(\\d+\\)$", "").trim(); +// String invocation = testName.replaceAll(".* \\((\\d+)\\)$", "($1)").trim(); +// testName = method + " " + invocation; +// } + testToMetaTests.get(test).add(testName); } } @@ -138,12 +139,11 @@ public long countIsolatedTests() { // All tests that trigger this meta-test collide with each other List collidingTests = metaTestReport.getTestsTriggered().stream() .map(TestFailureInfo::getTestCase) + .map(test -> test.endsWith("()") ? test.substring(0, test.length() - 2) : test) .toList(); for (String test : collidingTests) { - nonisolatedTests.computeIfAbsent(test, t -> { - return new HashSet<>(); - }) + nonisolatedTests.computeIfAbsent(test, t -> new HashSet<>()) .addAll(collidingTests.stream() .filter(other -> !other.equals(test)) .collect(Collectors.toSet())); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index 5bfcc39fb..1ff405134 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -263,7 +263,7 @@ public QualityResult getQualityResult() { public void logUnitTests(Map unitTests) { Map unitTestsNormalized = new HashMap<>(); for (String uniqueId : unitTests.keySet()) { - unitTestsNormalized.put(uniqueId, normalizeName(unitTests.get(uniqueId))); + unitTestsNormalized.put(uniqueId, normalizeName(uniqueId)); } this.qualityResult.setUnitTests(unitTestsNormalized); } @@ -300,7 +300,11 @@ public void logMetaTest(String name, MetaTestReport metaTestReport) { private String normalizeName(String name) { String normalizedName = name; - // parameterized unit test + // @Test + if (name.contains("method:")) { + normalizedName = name.split("method:")[1].split("\\(")[0]; + } + // @ParameterizedTest if (name.contains("test-template:")) { normalizedName = name.split("test-template:")[1].split("\\(")[0] + " (" + name.split("test-template-invocation:#")[1].split("]")[0] + ")"; diff --git a/andy/src/test/java/integration/quality/OverviewQualityResults.java b/andy/src/test/java/integration/quality/OverviewQualityResults.java index 19cd3668b..f1334ad75 100644 --- a/andy/src/test/java/integration/quality/OverviewQualityResults.java +++ b/andy/src/test/java/integration/quality/OverviewQualityResults.java @@ -87,10 +87,11 @@ void metaTestQualityIsAcceptable( static Stream testSuites() { return Stream.of( - // Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration") + Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration") // Arguments.of("ContainsAnyLibrary", "ContainsAnyOfficialSolution", "ContainsAnyConfiguration") // Arguments.of("LastIndexOfLibrary", "LastIndexOfOfficialSolution", "LastIndexOfConfiguration") - Arguments.of("IsSortedLibrary", "IsSortedOfficialSolution", "IsSortedConfiguration") + // Arguments.of("IsSortedLibrary", "IsSortedOfficialSolution", "IsSortedConfiguration") + // Arguments.of("AutoAssignStudentsLibrary", "AutoAssignStudentsOfficialSolution", "AutoAssignStudentsConfiguration") ); } } diff --git a/andy/src/test/resources/grader/fixtures/Config/AutoAssignStudentsConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/AutoAssignStudentsConfiguration.java new file mode 100644 index 000000000..51ced9777 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/AutoAssignStudentsConfiguration.java @@ -0,0 +1,74 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; +import nl.tudelft.cse1110.andy.execution.mode.Mode; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Mode mode() { + return Mode.PRACTICE; + } + + public int numberOfMutationsToConsider() { + return 15; + } + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.2f); + put("mutation", 0.2f); + put("meta", 0.6f); + put("codechecks", 0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.AutoAssigner", "delft.Workshop"); + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withStringReplacement( + "check that number of spots goes down", + "spotsPerDate.put(date, spotsPerDate.get(date) - 1);", + "spotsPerDate.put(date, spotsPerDate.get(date));" + ), + MetaTest.withStringReplacement( + "checks different dates", + ".findFirst()", + ".skip(1).findFirst()" + ), + MetaTest.withStringReplacement( + "tries date with single spot", + "public boolean hasAvailableDate() {", + "public boolean hasAvailableDate() { if(spotsPerDate.values().stream().anyMatch(v -> v == 1)) throw new RuntimeException();" + ), + MetaTest.withStringReplacement( + "all assignments failed", + "return assignments;", + "if(!assignments.getErrors().isEmpty() && assignments.getAssignments().isEmpty()) throw new RuntimeException(); return assignments;" + ), + MetaTest.withStringReplacement( + "check the failed assignments in Assignments", + "assignments.noAvailableSpots(workshop, student);", + "" + ), + MetaTest.withStringReplacement( + "check assignments in Assignments", + "assignments.assign(workshop, student, nextDate);", + "" + ) + ); + } + +} + diff --git a/andy/src/test/resources/grader/fixtures/Library/AutoAssignStudentsLibrary.java b/andy/src/test/resources/grader/fixtures/Library/AutoAssignStudentsLibrary.java new file mode 100644 index 000000000..154b56daa --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/AutoAssignStudentsLibrary.java @@ -0,0 +1,203 @@ +package delft; + +import java.util.*; +import java.time.*; +import java.time.format.*; +import java.lang.annotation.*; + +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.METHOD}) +@interface GeneratedExcludeFromJacoco { +} + +class Student { + private final int id; + private final String name; + private final String email; + + public Student(int id, String name, String email) { + this.id = id; + this.name = name; + this.email = email; + } + + public String getName() { + return this.name; + } + + public String getEmail() { + return this.email; + } + + @GeneratedExcludeFromJacoco + public int getId() { + return this.id; + } + + @Override + @GeneratedExcludeFromJacoco + public int hashCode() { + return Objects.hash(this.id, this.name, this.email); + } + + @Override + @GeneratedExcludeFromJacoco + public boolean equals(Object o) { + // self check + if (this == o) + return true; + // null check + if (o == null) + return false; + // type check and cast + if (getClass() != o.getClass()) + return false; + Student other = (Student) o; + // field comparison + return Objects.equals(id, other.id) + && Objects.equals(name, other.name) + && Objects.equals(email, other.email); + } +} + +class Workshop { + private final int id; + private final String name; + private final Map spotsPerDate; + + public Workshop(int id, String name, Map spotsPerDate) { + this.id = id; + this.name = name; + this.spotsPerDate = new HashMap(spotsPerDate); + } + + @GeneratedExcludeFromJacoco + public Map getSpotsPerDate() { + return new HashMap(this.spotsPerDate); + } + + public String getName() { + return this.name; + } + + @GeneratedExcludeFromJacoco + public int getId() { + return this.id; + } + + public boolean hasAvailableDate() { + // If any date has more than zero spots, this means there's an available date + return spotsPerDate.values().stream().anyMatch(v -> v > 0); + } + + // If there's an available date, it returns this date. + // This method should not be called if there's no spot available. + public ZonedDateTime getNextAvailableDate() { + // picks the first one that has available spots + ZonedDateTime firstAvailableDate = spotsPerDate + .entrySet() + .stream() + .filter(e -> e.getValue() > 0) // get all entries in the map that have a spot left + .map(Map.Entry::getKey) // get the dates (which are the keys in the map) + .sorted() // order the keys + .findFirst() // get the first one + .get(); + + return firstAvailableDate; + } + + // This method takes up a spot on the given date. + // You should not call this method with an invalid date. + // (In real code, you would want to write some defensive code here, + // we just don't do it for the sake of simplicity in the exam.) + public void takeASpot(ZonedDateTime date) { + spotsPerDate.put(date, spotsPerDate.get(date) - 1); + } + + @Override + @GeneratedExcludeFromJacoco + public int hashCode() { + return Objects.hash(this.id, this.name, this.spotsPerDate); + } + + @Override + @GeneratedExcludeFromJacoco + public boolean equals(Object o) { + // self check + if (this == o) + return true; + // null check + if (o == null) + return false; + // type check and cast + if (getClass() != o.getClass()) + return false; + Workshop other = (Workshop) o; + // field comparison + return Objects.equals(id, other.id) + && Objects.equals(name, other.name) + && Objects.equals(spotsPerDate, other.spotsPerDate); + } +} + +// In real life, this class would assign students to workshops +// by means of, e.g., persisting new information in a database. +// For the sake of simplicity in this exam, we just store this information as a log. +class AssignmentsLogger { + private final List errors = new ArrayList(); + private final List assignments = new ArrayList(); + // The formatter generates strings like "13/07/2022 14:00" + private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd/MM/yyyy HH:mm"); + + public void noAvailableSpots(Workshop workshop, Student student) { + // Example of the added string: "Python,Frank" + errors.add(String.format("%s,%s", workshop.getName(), student.getName())); + } + + public void assign(Workshop workshop, Student student, ZonedDateTime date) { + // Example of the added string: "Java,Mauricio,13/07/2022 14:00" + assignments.add(String.format("%s,%s,%s", workshop.getName(), student.getName(), format(date))); + } + + public List getErrors() { + return Collections.unmodifiableList(errors); + } + + public List getAssignments() { + return Collections.unmodifiableList(assignments); + } + + private String format(ZonedDateTime date) { + return date.format(formatter); + } +} + + +class AutoAssigner { + public AssignmentsLogger assign(List students, + List workshops) { + + AssignmentsLogger assignments = new AssignmentsLogger(); + + for(Workshop workshop : workshops) { + for(Student student : students) { + // Get the next available date for that workshop + // If there's no way, nothing we can do, just log it! + if(!workshop.hasAvailableDate()) { + assignments.noAvailableSpots(workshop, student); + } else { + // Great, that workshop has an available date! + // Let's assign the student to that workshop on that specific date + ZonedDateTime nextDate = workshop.getNextAvailableDate(); + assignments.assign(workshop, student, nextDate); + workshop.takeASpot(nextDate); + } + } + } + + return assignments; + + } + +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Solution/AutoAssignStudentsOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/AutoAssignStudentsOfficialSolution.java new file mode 100644 index 000000000..c66ff2802 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/AutoAssignStudentsOfficialSolution.java @@ -0,0 +1,145 @@ +package delft; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; +import static org.assertj.core.api.Assertions.within; +import java.time.temporal.ChronoUnit; + +import java.util.*; +import java.util.stream.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.*; +import org.junit.jupiter.params.provider.*; +import java.time.*; + +class AutoAssignerTest { + + @Test + void all_students_in_the_same_first_date() { + List students = Arrays.asList( + new Student(1, "Mauricio", "m.f.aniche@tudelft.nl"), + new Student(2, "Frank", "f.mulder@tudelft.nl") + ); + + List workshops = Arrays.asList( + new Workshop(1, "Java", new HashMap<>() {{ + put(date(2022, 1, 1, 14, 0), 5); + }}), + new Workshop(2, "Networks", new HashMap<>() {{ + put(date(2022, 1, 2, 14, 0), 5); + }}) + ); + + AutoAssigner assigner = new AutoAssigner(); + AssignmentsLogger result = assigner.assign(students, workshops); + + assertThat(result.getErrors()).isEmpty(); + assertThat(result.getAssignments()).containsExactlyInAnyOrder( + "Java,Mauricio,01/01/2022 14:00", + "Java,Frank,01/01/2022 14:00", + "Networks,Mauricio,02/01/2022 14:00", + "Networks,Frank,02/01/2022 14:00" + ); + } + + @Test + void uses_different_dates() { + List students = Arrays.asList( + new Student(1, "Mauricio", "m.f.aniche@tudelft.nl"), + new Student(2, "Frank", "f.mulder@tudelft.nl"), + new Student(3, "Martin", "m.mladenov@student.tudelft.nl"), + new Student(4, "Sara", "s.juhosova@student.tudelft.nl") + ); + + List workshops = Arrays.asList( + new Workshop(1, "Java", new HashMap<>() {{ + put(date(2022, 1, 1, 14, 0), 1); + put(date(2022, 1, 1, 17, 0), 2); + put(date(2022, 1, 1, 19, 0), 1); + }}), + new Workshop(2, "Networks", new HashMap<>() {{ + put(date(2022, 1, 2, 14, 0), 1); + put(date(2022, 1, 2, 17, 0), 2); + put(date(2022, 1, 2, 19, 0), 1); + }}) + ); + + AutoAssigner assigner = new AutoAssigner(); + AssignmentsLogger result = assigner.assign(students, workshops); + + assertThat(result.getErrors()).isEmpty(); + assertThat(result.getAssignments()).containsExactlyInAnyOrder( + "Java,Mauricio,01/01/2022 14:00", + "Java,Frank,01/01/2022 17:00", + "Java,Martin,01/01/2022 17:00", + "Java,Sara,01/01/2022 19:00", + "Networks,Mauricio,02/01/2022 14:00", + "Networks,Frank,02/01/2022 17:00", + "Networks,Martin,02/01/2022 17:00", + "Networks,Sara,02/01/2022 19:00" + ); + } + + @Test + void no_spaces_for_everybody() { + List students = Arrays.asList( + new Student(1, "Mauricio", "m.f.aniche@tudelft.nl"), + new Student(2, "Frank", "f.mulder@tudelft.nl") + ); + + List workshops = Arrays.asList( + new Workshop(1, "Java", new HashMap<>() {{ + put(date(2022, 1, 1, 14, 0), 1); + }}), + new Workshop(2, "Networks", new HashMap<>() {{ + put(date(2022, 1, 2, 14, 0), 1); + }}) + ); + + AutoAssigner assigner = new AutoAssigner(); + AssignmentsLogger result = assigner.assign(students, workshops); + + assertThat(result.getErrors()).containsExactlyInAnyOrder( + "Java,Frank", + "Networks,Frank" + ); + assertThat(result.getAssignments()).containsExactlyInAnyOrder( + "Java,Mauricio,01/01/2022 14:00", + "Networks,Mauricio,02/01/2022 14:00" + ); + } + + // corner cases + @Test + void everything_full() { + List students = Arrays.asList( + new Student(1, "Mauricio", "m.f.aniche@tudelft.nl"), + new Student(2, "Frank", "f.mulder@tudelft.nl") + ); + + List workshops = Arrays.asList( + new Workshop(1, "Java", new HashMap<>() {{ + put(date(2022, 1, 1, 14, 0), 0); + put(date(2022, 1, 1, 17, 0), 0); + }}), + new Workshop(2, "Networks", new HashMap<>() {{ + put(date(2022, 1, 2, 14, 0), 0); + put(date(2022, 1, 2, 17, 0), 0); + }}) + ); + + AutoAssigner assigner = new AutoAssigner(); + AssignmentsLogger result = assigner.assign(students, workshops); + + assertThat(result.getErrors()).containsExactlyInAnyOrder( + "Java,Mauricio","Networks,Mauricio", + "Java,Frank","Networks,Frank"); + assertThat(result.getAssignments()).isEmpty(); + } + + private ZonedDateTime date(int year, int month, int day, int hour, int minute) { + return ZonedDateTime.of(year, month, day, hour, minute, 0, 0, ZoneId.systemDefault()); + } + + +} \ No newline at end of file From e76a06a6895da8a2d35083cdac5b07c00b810c49 Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Mon, 16 Mar 2026 12:19:43 +0100 Subject: [PATCH 22/23] Making overview of results for domain testing test suites --- .../quality/OverviewQualityResults.java | 32 +++- .../Config/BalancingArraysConfiguration.java | 96 ++++++++++ .../CodeSnippetGeneratorConfiguration.java | 58 ++++++ .../Config/CountingClumpsConfiguration.java | 109 +++++++++++ .../Config/GetCheapestPriceConfiguration.java | 63 +++++++ .../Config/IntersectionConfiguration.java | 73 ++++++++ .../IsEqualCollectionConfiguration.java | 73 ++++++++ .../fixtures/Config/RepeatConfiguration.java | 67 +++++++ .../fixtures/Config/ReplaceConfiguration.java | 81 ++++++++ .../fixtures/Config/ReverseConfiguration.java | 92 +++++++++ .../SubstringsBetweenConfiguration.java | 110 +++++++++++ .../Config/SwapCaseConfiguration.java | 91 +++++++++ .../Config/ToCamelCaseConfiguration.java | 74 ++++++++ .../fixtures/Config/ZigZagConfiguration.java | 49 +++++ .../Library/BalancingArraysLibrary.java | 36 ++++ .../Library/CodeSnippetGeneratorLibrary.java | 59 ++++++ .../Library/CountingClumpsLibrary.java | 38 ++++ .../Library/GetCheapestPriceLibrary.java | 66 +++++++ .../fixtures/Library/IntersectionLibrary.java | 44 +++++ .../Library/IsEqualCollectionLibrary.java | 128 +++++++++++++ .../fixtures/Library/RepeatLibrary.java | 78 ++++++++ .../fixtures/Library/ReplaceLibrary.java | 145 +++++++++++++++ .../fixtures/Library/ReverseLibrary.java | 41 ++++ .../Library/SubstringsBetweenLibrary.java | 80 ++++++++ .../fixtures/Library/SwapCaseLibrary.java | 59 ++++++ .../fixtures/Library/ToCamelCaseLibrary.java | 92 +++++++++ .../fixtures/Library/ZigZagLibrary.java | 63 +++++++ .../CodeSnippetGeneratorOfficialSolution.java | 175 ++++++++++++++++++ .../CountingClumpsOfficialSolution.java | 28 +++ .../GetCheapestPriceOfficialSolution.java | 97 ++++++++++ .../IntersectionOfficialSolution.java | 50 +++++ .../IsEqualCollectionOfficialSolution.java | 54 ++++++ .../Solution/RepeatOfficialSolution.java | 33 ++++ .../Solution/ReplaceOfficialSolution.java | 87 +++++++++ .../Solution/ReverseOfficialSolution.java | 45 +++++ .../SubstringsBetweenOfficialSolution.java | 78 ++++++++ .../Solution/SwapCaseOfficialSolution.java | 33 ++++ .../Solution/ToCamelCaseOfficialSolution.java | 46 +++++ .../Solution/ZigZagOfficialSolution.java | 95 ++++++++++ 39 files changed, 2811 insertions(+), 7 deletions(-) create mode 100644 andy/src/test/resources/grader/fixtures/Config/BalancingArraysConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/CodeSnippetGeneratorConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/CountingClumpsConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/GetCheapestPriceConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/IntersectionConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/IsEqualCollectionConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/RepeatConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/ReplaceConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/ReverseConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/SubstringsBetweenConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/SwapCaseConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/ToCamelCaseConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Config/ZigZagConfiguration.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/BalancingArraysLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/CodeSnippetGeneratorLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/CountingClumpsLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/GetCheapestPriceLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/IntersectionLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/IsEqualCollectionLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/RepeatLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/ReplaceLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/ReverseLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/SubstringsBetweenLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/SwapCaseLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/ToCamelCaseLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Library/ZigZagLibrary.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/CodeSnippetGeneratorOfficialSolution.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/CountingClumpsOfficialSolution.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/GetCheapestPriceOfficialSolution.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/IntersectionOfficialSolution.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/IsEqualCollectionOfficialSolution.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/RepeatOfficialSolution.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/ReplaceOfficialSolution.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/ReverseOfficialSolution.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/SubstringsBetweenOfficialSolution.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/SwapCaseOfficialSolution.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/ToCamelCaseOfficialSolution.java create mode 100644 andy/src/test/resources/grader/fixtures/Solution/ZigZagOfficialSolution.java diff --git a/andy/src/test/java/integration/quality/OverviewQualityResults.java b/andy/src/test/java/integration/quality/OverviewQualityResults.java index f1334ad75..ea93190fb 100644 --- a/andy/src/test/java/integration/quality/OverviewQualityResults.java +++ b/andy/src/test/java/integration/quality/OverviewQualityResults.java @@ -6,19 +6,16 @@ import nl.tudelft.cse1110.andy.execution.mode.Action; import nl.tudelft.cse1110.andy.execution.mode.Mode; import nl.tudelft.cse1110.andy.execution.mode.ModeActionSelector; -import nl.tudelft.cse1110.andy.result.*; import nl.tudelft.cse1110.andy.writer.ResultWriter; import nl.tudelft.cse1110.andy.writer.standard.CodeSnippetGenerator; import nl.tudelft.cse1110.andy.writer.standard.RandomAsciiArtGenerator; import nl.tudelft.cse1110.andy.writer.standard.StandardResultWriter; import nl.tudelft.cse1110.andy.writer.standard.VersionInformation; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import testutils.ResultTestDataBuilder; import nl.tudelft.cse1110.andy.result.Result; import java.io.File; @@ -70,8 +67,8 @@ protected String generatedResult() { * For an overview of the results. To be used in production only. */ @ParameterizedTest - @MethodSource("testSuites") - void metaTestQualityIsAcceptable( + @MethodSource("domainTestingTestSuites") + void metaTestQuality( String libraryFile, String solutionFile, String configurationFile @@ -85,13 +82,34 @@ void metaTestQualityIsAcceptable( System.out.println(output); } - static Stream testSuites() { + static Stream domainTestingTestSuites() { return Stream.of( - Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration") + // Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration") + // ## Week 1 // Arguments.of("ContainsAnyLibrary", "ContainsAnyOfficialSolution", "ContainsAnyConfiguration") // Arguments.of("LastIndexOfLibrary", "LastIndexOfOfficialSolution", "LastIndexOfConfiguration") // Arguments.of("IsSortedLibrary", "IsSortedOfficialSolution", "IsSortedConfiguration") + // ## Week 2 + // Arguments.of("SwapCaseLibrary", "SwapCaseOfficialSolution", "SwapCaseConfiguration") + // Arguments.of("RepeatLibrary", "RepeatOfficialSolution", "RepeatConfiguration") + // Arguments.of("IntersectionLibrary", "IntersectionOfficialSolution", "IntersectionConfiguration") + // Arguments.of("GetCheapestPriceLibrary", "GetCheapestPriceOfficialSolution", "GetCheapestPriceConfiguration") + // Arguments.of("IsEqualCollectionLibrary", "IsEqualCollectionOfficialSolution", "IsEqualCollectionConfiguration") // Arguments.of("AutoAssignStudentsLibrary", "AutoAssignStudentsOfficialSolution", "AutoAssignStudentsConfiguration") + // ## Week 3 + // Arguments.of("CountingClumpsLibrary", "CountingClumpsOfficialSolution", "CountingClumpsConfiguration") + // Arguments.of("ReplaceLibrary", "ReplaceOfficialSolution", "ReplaceConfiguration") + // Arguments.of("ToCamelCaseLibrary", "ToCamelCaseOfficialSolution", "ToCamelCaseConfiguration") + // ## Week 4 + // Arguments.of("BalancingArraysLibrary", "BalancingArraysOfficialSolution", "BalancingArraysConfiguration") + // ## Week 5 + // Arguments.of("CodeSnippetGeneratorLibrary", "CodeSnippetGeneratorOfficialSolution", "CodeSnippetGeneratorConfiguration") + // ## Week 6 + // Arguments.of("SubstringsBetweenLibrary", "SubstringsBetweenOfficialSolution", "SubstringsBetweenConfiguration") + // ## Week 7 + // Arguments.of("ReverseLibrary", "ReverseOfficialSolution", "ReverseConfiguration") + // ## Week 8 + // Arguments.of("ZigZagLibrary", "ZigZagOfficialSolution", "ZigZagConfiguration") ); } } diff --git a/andy/src/test/resources/grader/fixtures/Config/BalancingArraysConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/BalancingArraysConfiguration.java new file mode 100644 index 000000000..5ac5bb148 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/BalancingArraysConfiguration.java @@ -0,0 +1,96 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.1f); + put("meta", 0.8f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.Splitting"); + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withLineReplacement(1, "finds bug where true is returned if input has odd length", 23, 24, + """ + if (nums == null || nums.length == 0) + return false; + + if (nums.length % 2 == 1) + return true; + """), + MetaTest.withLineReplacement(1, "finds bug where false is returned if input has odd length", 23, 24, + """ + if (nums == null || nums.length == 0) + return false; + + if (nums.length % 2 == 1) + return false; + """), + MetaTest.withStringReplacement(1, "finds bug where null check is wrong", + """ + if (nums == null || nums.length <= 1) + return false; + """, + """ + if (nums == null) return true; + + if (nums.length <= 1) + return false; + """), + MetaTest.withLineReplacement(2, "finds bug where method returns where sum is even", 28, 34, + "return (sum % 2 == 0);"), + MetaTest.withStringReplacement(1, "finds bug where only one loop iteration is performed", + "half > 0;", + "i < 1;"), + MetaTest.withLineReplacement(2, "finds bug where the split always happens in the middle", 25, 35, + """ + return sum(nums, 0, nums.length / 2) == sum(nums, nums.length / 2, nums.length) + || sum(nums, 0, nums.length / 2 + 1) == sum(nums, nums.length / 2 + 1, nums.length); + } + + private static int sum(int[] nums, int lowerBound, int upperBound) { + int result = 0; + for (int i = lowerBound; i < upperBound; i++) + result += nums[i]; + return result; + } + """), + MetaTest.withLineReplacement(1, "finds bug where method only works for input of length 2", + 23, 34, + "return (nums != null && nums.length == 2 && nums[0] == nums[1]);"), + MetaTest.withLineReplacement(1, "finds bug where wrong result is returned for input of length 1", 23, 24, + """ + if (nums == null || nums.length == 0) + return false; + + if (nums.length == 1) + return true; + """), + MetaTest.withLineReplacement(1, "finds bug where wrong result is returned when input is empty", + 23, 24, + """ + if (nums == null || nums.length == 1) + return false; + + if (nums.length == 0) + return true; + """) + ); + } +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Config/CodeSnippetGeneratorConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/CodeSnippetGeneratorConfiguration.java new file mode 100644 index 000000000..cf63d2624 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/CodeSnippetGeneratorConfiguration.java @@ -0,0 +1,58 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; +import nl.tudelft.cse1110.andy.execution.mode.Mode; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + @Override + public Mode mode() { + return Mode.GRADING; + } + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.15f); + put("meta", 0.75f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.CodeSnippetUtils"); + } + + + @Override + public List metaTests() { + return List.of( + MetaTest.withStringReplacement("finds bug where method does not work if line number is near beginning of file", + "Math.max(0, lineNumberZeroIndexed - SURROUNDING_LINES)", + "lineNumberZeroIndexed - SURROUNDING_LINES"), + MetaTest.withStringReplacement("finds off by one error at end of file", + "Math.min(lines.size() - 1, lineNumberZeroIndexed + SURROUNDING_LINES)", + "Math.min(lines.size(), lineNumberZeroIndexed + SURROUNDING_LINES)"), + MetaTest.withStringReplacement(2, "finds bug where method does not preserve relative indentation", + ".map(x -> x.isBlank() ? \"\" : x.substring(spacesToTrim))", + ".map(x -> x.isBlank() ? \"\" : x.substring(getNumberOfLeadingSpaces(x)))"), + MetaTest.withStringReplacement(3, "finds bug where method does not trim indentation correctly with shorter blank/whitespace lines in snippet", + """ + .filter(s -> !s.isBlank()) + .mapToInt(CodeSnippetUtils::getNumberOfLeadingSpaces) + """, + """ + .mapToInt(s -> s.isBlank() ? s.length() : getNumberOfLeadingSpaces(s)) + """), + MetaTest.withStringReplacement("finds bug where arrows are placed on all lines", + "String s = i == relativeLineNumber ?", + "String s = true ?") + ); + } +} diff --git a/andy/src/test/resources/grader/fixtures/Config/CountingClumpsConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/CountingClumpsConfiguration.java new file mode 100644 index 000000000..147494e25 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/CountingClumpsConfiguration.java @@ -0,0 +1,109 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.1f); + put("meta", 0.8f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.Clumps"); + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withLineReplacement("finds bug where method always finds clumps", 23, 36, + "return nums.length / 2;"), + MetaTest.withLineReplacement("finds bug where method always returns zero", 20, 36, + "return 0;"), + MetaTest.withLineReplacement("finds bug where method checks in pairs", 24, 35, + """ + for (int i = 0; i < nums.length; i+=2) { + if (i + 1 < nums.length && nums[i] == nums[i+1]) { + count += 1; + } + } + """), + MetaTest.withLineReplacement("finds bug where method does not support more than two per clump", 25, 35, + """ + for (int i = 1; i < nums.length; i++) { + if (nums[i] == prev) { + count += 1; + } + if (nums[i] != prev) { + prev = nums[i]; + } + } + """), + MetaTest.withStringReplacement("finds bug where method does not support multiple clumps", + "return count;", + "return count > 1 ? 1 : count;"), + MetaTest.withStringReplacement("finds bug where check for empty is missing", + "if (nums == null || nums.length == 0)", + "if (nums == null)"), + MetaTest.withStringReplacement("finds bug where check for null is missing", + "if (nums == null || nums.length == 0)", + "if (nums.length == 0)"), + MetaTest.withLineReplacement("finds bug where method only checks the first two elements", 20, 36, + """ + if (nums != null && nums.length >= 2 && nums[0] == nums[1]) { + return 1; + } + else { + return 0; + } + """), + MetaTest.withLineReplacement("finds bug where method only checks the last two elements", 20, 36, + """ + if (nums != null && nums.length >= 2 && nums[nums.length - 2] == nums[nums.length - 1]) { + return 1; + } + else { + return 0; + } + """), + MetaTest.withLineReplacement("finds bug where elements are skipped after clumps", 34, 34, + """ + } + if (inClump) { + i++; + } + """), + MetaTest.withLineReplacement("finds bug where first element is skipped", 20, 26, + """ + if (nums == null || nums.length <= 1) { + return 0; + } + int count = 0; + int prev = nums[1]; + boolean inClump = false; + for (int i = 2; i < nums.length; i++) { + """), + MetaTest.withStringReplacement("finds bug where last element is skipped", + "i < nums.length;", + "i < nums.length - 1;"), + MetaTest.withStringReplacement("finds bug where wrong result is returned for one element", + "int count = 0;", + """ + if (nums.length == 1) { + return 1; + } + int count = 0; + """) + ); + } +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Config/GetCheapestPriceConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/GetCheapestPriceConfiguration.java new file mode 100644 index 000000000..2a47988d1 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/GetCheapestPriceConfiguration.java @@ -0,0 +1,63 @@ +package delft; + +import nl.tudelft.cse1110.andy.codechecker.checks.*; +import nl.tudelft.cse1110.andy.codechecker.engine.CheckScript; +import nl.tudelft.cse1110.andy.codechecker.engine.SingleCheck; +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; +import nl.tudelft.cse1110.andy.execution.mode.Mode; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Mode mode() { + return Mode.PRACTICE; + } + + @Override + public List classesUnderTest() { + return List.of("delft.SeatFinder"); + } + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.2f); + put("mutation", 0.3f); + put("meta", 0.5f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List metaTests() { + return List.of( + MetaTest.insertAt("finds bug where no exception is thrown if `prices` is null", + 32, "if (prices == null) return -1;"), + MetaTest.insertAt("finds bug where no exception is thrown if `taken` is null", + 32, "if (taken == null) return -1;"), + MetaTest.insertAt("finds bug where no exception is thrown if `prices` and `taken` have different lengths", + 35, "if (prices.length != taken.length) return 0;"), + MetaTest.insertAt("finds bug where no exception is thrown for numberOfSeats == 0", + 38, "if (numberOfSeats == 0) return 0;"), + MetaTest.insertAt("finds bug where no exception is thrown for numberOfSeats < 0", + 38, "if (numberOfSeats < 0) return 0;"), + MetaTest.withLineReplacement("finds bug where seats array is not sorted", + 42, 46, "int[] seats = IntStream.range(0, prices.length).toArray();"), + MetaTest.withLineReplacement("finds bug where loop continues when numberOfTickets == numberOfSeats", + 57, 57, "continue;"), + MetaTest.withStringReplacement("finds bug where seats are never considered taken", + "!taken[seat]", "true"), + MetaTest.withStringReplacement("finds bug where discount is only applied when total price equals 100", + "totalPrice > 100.00", + "totalPrice == 100.00"), + MetaTest.withLineReplacement("finds bug where discount is never applied", + 62, 62, "") + ); + } +} + diff --git a/andy/src/test/resources/grader/fixtures/Config/IntersectionConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/IntersectionConfiguration.java new file mode 100644 index 000000000..dbca005a6 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/IntersectionConfiguration.java @@ -0,0 +1,73 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.1f); + put("meta", 0.8f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.ListUtils"); + } + + @Override + public int numberOfMutationsToConsider() { + return 2; + } + + @Override + public List metaTests() { + return List.of( + MetaTest.insertAt("finds bug where method returns empty list when list1 is null", 29, + "if (list1 == null) return result;" + ), + MetaTest.insertAt("finds bug where method returns empty list when list2 is null", 29, + "if (list2 == null) return result;" + ), + MetaTest.withStringReplacement("finds bug where method returns wrong elements", + "if (hashSet.contains(e))", + "if (!hashSet.contains(e))" + ), + MetaTest.insertAt("finds bug where method returns null when there are no elements in the intersection", 42, + "if (result.isEmpty()) return null;" + ), + MetaTest.withStringReplacement("finds bug where method does not work correctly when one of the lists is empty", + "if (hashSet.contains(e))", + "if (hashSet.isEmpty() || hashSet.contains(e))" + ), + MetaTest.withStringReplacement("finds bug where method adds repeated elements to result multiple times", + "hashSet.remove(e);", + "" + ), + MetaTest.withStringReplacement("finds bug where method always returns smaller", + "return result;", + "return new ArrayList<>(smaller);"), + MetaTest.insertAt("finds bug where method finds only the first element in the intersection", 40, + "break;" + ), + MetaTest.withLineReplacement("finds bug where method compares only elements at the same index", 29, 41, + """ + for (int i = 0; i < list1.size() && i < list2.size(); i++) { + if (list1.get(i).equals(list2.get(i))) { + result.add(list1.get(i)); + } + } + """ + ) + ); + } + +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Config/IsEqualCollectionConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/IsEqualCollectionConfiguration.java new file mode 100644 index 000000000..7fe737689 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/IsEqualCollectionConfiguration.java @@ -0,0 +1,73 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.15f); + put("meta", 0.75f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.CollectionUtils", "delft.CollectionUtils.CardinalityHelper"); + } + + @Override + public int numberOfMutationsToConsider() { + return 9; + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withLineReplacement("finds bug where true is returned when sizes are not equal", 115, 115, + "return true;" + ), + MetaTest.withLineReplacement("finds bug where count per element is always 1", + 91, 91, + "" + ), + MetaTest.insertAt("finds bug where only the first element is considered when building the cardinality map", 93, + "break;" + ), + MetaTest.withLineReplacement("finds bug where method does not compare list and cardinality sizes (returns true when a is empty)", 114, 120, + "final CardinalityHelper helper = new CardinalityHelper<>(a, b);" + ), + MetaTest.withLineReplacement("finds bug where NullPointerException is thrown when list A contains elements not in list B", 62, 65, + "return count.intValue();" + ), + MetaTest.withStringReplacement("finds bug where method only checks whether elements exist but not their cardinalities", + "return count.intValue();", + "return 1;" + ), + MetaTest.withLineReplacement("finds bug where method does not check cardinalities but only whether the lists are equal", 117, 125, + """ + ArrayList listA = new ArrayList<>(a); + ArrayList listB = new ArrayList<>(b); + for(int i = 0; i < listA.size(); i++){ + if(!listA.get(i).equals(listB.get(i))){ + return false; + } + } + """ + ), + MetaTest.withStringReplacement("finds bug where method compares first list to itself", + "return getFreq(obj, cardinalityB);", + "return getFreq(obj, cardinalityA);" + ) + ); + } + +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Config/RepeatConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/RepeatConfiguration.java new file mode 100644 index 000000000..74ac34893 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/RepeatConfiguration.java @@ -0,0 +1,67 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.3f); + put("meta", 0.6f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.DelftStringUtilities"); + } + + @Override + public int numberOfMutationsToConsider() { + return 22; + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withStringReplacement("finds bug where method returns empty if str == null", + "return null;", + "return EMPTY;"), + MetaTest.withStringReplacement("finds bug where method returns str if repeat <= 0", + "return EMPTY;", + "return str;"), + MetaTest.withStringReplacement("finds bug where method returns empty if (repeat == 1 || inputLength == 0)", + "return str;", + "return EMPTY;"), + MetaTest.withLineReplacement("finds bug where method always repeats only the first character", + 57, 76, + "return repeat(str.charAt(0), repeat);"), + MetaTest.withStringReplacement("finds bug where method returns str if input length is 1", + "return repeat(str.charAt(0), repeat);", + "return str;"), + MetaTest.withLineReplacement("finds bug where ch0 and ch1 are swapped", + 66, 67, + """ + output2[i] = ch1; + output2[i + 1] = ch0;""" + ), + MetaTest.withStringReplacement("finds bug where string is repeated one time less", + "i < repeat", + "i < repeat - 1"), + MetaTest.insertAt("finds bug where method returns null if the string is empty", 47, + """ + if (str.isEmpty()) { + return null; + }""") + ); + } + +} + diff --git a/andy/src/test/resources/grader/fixtures/Config/ReplaceConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/ReplaceConfiguration.java new file mode 100644 index 000000000..cdabea293 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/ReplaceConfiguration.java @@ -0,0 +1,81 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.25f); + put("mutation", 0.25f); + put("meta", 0.5f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.DelftStringUtilities"); + } + + @Override + public int numberOfMutationsToConsider() { + return 16; + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withStringReplacement("finds bug where at most one replacement is made", + "--max == 0", + "true"), + MetaTest.withStringReplacement("finds bug where all occurrences are always replaced", + """ + if (--max == 0) { + break; + }""", + ""), + MetaTest.withLineReplacement("finds bug where input is returned unchanged", + 32, 57, + "return text;"), + MetaTest.insertAt("finds bug where case is always ignored", 35, + "ignoreCase = true;" + ), + MetaTest.insertAt("finds bug where case is never ignored", 35, + "ignoreCase = false;" + ), + MetaTest.withStringReplacement("finds bug where starting position is determined by end + replacement.length() + 1", + "start = end + replLength;", + "start = end + replacement.length() + 1;"), + MetaTest.insertAt("finds bug where check for empty searchString is broken", 32, + """ + if (isEmpty(searchString)) { + return replacement; + } + """), + MetaTest.insertAt("finds bug where check for empty text is broken", 32, + """ + if (isEmpty(text)) { + return replacement; + } + """), + MetaTest.insertAt("finds bug where a null replacement is considered the same as an empty replacement", 32, + """ + if (replacement == null) { + replacement = ""; + } + """), + MetaTest.insertAt("finds bug where max cannot be 0", 32, + """ + if (max == 0) { + max = 1; + } + """) + ); + } +} diff --git a/andy/src/test/resources/grader/fixtures/Config/ReverseConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/ReverseConfiguration.java new file mode 100644 index 000000000..3c8d7212f --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/ReverseConfiguration.java @@ -0,0 +1,92 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.1f); + put("meta", 0.8f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.ArrayUtils"); + } + + @Override + public int numberOfMutationsToConsider() { + return 6; + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withStringReplacement("finds bug where null check is missing", + """ + if (array == null) { + return; + } + """, + "" + ), + MetaTest.withStringReplacement("finds bug where method always starts at index 0", + "int i = startIndexInclusive < 0 ? 0 : startIndexInclusive;", + "int i = 0;" + ), + MetaTest.withStringReplacement("finds bug where negative start index is not promoted", + "int i = startIndexInclusive < 0 ? 0 : startIndexInclusive;", + "int i = startIndexInclusive;" + ), + MetaTest.withStringReplacement("finds bug where start index is considered exclusive", + "int i = startIndexInclusive < 0 ? 0 : startIndexInclusive;", + "int i = (startIndexInclusive + 1) < 0 ? 0 : (startIndexInclusive + 1);" + ), + MetaTest.withStringReplacement("finds bug where index 0 is skipped", + "int i = startIndexInclusive < 0 ? 0 : startIndexInclusive;", + "int i = startIndexInclusive < 1 ? 1 : startIndexInclusive;" + ), + MetaTest.withStringReplacement("finds bug where method always ends at last index", + "int j = Math.min(array.length, endIndexExclusive) - 1;", + "int j = array.length - 1;" + ), + MetaTest.withStringReplacement("finds bug where end index greater than the length of the array is not demoted", + "int j = Math.min(array.length, endIndexExclusive) - 1;", + "int j = endIndexExclusive - 1;" + ), + MetaTest.withStringReplacement("finds bug where end index is considered inclusive", + "int j = Math.min(array.length, endIndexExclusive) - 1;", + "int j = Math.min(array.length - 1, endIndexExclusive);" + ), + MetaTest.withStringReplacement("finds bug where last index of array is skipped", + "int j = Math.min(array.length, endIndexExclusive) - 1;", + "int j = Math.min(array.length - 1, endIndexExclusive) - 1;" + ), + MetaTest.withLineReplacement("finds bug where start and end indices are swapped if start is greater", + 30, 31, + """ + int s = startIndexInclusive; + int e = endIndexExclusive; + if (s > e) { + s = endIndexExclusive; + e = startIndexInclusive; + } + int i = s < 0 ? 0 : s; + int j = Math.min(array.length, e) - 1;""" + ), + MetaTest.insertAt("finds bug where exception is thrown if range has one element", 32, + "if (i == j) throw new RuntimeException(\"killed the mutant\");" + ) + ); + } +} diff --git a/andy/src/test/resources/grader/fixtures/Config/SubstringsBetweenConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/SubstringsBetweenConfiguration.java new file mode 100644 index 000000000..e9521bba8 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/SubstringsBetweenConfiguration.java @@ -0,0 +1,110 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.4f); + put("meta", 0.5f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.DelftStringUtilities"); + } + + @Override + public int numberOfMutationsToConsider() { + return 16; + } + + + @Override + public List metaTests() { + return List.of( + MetaTest.withStringReplacement("finds bug where check for null str is missing", + "if (str == null || isEmpty(open) || isEmpty(close)) {", + "if (isEmpty(open) || isEmpty(close)) {"), + + MetaTest.withStringReplacement("finds bug where check for null open is missing", + "if (str == null || isEmpty(open) || isEmpty(close)) {", + "if (str == null || open.isEmpty() || isEmpty(close)) {"), + + MetaTest.withStringReplacement("finds bug where check for empty open is missing", + "if (str == null || isEmpty(open) || isEmpty(close)) {", + "if (str == null || open == null || isEmpty(close)) {"), + + MetaTest.withStringReplacement("finds bug where check for null close is missing", + "if (str == null || isEmpty(open) || isEmpty(close)) {", + "if (str == null || isEmpty(open) || close.isEmpty()) {"), + + MetaTest.withStringReplacement("finds bug where check for empty close is missing", + "if (str == null || isEmpty(open) || isEmpty(close)) {", + "if (str == null || isEmpty(open) || close == null) {"), + + MetaTest.withLineReplacement("finds bug where null is returned for empty input", 40, 40, + "return null;"), + + MetaTest.withStringReplacement("finds bug where first character is skipped", + "int pos = 0;", + "int pos = 1;"), + + MetaTest.withLineReplacement("finds bug where empty list is returned when there are no matches", + 59, 61, ""), + + MetaTest.withLineReplacement("finds bug where only the first match is returned", 62, 62, + "return new String[] { list.get(0) };"), + + MetaTest.withStringReplacement("finds bug where method skips a character after finding a match", + "pos = end + closeLen;", + "pos = end + closeLen + 1;"), + + MetaTest.withLineReplacement("finds bug where method does not work with open string longer than 1 character", 34, 37, + """ + public static String[] substringsBetween(final String str, String open, final String close) { + if (str == null || isEmpty(open) || isEmpty(close)) { + return null; + } + open = open.substring(1, open.length()); + """), + + MetaTest.withLineReplacement("finds bug where method does not work with close string longer than 1 character", 34, 37, + """ + public static String[] substringsBetween(final String str, final String open, String close) { + if (str == null || isEmpty(open) || isEmpty(close)) { + return null; + } + close = close.substring(0, open.length() - 1); + """), + MetaTest.withLineReplacement("finds bug where matching happens greedily instead of lazily", + 42, 58, + """ + // Re-implementation of the method using a regular expression. + // With (.*?), this mutant does the same as the original method. + // With (.*), the matching happens greedily. + // For example, if str=="ababa", open=="a", close=="a", + // lazy matching would return substring "b", + // and greedy matching would return substring "bab". + String regex = Pattern.quote(open) + "(.*)" + Pattern.quote(close); + Matcher matcher = Pattern.compile(regex).matcher(str); + + List list = new ArrayList<>(); + + while (matcher.find()) { + list.add(matcher.group(1)); + }""") + ); + } + +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Config/SwapCaseConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/SwapCaseConfiguration.java new file mode 100644 index 000000000..148a57f60 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/SwapCaseConfiguration.java @@ -0,0 +1,91 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.1f); + put("meta", 0.8f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.DelftWordUtilities"); + } + + @Override + public int numberOfMutationsToConsider() { + return 10; + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withStringReplacement("finds bug where last character is not considered", + "buffer.length", + "buffer.length - 1"), + MetaTest.withStringReplacement("finds bug where first character is not considered", + "i = 0", + "i = 1"), + MetaTest.withStringReplacement("finds bug where first if-statement is negated", + "DelftWordUtilities.isEmpty(str)", + "!DelftWordUtilities.isEmpty(str)"), + MetaTest.withStringReplacement("finds bug where second if-statement is negated", + "Character.isUpperCase(ch)", + "!Character.isUpperCase(ch)"), + MetaTest.withStringReplacement("finds bug where third if-statement is negated", + "Character.isLowerCase(ch)", + "!Character.isLowerCase(ch)"), + MetaTest.withStringReplacement("finds bug where case is only swapped for uppercase characters", + """ + else if (Character.isLowerCase(ch)) { + buffer[i] = Character.toUpperCase(ch); + }""", + ""), + MetaTest.withStringReplacement("finds bug where case is only swapped for lowercase characters", + """ + if (Character.isUpperCase(ch)) { + buffer[i] = Character.toLowerCase(ch); + } else if (Character.isLowerCase(ch)) { + buffer[i] = Character.toUpperCase(ch); + } + """, + """ + if (Character.isLowerCase(ch)) { + buffer[i] = Character.toUpperCase(ch); + } + """), + MetaTest.withStringReplacement("finds bug where function does not swap uppercase characters", + """ + if (Character.isUpperCase(ch)) { + buffer[i] = Character.toLowerCase(ch); + """, + """ + if (Character.isUpperCase(ch)) { + buffer[i] = Character.toUpperCase(ch); + """), + MetaTest.withStringReplacement("finds bug where function does not swap lowercase characters", + """ + else if (Character.isLowerCase(ch)) { + buffer[i] = Character.toUpperCase(ch); + } + """, + """ + else if (Character.isLowerCase(ch)) { + buffer[i] = Character.toLowerCase(ch); + } + """) + ); + } + +} diff --git a/andy/src/test/resources/grader/fixtures/Config/ToCamelCaseConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/ToCamelCaseConfiguration.java new file mode 100644 index 000000000..ae1b51e1b --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/ToCamelCaseConfiguration.java @@ -0,0 +1,74 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.2f); + put("mutation", 0.3f); + put("meta", 0.5f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.DelftCaseUtilities"); + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withStringReplacement("finds bug where method does not check for null", + "if (str == null || str.isEmpty())", + "if (str.isEmpty())"), + + MetaTest.withLineReplacement("finds bug where null is returned if input is empty", + 36, 36, + "return null;"), + + MetaTest.withStringReplacement("finds bug where method does not split by space unless explicitly specified", + "delimiterHashSet.add(Character.codePointAt(new char[]{' '}, 0));", + ""), + + MetaTest.withLineReplacement("finds bug where method only splits by space", 87, 89, + ""), + + MetaTest.withLineReplacement("finds bug where method does not split by space if other delimiters are specified", + 83, 86, + """ + if (delimiters == null || delimiters.length == 0) { + delimiterHashSet.add(Character.codePointAt(new char[]{' '}, 0)); + return delimiterHashSet; + } + """), + + MetaTest.withStringReplacement("finds bug where delimiters parameter cannot be null", + "if (delimiters == null || delimiters.length == 0) {", + "if (delimiters.length == 0) {"), + + MetaTest.withLineReplacement("finds bug where only the first delimiter is used", + 87, 89, + "delimiterHashSet.add(Character.codePointAt(delimiters, 0));"), + + MetaTest.withStringReplacement("finds bug where input is not converted to lowercase", + "str = str.toLowerCase();", + ""), + + MetaTest.withLineReplacement("finds bug where first letter is always capitalised", 43, 46, + "boolean capitalizeNext = true;"), + + MetaTest.withLineReplacement("finds bug where first letter is never capitalised", 43, 46, + "boolean capitalizeNext = false;") + ); + } + +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Config/ZigZagConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/ZigZagConfiguration.java new file mode 100644 index 000000000..121b9c4d7 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/ZigZagConfiguration.java @@ -0,0 +1,49 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.4f); + put("mutation", 0.4f); + put("meta", 0.2f); + put("codechecks", 0.0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.ZigZag"); + } + + @Override + public int numberOfMutationsToConsider() { + return 24; + } + + @Override + public List metaTests() { + return List.of( + // MetaTest.withStringReplacement("single row", + // "return s;", + // "throw new RuntimeException(\"killed mutant\");"), + // MetaTest.withLineReplacement("string length equals number of rows", 17, 17, + // """ + // if(s.length() == numRows) + // throw new RuntimeException("killed the mutant"); + // """), + MetaTest.withStringReplacement("finds bug where goingDown starts from `true`", + "boolean goingDown = false;", + "boolean goingDown = true;") + ); + } + +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Library/BalancingArraysLibrary.java b/andy/src/test/resources/grader/fixtures/Library/BalancingArraysLibrary.java new file mode 100644 index 000000000..b7a99d37d --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/BalancingArraysLibrary.java @@ -0,0 +1,36 @@ +package delft; + +class Splitting { + + private Splitting() { + // Empty Constructor + } + + /** + * Given a non-empty array of non-negative numbers, return true if there is a place to split the array + * so that the sum of the numbers on one side is equal to the sum of the numbers + * on the other side. + * + * @param nums + * the array containing the input numbers. The array must be non-null + * and with len > 1; the program returns false in case these + * pre-conditions are violated. The input array sum must not also cause + * integer overflow, in which cause no certain behaviour should be + * expected. + * @return true if they can be split in the way defined above + */ + public static boolean canBalance(int[] nums) { + if (nums == null || nums.length <= 1) + return false; + int sum = 0; + for (int num : nums) + sum += num; + if (sum % 2 == 1) + return false; + int half = sum / 2; + for (int i = 0; half > 0; i++) { + half -= nums[i]; + } + return (half == 0); + } +} diff --git a/andy/src/test/resources/grader/fixtures/Library/CodeSnippetGeneratorLibrary.java b/andy/src/test/resources/grader/fixtures/Library/CodeSnippetGeneratorLibrary.java new file mode 100644 index 000000000..38193dcee --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/CodeSnippetGeneratorLibrary.java @@ -0,0 +1,59 @@ +package delft; + +import java.util.List; + +class CodeSnippetUtils { + + private CodeSnippetUtils() {} + + private static final int SURROUNDING_LINES = 2; + private static final int ARROW_SIZE = 4; + + /** + * Generate a snippet of code from the given code, starting 2 lines before the given line + * and ending 2 lines after the given line (inclusive), with an arrow pointing towards the line. + * + * @param lines The lines of the source code. + * @param lineNumber The line number to point to (1-indexed). + * @return A snippet of code with an arrow pointing to the given line. + */ + public static String generateCodeSnippet(List lines, int lineNumber) { + final int lineNumberZeroIndexed = lineNumber - 1; + + // extract relevant lines + int start = Math.max(0, lineNumberZeroIndexed - SURROUNDING_LINES); + int end = Math.min(lines.size() - 1, lineNumberZeroIndexed + SURROUNDING_LINES); + List linesToShow = lines.subList(start, end + 1); + + // trim spaces at the beginning of the extracted lines + int spacesToTrim = linesToShow.stream() + .filter(s -> !s.isBlank()) + .mapToInt(CodeSnippetUtils::getNumberOfLeadingSpaces) + .min() + .orElse(0); + String[] trimmedLines = linesToShow.stream() + .map(x -> x.isBlank() ? "" : x.substring(spacesToTrim)) + .toArray(String[]::new); + + // add arrow or spaces + // (prepend "--> " to the line, and " " to all other lines) + int relativeLineNumber = lineNumberZeroIndexed - start; + for (int i = 0; i < trimmedLines.length; i++) { + if (trimmedLines[i].isBlank()) continue; + String s = i == relativeLineNumber ? + "-".repeat(ARROW_SIZE - 2) + "> " : + " ".repeat(ARROW_SIZE); + trimmedLines[i] = s + trimmedLines[i]; + } + + return String.join("\n", trimmedLines); + } + + private static int getNumberOfLeadingSpaces(String line) { + int count = 0; + while (line.charAt(count) == ' ') { + count++; + } + return count; + } +} diff --git a/andy/src/test/resources/grader/fixtures/Library/CountingClumpsLibrary.java b/andy/src/test/resources/grader/fixtures/Library/CountingClumpsLibrary.java new file mode 100644 index 000000000..e2e2e54fd --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/CountingClumpsLibrary.java @@ -0,0 +1,38 @@ +package delft; + +class Clumps { + + private Clumps() { + // Empty constructor + } + + /** + * Counts the number of "clumps" that are in the array. A clump is a sequence of + * the same element with a length of at least 2. + * + * @param nums + * the array to count the clumps of. The array must be non-null and + * len > 0; the program returns 0 in case any pre-condition is + * violated. + * @return the number of clumps in the array. + */ + public static int countClumps(int[] nums) { + if (nums == null || nums.length == 0) { + return 0; + } + int count = 0; + int prev = nums[0]; + boolean inClump = false; + for (int i = 1; i < nums.length; i++) { + if (nums[i] == prev && !inClump) { + inClump = true; + count += 1; + } + if (nums[i] != prev) { + prev = nums[i]; + inClump = false; + } + } + return count; + } +} diff --git a/andy/src/test/resources/grader/fixtures/Library/GetCheapestPriceLibrary.java b/andy/src/test/resources/grader/fixtures/Library/GetCheapestPriceLibrary.java new file mode 100644 index 000000000..42fb234ae --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/GetCheapestPriceLibrary.java @@ -0,0 +1,66 @@ +package delft; + +import java.util.Comparator; +import java.util.stream.IntStream; + +class SeatFinder { + + private SeatFinder() { + // Empty constructor + } + + /** + * Finds the cheapest seats and returns the total price for those seats. + * + * If there are fewer seats available than requested, + * only the prices for the ones that are available are counted. + * + * When the total price of the seats is larger than 100, + * a discount of 5 euros is applied. + * + * The prices and taken arrays cannot be null and need to have the same size. + * The number of seats requested should be at least 1. + * + * @param prices an array indicating the price of each seat + * @param taken an array indicating whether a seat has been taken already + * @param numberOfSeats the number of seats requested + * @throws IllegalArgumentException when any of the arguments do not + * adhere to their requirements + * @return total price for the seats + */ + public static double getCheapestPrice(double[] prices, boolean[] taken, int numberOfSeats) { + if (prices == null || taken == null) { + throw new IllegalArgumentException("prices and taken arrays cannot be null"); + } + if (prices.length != taken.length) { + throw new IllegalArgumentException("prices and taken arrays do not have the same length"); + } + if (numberOfSeats <= 0) { + throw new IllegalArgumentException("The number of seats has to be at least 1"); + } + + int[] seats = IntStream.range(0, prices.length) + .boxed() + .sorted(Comparator.comparingDouble(s -> prices[s])) + .mapToInt(Integer::intValue) + .toArray(); + + int numberOfTickets = 0; + double totalPrice = 0; + for (int i = 0; i < seats.length; i++) { + int seat = seats[i]; + if (!taken[seat]) { + totalPrice += prices[seat]; + numberOfTickets++; + } + if (numberOfTickets == numberOfSeats) { + break; + } + } + + if (totalPrice > 100.00) { + return totalPrice - 5.00; + } + return totalPrice; + } +} diff --git a/andy/src/test/resources/grader/fixtures/Library/IntersectionLibrary.java b/andy/src/test/resources/grader/fixtures/Library/IntersectionLibrary.java new file mode 100644 index 000000000..b0d99c49e --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/IntersectionLibrary.java @@ -0,0 +1,44 @@ +package delft; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; + +class ListUtils { + + /** ListUtils should not normally be instantiated. */ + private ListUtils() { + } + + /** + * Returns a new list containing all elements that are contained in both given + * lists. + * + * @param + * the element type + * @param list1 + * the first list + * @param list2 + * the second list + * @return the intersection of those two lists + * @throws NullPointerException + * if either list is null + */ + public static List intersection(final List list1, final List list2) { + final List result = new ArrayList<>(); + List smaller = list1; + List larger = list2; + if (list1.size() > list2.size()) { + smaller = list2; + larger = list1; + } + final HashSet hashSet = new HashSet<>(smaller); + for (final E e : larger) { + if (hashSet.contains(e)) { + result.add(e); + hashSet.remove(e); + } + } + return result; + } +} diff --git a/andy/src/test/resources/grader/fixtures/Library/IsEqualCollectionLibrary.java b/andy/src/test/resources/grader/fixtures/Library/IsEqualCollectionLibrary.java new file mode 100644 index 000000000..6769c173a --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/IsEqualCollectionLibrary.java @@ -0,0 +1,128 @@ +package delft; + +import java.util.*; + +class CollectionUtils { + + /** CollectionUtils should not normally be instantiated. */ + private CollectionUtils() { + } + + /** + * Helper class to easily access cardinality properties of two collections. + * + * @param + * the element type + */ + private static class CardinalityHelper { + + /** Contains the cardinality for each object in collection A. */ + final Map cardinalityA; + + /** Contains the cardinality for each object in collection B. */ + final Map cardinalityB; + + /** + * Create a new CardinalityHelper for two collections. + * + * @param a + * the first collection + * @param b + * the second collection + */ + public CardinalityHelper(final Iterable a, final Iterable b) { + cardinalityA = CollectionUtils.getCardinalityMap(a); + cardinalityB = CollectionUtils.getCardinalityMap(b); + } + + /** + * Returns the frequency of this object in collection A. + * + * @param obj + * the object + * @return the frequency of the object in collection A + */ + public int freqA(final Object obj) { + return getFreq(obj, cardinalityA); + } + + /** + * Returns the frequency of this object in collection B. + * + * @param obj + * the object + * @return the frequency of the object in collection B + */ + public int freqB(final Object obj) { + return getFreq(obj, cardinalityB); + } + + private int getFreq(final Object obj, final Map freqMap) { + final Integer count = freqMap.get(obj); + if (count != null) { + return count.intValue(); + } + return 0; + } + } + + /** + * Returns a {@link Map} mapping each unique element in the given + * {@link Collection} to an {@link Integer} representing the number of + * occurrences of that element in the {@link Collection}. + * + *

+ * Only those elements present in the collection will appear as keys in the map. + * + * @param + * the type of object in the returned {@link Map}. This is a super + * type of <I>. + * @param coll + * the collection to get the cardinality map for, must not be null + * @return the populated cardinality map + */ + private static Map getCardinalityMap(final Iterable coll) { + final Map count = new HashMap<>(); + for (final O obj : coll) { + final Integer c = count.get(obj); + if (c == null) { + count.put(obj, Integer.valueOf(1)); + } else { + count.put(obj, Integer.valueOf(c.intValue() + 1)); + } + } + return count; + } + + /** + * Returns {@code true} iff the given {@link Collection}s contain exactly the + * same elements with exactly the same cardinalities. + * + *

+ * That is, iff the cardinality of e in a is equal to the + * cardinality of e in b, for each element e in a or + * b. + * + * @param a + * the first collection, must not be null + * @param b + * the second collection, must not be null + * @return true iff the collections contain the same elements with + * the same cardinalities. + */ + public static boolean isEqualCollection(final Collection a, final Collection b) { + if (a.size() != b.size()) { + return false; + } + final CardinalityHelper helper = new CardinalityHelper<>(a, b); + if (helper.cardinalityA.size() != helper.cardinalityB.size()) { + return false; + } + for (final Object obj : helper.cardinalityA.keySet()) { + if (helper.freqA(obj) != helper.freqB(obj)) { + return false; + } + } + return true; + } +} diff --git a/andy/src/test/resources/grader/fixtures/Library/RepeatLibrary.java b/andy/src/test/resources/grader/fixtures/Library/RepeatLibrary.java new file mode 100644 index 000000000..593577e23 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/RepeatLibrary.java @@ -0,0 +1,78 @@ +package delft; + +class DelftStringUtilities { + + private DelftStringUtilities() { + // Override default constructor, to prevent it from getting considered in the + // coverage report. + } + + /** + * The empty String {@code ""}. + * + * @since 2.0 + */ + public static final String EMPTY = ""; + + /** + * Returns padding using the specified delimiter repeated to a given length. + * + *

+ * Note: this method does not support padding with + * Unicode + * Supplementary Characters as they require a pair of {@code char}s to be represented. If you are needing to support + * full I18N of your applications consider using {@link #repeat(String, int)} instead. + * + * @param ch character to repeat + * @param repeat number of times to repeat char, should be positive + * @return String with repeated character + * @see #repeat(String, int) + */ + private static String repeat(final char ch, final int repeat) { + final char[] buf = new char[repeat]; + for (int i = repeat - 1; i >= 0; i--) { + buf[i] = ch; + } + return new String(buf); + } + + /** + * Repeat a String {@code repeat} times to form a new String. + * + * @param str the String to repeat, may be null + * @param repeat number of times to repeat str, negative treated as zero + * @return a new String consisting of the original String repeated, {@code null} if null String input + */ + public static String repeat(final String str, final int repeat) { + if (str == null) { + return null; + } + if (repeat <= 0) { + return EMPTY; + } + final int inputLength = str.length(); + if (repeat == 1 || inputLength == 0) { + return str; + } + final int outputLength = inputLength * repeat; + switch (inputLength) { + case 1: + return repeat(str.charAt(0), repeat); + case 2: + final char ch0 = str.charAt(0); + final char ch1 = str.charAt(1); + final char[] output2 = new char[outputLength]; + for (int i = repeat * 2 - 2; i >= 0; i--, i--) { + output2[i] = ch0; + output2[i + 1] = ch1; + } + return new String(output2); + default: + final StringBuilder buf = new StringBuilder(outputLength); + for (int i = 0; i < repeat; i++) { + buf.append(str); + } + return buf.toString(); + } + } +} diff --git a/andy/src/test/resources/grader/fixtures/Library/ReplaceLibrary.java b/andy/src/test/resources/grader/fixtures/Library/ReplaceLibrary.java new file mode 100644 index 000000000..47b7b50eb --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/ReplaceLibrary.java @@ -0,0 +1,145 @@ +package delft; + +class DelftStringUtilities extends DelftStringUtilitiesExtra { + + private DelftStringUtilities() { + // Empty constructor + } + + /** + * Replaces a String with another String inside a larger String, for the first + * {@code max} values of the search String, case sensitively/insensisitively + * based on {@code ignoreCase} value. + * + *

+ * A {@code null} reference passed to this method is a no-op. + * + * @param text + * text to search and replace in, may be null + * @param searchString + * the String to search for (case insensitive), may be null + * @param replacement + * the String to replace it with, may be null + * @param max + * maximum number of values to replace, or {@code -1} if no maximum + * @param ignoreCase + * if true replace is case insensitive, otherwise case sensitive + * @return the text with any replacements processed, {@code null} if null String + * input + */ + public static String replace(final String text, String searchString, String replacement, int max, + boolean ignoreCase) { + if (isEmpty(text) || isEmpty(searchString) || replacement == null || max == 0) { + return text; + } + if (ignoreCase) { + searchString = searchString.toLowerCase(); + } + int start = 0; + int end = ignoreCase ? indexOfIgnoreCase(text, searchString, start) : indexOf(text, searchString, start); + if (end == INDEX_NOT_FOUND) { + return text; + } + final int replLength = searchString.length(); + int increase = replacement.length() - replLength; + increase = increase < 0 ? 0 : increase; + increase *= max < 0 ? 16 : max > 64 ? 64 : max; + final StringBuilder buf = new StringBuilder(text.length() + increase); + while (end != INDEX_NOT_FOUND) { + buf.append(text, start, end).append(replacement); + start = end + replLength; + if (--max == 0) { + break; + } + end = ignoreCase ? indexOfIgnoreCase(text, searchString, start) : indexOf(text, searchString, start); + } + buf.append(text, start, text.length()); + return buf.toString(); + } +} + +class DelftStringUtilitiesExtra { + + public static final int INDEX_NOT_FOUND = -1; + + public static boolean isEmpty(final CharSequence cs) { + return cs == null || cs.length() == 0; + } + + public static int indexOfIgnoreCase(final CharSequence str, final CharSequence searchStr) { + return indexOfIgnoreCase(str, searchStr, 0); + } + + public static int indexOfIgnoreCase(final CharSequence str, final CharSequence searchStr, int startPos) { + if (str == null || searchStr == null) { + return INDEX_NOT_FOUND; + } + if (startPos < 0) { + startPos = 0; + } + final int endLimit = str.length() - searchStr.length() + 1; + if (startPos > endLimit) { + return INDEX_NOT_FOUND; + } + if (searchStr.length() == 0) { + return startPos; + } + for (int i = startPos; i < endLimit; i++) { + if (regionMatches(str, true, i, searchStr, 0, searchStr.length())) { + return i; + } + } + return INDEX_NOT_FOUND; + } + + static boolean regionMatches(final CharSequence cs, final boolean ignoreCase, final int thisStart, + final CharSequence substring, final int start, final int length) { + if (cs instanceof String && substring instanceof String) { + return ((String) cs).regionMatches(ignoreCase, thisStart, (String) substring, start, length); + } + int index1 = thisStart; + int index2 = start; + int tmpLen = length; + // Extract these first so we detect NPEs the same as the java.lang.String + // version + final int srcLen = cs.length() - thisStart; + final int otherLen = substring.length() - start; + // Check for invalid parameters + if (thisStart < 0 || start < 0 || length < 0) { + return false; + } + // Check that the regions are long enough + if (srcLen < length || otherLen < length) { + return false; + } + while (tmpLen-- > 0) { + final char c1 = cs.charAt(index1++); + final char c2 = substring.charAt(index2++); + if (c1 == c2) { + continue; + } + if (!ignoreCase) { + return false; + } + // The same check as in String.regionMatches(): + if (Character.toUpperCase(c1) != Character.toUpperCase(c2) + && Character.toLowerCase(c1) != Character.toLowerCase(c2)) { + return false; + } + } + return true; + } + + static int indexOf(final CharSequence cs, final CharSequence searchChar, final int start) { + return cs.toString().indexOf(searchChar.toString(), start); + // if (cs instanceof String && searchChar instanceof String) { + // // TODO: Do we assume searchChar is usually relatively small; + // // If so then calling toString() on it is better than reverting to + // // the green implementation in the else block + // return ((String) cs).indexOf((String) searchChar, start); + // } else { + // // TODO: Implement rather than convert to String + // return cs.toString().indexOf(searchChar.toString(), start); + // } + } +} diff --git a/andy/src/test/resources/grader/fixtures/Library/ReverseLibrary.java b/andy/src/test/resources/grader/fixtures/Library/ReverseLibrary.java new file mode 100644 index 000000000..c320dec12 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/ReverseLibrary.java @@ -0,0 +1,41 @@ +package delft; + +class ArrayUtils { + + /** ArrayUtils should not normally be instantiated. */ + private ArrayUtils() { + } + + /** + * Reverses the order of the given array in the given range. + * + *

+ * This method does nothing for a {@code null} input array. + * + * @param array + * the array to reverse, may be {@code null} + * @param startIndexInclusive + * the starting index. Undervalue (<0) is promoted to 0, overvalue + * (>array.length) results in no change. + * @param endIndexExclusive + * elements up to endIndex-1 are reversed in the array. Undervalue + * (< start index) results in no change. Overvalue + * (>array.length) is demoted to array length. + * @since 3.2 + */ + public static void reverse(final int[] array, final int startIndexInclusive, final int endIndexExclusive) { + if (array == null) { + return; + } + int i = startIndexInclusive < 0 ? 0 : startIndexInclusive; + int j = Math.min(array.length, endIndexExclusive) - 1; + int tmp; + while (j > i) { + tmp = array[j]; + array[j] = array[i]; + array[i] = tmp; + j--; + i++; + } + } +} diff --git a/andy/src/test/resources/grader/fixtures/Library/SubstringsBetweenLibrary.java b/andy/src/test/resources/grader/fixtures/Library/SubstringsBetweenLibrary.java new file mode 100644 index 000000000..bbb189bae --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/SubstringsBetweenLibrary.java @@ -0,0 +1,80 @@ +package delft; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.*; + +class DelftStringUtilities { + private DelftStringUtilities() { + // Override default constructor, to prevent it from getting considered in the + // coverage report. + } + + /** + * Searches a String for substrings delimited by a start and end tag, returning + * all matching substrings in an array. + * + *

+ * A {@code null} input String returns {@code null}. A {@code null} open/close + * returns {@code + * null} (no match). An empty ("") open/close returns {@code null} (no match). + * + * @param str + * the String containing the substrings, null returns null, empty + * returns empty + * @param open + * the String identifying the start of the substring, empty returns + * null + * @param close + * the String identifying the end of the substring, empty returns + * null + * @return a String Array of substrings, or {@code null} if no match + * @since 2.3 + */ + public static String[] substringsBetween(final String str, final String open, final String close) { + if (str == null || isEmpty(open) || isEmpty(close)) { + return null; + } + final int strLen = str.length(); + if (strLen == 0) { + return new String[0]; + } + final int closeLen = close.length(); + final int openLen = open.length(); + final List list = new ArrayList<>(); + int pos = 0; + while (pos < strLen - closeLen) { + int start = str.indexOf(open, pos); + if (start < 0) { + break; + } + start += openLen; + final int end = str.indexOf(close, start); + if (end < 0) { + break; + } + list.add(str.substring(start, end)); + pos = end + closeLen; + } + if (list.isEmpty()) { + return null; + } + return list.toArray(new String[0]); + } + + /** + * Checks if a CharSequence is empty ("") or null. * + * + *

+ * NOTE: This method changed in Lang version 2.0. It no longer trims the + * CharSequence. That functionality is available in isBlank(). + * + * @param cs + * the CharSequence to check, may be null + * @return {@code true} if the CharSequence is empty or null + * @since 3.0 Changed signature from isEmpty(String) to isEmpty(CharSequence) + */ + public static boolean isEmpty(final CharSequence cs) { + return cs == null || cs.length() == 0; + } +} diff --git a/andy/src/test/resources/grader/fixtures/Library/SwapCaseLibrary.java b/andy/src/test/resources/grader/fixtures/Library/SwapCaseLibrary.java new file mode 100644 index 000000000..3c7cd7cc9 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/SwapCaseLibrary.java @@ -0,0 +1,59 @@ +package delft; + +class DelftWordUtilities { + + private DelftWordUtilities() { + // Override default constructor, to prevent it from getting considered in the coverage report. + } + + // ----------------------------------------------------------------------- + /** + * Swaps the case of a String using a word based algorithm. + * + *

    + *
  • Upper case character converts to Lower case + *
  • Other Lower case character converts to Upper case + *
+ * + *

+ * Whitespace is defined by {@link Character#isWhitespace(char)}. A {@code null} + * input String returns {@code null}. + * + * @param str + * the String to swap case, may be null + * @return the changed String, {@code null} if null String input + */ + public static String swapCase(final String str) { + if (DelftWordUtilities.isEmpty(str)) { + return str; + } + final char[] buffer = str.toCharArray(); + boolean whitespace = true; + for (int i = 0; i < buffer.length; i++) { + final char ch = buffer[i]; + if (Character.isUpperCase(ch)) { + buffer[i] = Character.toLowerCase(ch); + } else if (Character.isLowerCase(ch)) { + buffer[i] = Character.toUpperCase(ch); + } + } + return new String(buffer); + } + + + /** + * Checks if a CharSequence is empty ("") or null. * + * + *

+ * NOTE: This method changed in Lang version 2.0. It no longer trims the + * CharSequence. That functionality is available in isBlank(). + * + * @param cs + * the CharSequence to check, may be null + * @return {@code true} if the CharSequence is empty or null + * @since 3.0 Changed signature from isEmpty(String) to isEmpty(CharSequence) + */ + private static boolean isEmpty(final CharSequence cs) { + return cs == null || cs.length() == 0; + } +} diff --git a/andy/src/test/resources/grader/fixtures/Library/ToCamelCaseLibrary.java b/andy/src/test/resources/grader/fixtures/Library/ToCamelCaseLibrary.java new file mode 100644 index 000000000..c27419382 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/ToCamelCaseLibrary.java @@ -0,0 +1,92 @@ +package delft; + +import java.util.HashSet; +import java.util.Set; + +class DelftCaseUtilities { + + /** + * Converts all the delimiter separated words in a String into camelCase, that + * is each word is made up of a titlecase character and then a series of + * lowercase characters. + * + *

+ * The delimiters represent a set of characters understood to separate words. + * The first non-delimiter character after a delimiter will be capitalized. The + * first String character may or may not be capitalized and it's determined by + * the user input for capitalizeFirstLetter variable. + * + *

+ * A null input String returns null. Capitalization + * uses the Unicode title case, normally equivalent to upper case and cannot + * perform locale-sensitive mappings. + * + * @param str + * the String to be converted to camelCase, may be null + * @param capitalizeFirstLetter + * boolean that determines if the first character of first word + * should be title case. + * @param delimiters + * set of characters to determine capitalization, null and/or empty + * array means whitespace + * @return camelCase of String, null if null String input + */ + public String toCamelCase(String str, final boolean capitalizeFirstLetter, final char... delimiters) { + if (str == null || str.isEmpty()) { + return str; + } + str = str.toLowerCase(); + final int strLen = str.length(); + final int[] newCodePoints = new int[strLen]; + int outOffset = 0; + final Set delimiterSet = generateDelimiterSet(delimiters); + boolean capitalizeNext = false; + if (capitalizeFirstLetter) { + capitalizeNext = true; + } + for (int index = 0; index < strLen;) { + final int codePoint = str.codePointAt(index); + if (delimiterSet.contains(codePoint)) { + capitalizeNext = true; + if (outOffset == 0) { + capitalizeNext = false; + } + index += Character.charCount(codePoint); + } else if (capitalizeNext) { + final int titleCaseCodePoint = Character.toTitleCase(codePoint); + newCodePoints[outOffset++] = titleCaseCodePoint; + index += Character.charCount(titleCaseCodePoint); + capitalizeNext = false; + } else { + newCodePoints[outOffset++] = codePoint; + index += Character.charCount(codePoint); + } + } + if (outOffset != 0) { + return new String(newCodePoints, 0, outOffset); + } + return str; + } + + /** + * Converts an array of delimiters to a hash set of code points. Code point of + * space(32) is added as the default value. The generated hash set provides O(1) + * lookup time. + * + * @param delimiters + * set of characters to determine capitalization, null means + * whitespace + * @return Set + */ + private Set generateDelimiterSet(final char[] delimiters) { + final Set delimiterHashSet = new HashSet<>(); + delimiterHashSet.add(Character.codePointAt(new char[]{' '}, 0)); + if (delimiters == null || delimiters.length == 0) { + return delimiterHashSet; + } + for (int index = 0; index < delimiters.length; index++) { + delimiterHashSet.add(Character.codePointAt(delimiters, index)); + } + return delimiterHashSet; + } +} diff --git a/andy/src/test/resources/grader/fixtures/Library/ZigZagLibrary.java b/andy/src/test/resources/grader/fixtures/Library/ZigZagLibrary.java new file mode 100644 index 000000000..016eabff1 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Library/ZigZagLibrary.java @@ -0,0 +1,63 @@ +package delft; + + + +import java.util.*; +import java.util.stream.Collectors; + + +class ZigZag { + + public String zigzag(String s, int numRows) { + // some pre-condition check + if(s.length() < 1 || s.length() > 1000) + throw new IllegalArgumentException("1 <= s.length <= 1000"); + if(numRows < 1 || numRows > 1000) + throw new IllegalArgumentException("1 <= numRows <= 1000"); + + // early return: if the number of rows is 1, then, we return the same string + if (numRows == 1) return s; + + // We create a list of strings, based on the number of rows we need + List rows = new ArrayList<>(); + for (int i = 0; i < Math.min(numRows, s.length()); i++) + rows.add(new StringBuilder()); + + int curRow = 0; + boolean goingDown = false; + + // We visit character by character, and we put it in the list of strings. + // We change directions whenever we reach the top or the bottom of the list. + for (char c : s.toCharArray()) { + // add the letter + rows.get(curRow).append(c); + + // are we at the top or the bottom of the list? + boolean topOrBottom = curRow == 0 || curRow == numRows - 1; + + // add spaces if we are 'zagging' + if(!goingDown && !topOrBottom) { + for(int i = 0; i < rows.size(); i++) { + if(i!=curRow) + rows.get(i).append(" "); + } + } + + // invert the direction in case we reached the top or the bottom + if (topOrBottom) goingDown = !goingDown; + + // go to the next current row + curRow += goingDown ? 1 : -1; + } + + // we return the final string by simply combining all + // the stringbuilders into a single string + return rows + .stream() + .map(x->x.toString().trim()) + .collect(Collectors.joining("\n")) + .trim(); + } + +} + diff --git a/andy/src/test/resources/grader/fixtures/Solution/CodeSnippetGeneratorOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/CodeSnippetGeneratorOfficialSolution.java new file mode 100644 index 000000000..51e712c0b --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/CodeSnippetGeneratorOfficialSolution.java @@ -0,0 +1,175 @@ +package delft; + +import java.util.*; +import java.util.stream.*; +import org.assertj.core.data.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.*; +import org.junit.jupiter.params.provider.*; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + +class CodeSnippetUtilsTest { + + @Test + void middleOfFile() { + String actual = CodeSnippetUtils.generateCodeSnippet(List.of( + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h" + ), 5); + assertThat(actual).isEqualTo( + " c\n" + + " d\n" + + "--> e\n" + + " f\n" + + " g" + ); + } + + @Test + void beginningOfFile() { + String actual = CodeSnippetUtils.generateCodeSnippet(List.of( + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h" + ), 2); + assertThat(actual).isEqualTo( + " a\n" + + "--> b\n" + + " c\n" + + " d" + ); + } + + @Test + void firstLine() { + String actual = CodeSnippetUtils.generateCodeSnippet(List.of( + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h" + ), 1); + assertThat(actual).isEqualTo( + "--> a\n" + + " b\n" + + " c" + ); + } + + @Test + void endOfFile() { + String actual = CodeSnippetUtils.generateCodeSnippet(List.of( + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h" + ), 7); + assertThat(actual).isEqualTo( + " e\n" + + " f\n" + + "--> g\n" + + " h" + ); + } + + @Test + void lastLine() { + String actual = CodeSnippetUtils.generateCodeSnippet(List.of( + "a", + "b", + "c", + "d", + "e", + "f", + "g", + "h" + ), 8); + assertThat(actual).isEqualTo( + " f\n" + + " g\n" + + "--> h" + ); + } + + @Test + void indentation() { + String actual = CodeSnippetUtils.generateCodeSnippet(List.of( + " a", + " b", + " if(c) {", + " d", + " }", + " f", + "g", + " h" + ), 3); + assertThat(actual).isEqualTo( + " a\n" + + " b\n" + + "--> if(c) {\n" + + " d\n" + + " }" + ); + } + + @Test + void indentationMixedWithBlankLines() { + String actual = CodeSnippetUtils.generateCodeSnippet(List.of( + " a", + "", + " if(c) {", + " ", + " d", + " }", + " f", + "g", + " h" + ), 3); + assertThat(actual).isEqualTo( + " a\n" + + "\n" + + "--> if(c) {\n" + + "\n" + + " d" + ); + } + + @Test + void fileBoundaryWithIndentation() { + String actual = CodeSnippetUtils.generateCodeSnippet(List.of( + " b", + " if(c) {", + " d", + " }", + "f", + "g" + ), 2); + assertThat(actual).isEqualTo( + " b\n" + + "--> if(c) {\n" + + " d\n" + + " }" + ); + } + +} diff --git a/andy/src/test/resources/grader/fixtures/Solution/CountingClumpsOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/CountingClumpsOfficialSolution.java new file mode 100644 index 000000000..6ee5b3395 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/CountingClumpsOfficialSolution.java @@ -0,0 +1,28 @@ +package delft; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.*; +import java.util.stream.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.*; +import org.junit.jupiter.params.provider.*; + +class ClumpsTest { + + @MethodSource("generator") + @ParameterizedTest(name = "{0}") + void clumpsTest(String name, int[] nums, int res) { + assertThat(Clumps.countClumps(nums)).isEqualTo(res); + } + + private static Stream generator() { + return Stream.of(Arguments.of("null array", null, 0), Arguments.of("empty array", new int[0], 0), + Arguments.of("array with one element", new int[]{9}, 0), + Arguments.of("array with multiple elements no clump", new int[]{3, 6, 2, 7, 4, 2}, 0), + Arguments.of("array with one continuous clump", new int[]{42, 42, 42, 42}, 1), + Arguments.of("array with multiple clumps", new int[]{1, 1, 3, 0, 0}, 2), + Arguments.of("array with multiple clumps side-by-side", new int[]{1, 1, 0, 0, 3}, 2)); + } +} diff --git a/andy/src/test/resources/grader/fixtures/Solution/GetCheapestPriceOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/GetCheapestPriceOfficialSolution.java new file mode 100644 index 000000000..c4e331898 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/GetCheapestPriceOfficialSolution.java @@ -0,0 +1,97 @@ +package delft; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.*; +import java.util.stream.*; +import org.assertj.core.data.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.*; +import org.junit.jupiter.params.provider.*; + + +class SeatFinderTest { + + @ParameterizedTest(name = "{0}") + @MethodSource("validCaseGenerator") + void validCasesTests(String description, double expectedResult, + double[] prices, boolean[] taken, int numberOfSeats) { + assertThat(SeatFinder.getCheapestPrice(prices, taken, numberOfSeats)) + .isCloseTo(expectedResult, Percentage.withPercentage(0.1)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("illegalArgumentsGenerator") + void illegalArgumentsTests(String description, double[] prices, boolean[] taken, int numberOfSeats) { + assertThatIllegalArgumentException() + .isThrownBy(() -> SeatFinder.getCheapestPrice(prices, taken, numberOfSeats)); + } + + private static Stream validCaseGenerator() { + return Stream.of( + Arguments.of("Single seat, gets taken", + 13.45, arrayOf(13.45), arrayOf(false), 1), + + Arguments.of("Single seat, already taken (and discount condition out point)", + 0, arrayOf(23.67), arrayOf(true), 1), + + Arguments.of("Multiple seats, same prices, none taken, all requested", + 33.33, arrayOf(11.11, 11.11, 11.11), arrayOf(false, false, false), 3), + + Arguments.of("Multiple seats, same prices, all are already taken", + 0, arrayOf(23.56, 23.56), arrayOf(true, true), 2), + + Arguments.of("Multiple seats, same prices, some are already taken, more requested", + 21.00, arrayOf(10.50, 10.50, 10.50, 10.50), arrayOf(true, false, false, true), 3), + + Arguments.of("Multiple seats, same prices, some are already taken, not all are requested", + 15.20, arrayOf(7.60, 7.60, 7.60, 7.60), arrayOf(false, false, true, false), 2), + + Arguments.of("Multiple seats, different prices, none taken, one seat requested", + 6.23, arrayOf(16.78, 32.60, 6.23, 24.53), arrayOf(false, false, false, false), 1), + + Arguments.of("Multiple seats, different prices, none taken, multiple requested", + 57.84, arrayOf(23.54, 54.21, 34.30), arrayOf(false, false, false), 2), + + Arguments.of("Multiple seats, different prices, some taken, multiple requested", + 35.68, arrayOf(19.36, 23.75, 17.25, 16.32), arrayOf(false, false, true, false), 2), + + Arguments.of("Discount condition on point (100) does not apply discount", + 100.00, arrayOf(100.00), arrayOf(false), 1), + + Arguments.of("Discount condition off point (100.01) applies discount", + 95.01, arrayOf(100.01), arrayOf(false), 1), + + Arguments.of("Discount condition in point (245.00) applies discount", + 240.00, arrayOf(245.00), arrayOf(false), 1) + ); + } + + private static Stream illegalArgumentsGenerator() { + return Stream.of( + Arguments.of("prices is null", + null, arrayOf(true), 1), + + Arguments.of("taken is null", + arrayOf(1.0), null, 1), + + Arguments.of("prices and taken have different lengths", + arrayOf(1.0, 2.0), arrayOf(false, true, false), 1), + + Arguments.of("numberOfSeats validity condition on point (0)", + arrayOf(0.5, 1.2), arrayOf(false, true), 0), + + Arguments.of("numberOfSeats validity condition in point", + arrayOf(1.6, 4.0, 3.2), arrayOf(true, false, false), -67) + ); + } + + private static double[] arrayOf(double... values) { + return values; + } + + private static boolean[] arrayOf(boolean... values) { + return values; + } +} diff --git a/andy/src/test/resources/grader/fixtures/Solution/IntersectionOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/IntersectionOfficialSolution.java new file mode 100644 index 000000000..36e3e2f9a --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/IntersectionOfficialSolution.java @@ -0,0 +1,50 @@ +package delft; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.*; +import java.util.stream.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.*; +import org.junit.jupiter.params.provider.*; + +class ListUtilsTest { + + @ParameterizedTest + @MethodSource("generator") + void containsAny(String description, List coll1, List coll2, List expectedResult) { + List result = ListUtils.intersection(coll1, coll2); + Collections.sort(result); + Collections.sort(expectedResult); + assertThat(result).isEqualTo(expectedResult); + } + + private static Stream generator() { + Arguments t1 = Arguments.of("empty list 1", Collections.emptyList(), Arrays.asList(1), Collections.emptyList()); + Arguments t2 = Arguments.of("empty list 2", Arrays.asList(1), Collections.emptyList(), Collections.emptyList()); + Arguments t3 = Arguments.of("0 elements in the intersection", Arrays.asList(1, 2, 3), Arrays.asList(4, 5, 6), + Collections.emptyList()); + Arguments t4 = Arguments.of("1 element in the intersection", Arrays.asList(1, 2, 3), Arrays.asList(3, 4, 5), + Arrays.asList(3)); + Arguments t5 = Arguments.of("many elements in the intersection", Arrays.asList(1, 2, 3), Arrays.asList(2, 3, 4), + Arrays.asList(2, 3)); + Arguments t6 = Arguments.of("repeated elements", Arrays.asList(1), Arrays.asList(2, 1, 1, 2), + Arrays.asList(1)); + return Stream.of(t1, t2, t3, t4, t5, t6); + } + + @ParameterizedTest + @MethodSource("nullGenerator") + void nullList(String description, List coll1, List coll2) { + assertThatThrownBy(() -> ListUtils.intersection(coll1, coll2)).isInstanceOf(NullPointerException.class); + } + + private static Stream nullGenerator() { + Arguments tc1 = Arguments.of("c1 null", null, Arrays.asList(1, 4, 5)); + Arguments tc2 = Arguments.of("c2 null", Arrays.asList(1, 2, 3), null); + Arguments tc3 = Arguments.of("both null", null, null); + Arguments tc4 = Arguments.of("c1 null c2 empty", null, new ArrayList()); + return Stream.of(tc1, tc2, tc3, tc4); + } +} diff --git a/andy/src/test/resources/grader/fixtures/Solution/IsEqualCollectionOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/IsEqualCollectionOfficialSolution.java new file mode 100644 index 000000000..8e5b2f131 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/IsEqualCollectionOfficialSolution.java @@ -0,0 +1,54 @@ +package delft; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.*; +import java.util.stream.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.*; +import org.junit.jupiter.params.provider.*; + +class CollectionUtilsTest { + + @ParameterizedTest + @MethodSource("generator") + void containsAny(String description, Collection coll1, Collection coll2, boolean expectedResult) { + boolean result = CollectionUtils.isEqualCollection(coll1, coll2); + assertThat(result).isEqualTo(expectedResult); + } + + private static Stream generator() { + Arguments t1 = Arguments.of("empty list A", Collections.emptyList(), Arrays.asList(1), false); + Arguments t2 = Arguments.of("empty list B", Arrays.asList(1), Collections.emptyList(), false); + Arguments t3 = Arguments.of("same one item A, one item B", Arrays.asList(1), Arrays.asList(1), true); + Arguments t4 = Arguments.of("different one item A, one item B", Arrays.asList(1), Arrays.asList(2), false); + Arguments t5 = Arguments.of("A is exactly same as B", Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true); + Arguments t6 = Arguments.of("A is equals to B, but elements are different in order", Arrays.asList(1, 2, 3), + Arrays.asList(1, 3, 2), true); + Arguments t7 = Arguments.of("A is equals to B, multiple cardinalities", Arrays.asList(1, 2, 3, 3), + Arrays.asList(3, 1, 3, 2), true); + Arguments t8 = Arguments.of("A is not equal to B, multiple cardinalities", Arrays.asList(1, 2, 3, 3), + Arrays.asList(3, 1, 2, 2), false); + Arguments t9 = Arguments.of("A is not equal to B, different cardinalities", Arrays.asList(1, 2, 3, 3), + Arrays.asList(3, 1, 2), false); + Arguments t10 = Arguments.of("A larger than B", Arrays.asList(1, 2, 3), Arrays.asList(1, 2), false); + Arguments t11 = Arguments.of("B larger than A", Arrays.asList(1, 2), Arrays.asList(1, 2, 3), false); + Arguments t12 = Arguments.of("A and B different cardinality, same size", Arrays.asList(1, 1), + Arrays.asList(1, 2), false); + return Stream.of(t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12); + } + + @ParameterizedTest + @MethodSource("nullGenerator") + void nullList(String description, Collection coll1, Collection coll2) { + assertThatThrownBy(() -> CollectionUtils.isEqualCollection(coll1, coll2)).isInstanceOf(Exception.class); + } + + private static Stream nullGenerator() { + Arguments tc1 = Arguments.of("c1 null", null, Arrays.asList(1, 4, 5)); + Arguments tc2 = Arguments.of("c2 null", Arrays.asList(1, 2, 3), null); + Arguments tc3 = Arguments.of("both null", null, null); + return Stream.of(tc1, tc2, tc3); + } +} diff --git a/andy/src/test/resources/grader/fixtures/Solution/RepeatOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/RepeatOfficialSolution.java new file mode 100644 index 000000000..594b5efbc --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/RepeatOfficialSolution.java @@ -0,0 +1,33 @@ +package delft; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.*; +import java.util.stream.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.*; +import org.junit.jupiter.params.provider.*; + +class DelftStringUtilitiesTest { + + @ParameterizedTest + @MethodSource("generator") + void repeat(String description, String originalString, int times, String expectedString) { + assertThat(DelftStringUtilities.repeat(originalString, times)).isEqualTo(expectedString); + } + + static Stream generator() { + Arguments tc1 = Arguments.of("str null", null, 2, null); + Arguments tc2 = Arguments.of("empty string", "", 2, ""); + Arguments tc3 = Arguments.of("repeat negative", "x", -1, ""); + Arguments tc4 = Arguments.of("zero repetitions", "x", 0, ""); + Arguments tc5 = Arguments.of("1 char, 1 repetition", "x", 1, "x"); + Arguments tc6 = Arguments.of("2 chars, 1 repetition", "ab", 1, "ab"); + Arguments tc7 = Arguments.of("3+ chars, 1 repetition", "xxx", 1, "xxx"); + Arguments tc8 = Arguments.of("1 char, N repetitions", "x", 3, "xxx"); + Arguments tc9 = Arguments.of("2 chars, N repetitions", "ab", 3, "ababab"); + Arguments tc10 = Arguments.of("3+ chars, N repetitions", "xxx", 3, "xxxxxxxxx"); + return Stream.of(tc1, tc2, tc3, tc4, tc5, tc6, tc7, tc8, tc9, tc10); + } +} diff --git a/andy/src/test/resources/grader/fixtures/Solution/ReplaceOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/ReplaceOfficialSolution.java new file mode 100644 index 000000000..6cd5dbbe8 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/ReplaceOfficialSolution.java @@ -0,0 +1,87 @@ +package delft; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.*; +import java.util.stream.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.*; +import org.junit.jupiter.params.provider.*; + +class DelftStringUtilitiesTest { + + @ParameterizedTest + @MethodSource("generator") + void test(String text, String searchString, String replacement, int max, boolean ignoreCase, String expectedResult) { + assertThat(DelftStringUtilities.replace(text, searchString, replacement, max, ignoreCase)).isEqualTo(expectedResult); + } + + private static Stream generator() { + return Stream.of( + // Test where the text is null + Arguments.of(null, "day", "night", -1, false, null), + + // Test where the search text is null + Arguments.of("I went to the beach the other day.", null, "mall", 4, false, "I went to the beach the other day."), + + // Test where the replacement text is null + Arguments.of("I went to the beach the other day.", "beach", null, 2, false, "I went to the beach the other day."), + + // Test where the text is empty + Arguments.of("", "day", "night", 1, false, ""), + + // Test where the search string is empty + Arguments.of("I like programming.", "", "hacking", 1, false, "I like programming."), + + // Test where the maximum number of replacements is 0 + Arguments.of("I like programming.", "programming", "hacking", 0, false, "I like programming."), + + // Test where the search string is empty + Arguments.of("I like programming.", "", "hacking", 1, false, "I like programming."), + + // Test where the ignore case is true + Arguments.of("SQT, ADS, OOP", "sqt", "PTS", 1, true, "PTS, ADS, OOP"), + + // Test where the ignore case is false + Arguments.of("IDM, LA, RNL", "idm", "PTS", 1, false, "IDM, LA, RNL"), + + // Test where the search string is not contained in the text + Arguments.of("I went to the beach the other day.", "night", "afternoon", 1, true, "I went to the beach the other day."), + + // Test where the search string is not contained in the text + Arguments.of("I went to the beach the other day.", "night", "afternoon", 1, true, "I went to the beach the other day."), + + // Test where the replacement string is longer than the search string + Arguments.of("SQT, ADS, OOP", "SQT", "OOPP", 1, false, "OOPP, ADS, OOP"), + + // Test where the replacement string is shorter than the search string + Arguments.of("SQT, ADS, OOP", "SQT", "LA", 2, false, "LA, ADS, OOP"), + + // Test where the maximum is negative + Arguments.of("OOPP, LA, IDM, Statistics", "Statistics", "SQT", -1, false, "OOPP, LA, IDM, SQT"), + + // Test where the maximum is 0 + Arguments.of("OOPP, LA, IDM, Calculus", "Calculus", "SQT", 0, false, "OOPP, LA, IDM, Calculus"), + + // Test where the maximum number is 64 + Arguments.of(("abc").repeat(64), "abc", "def", 64, false, ("def").repeat(64)), + + // Test where the maximum number is higher than 64 + Arguments.of(("abc").repeat(70), "abc", "def", 70, false, ("def").repeat(70)), + + // The replacement string is longer than the search string and the maximum number is more than 64 + Arguments.of(("abc").repeat(64), "abc", "defg", 64, false, ("defg").repeat(64)), + + // The maximum number is 16 + Arguments.of(("abc").repeat(16), "a", "aaa", 16, true, ("aaabc").repeat(16)), + + // The replacement string is shorter than the long string + Arguments.of("SQT, IDM, LA", "SQ", "F", 1, false, "FT, IDM, LA"), + + // Test where the search string is both lowercase and uppercase letters, case insensitivity + Arguments.of("SqT, SQT, SQt", "qt", "AB", 2, true, "SAB, SAB, SQt") + ); + } + +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Solution/ReverseOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/ReverseOfficialSolution.java new file mode 100644 index 000000000..4785abffa --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/ReverseOfficialSolution.java @@ -0,0 +1,45 @@ +package delft; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.*; +import java.util.stream.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.*; +import org.junit.jupiter.params.provider.*; + +class ArrayUtilsTest { + + @ParameterizedTest(name = "{0}") + @MethodSource("generator") + void reverse(String description, int[] arrayOfInts, int startIndex, int endIndex, int[] expectedArray) { + ArrayUtils.reverse(arrayOfInts, startIndex, endIndex); + assertThat(arrayOfInts).isEqualTo(expectedArray); + } + + private static Stream generator() { + Arguments tc1 = Arguments.of("startIndex < endIndex", ints(1, 2, 3, 4, 5), 1, 4, ints(1, 4, 3, 2, 5)); + Arguments tc2 = Arguments.of("startIndex > endIndex", ints(1, 2, 3, 4, 5), 4, 1, ints(1, 2, 3, 4, 5)); + Arguments tc3 = Arguments.of("startIndex == endIndex", ints(1, 2, 3, 4, 5), 1, 1, ints(1, 2, 3, 4, 5)); + Arguments tc4 = Arguments.of("startIndex == endIndex + 1", ints(1, 2, 3, 4, 5), 1, 2, ints(1, 2, 3, 4, 5)); + Arguments tc5 = Arguments.of("startIndex == endIndex + 2", ints(1, 2, 3, 4, 5), 1, 3, ints(1, 3, 2, 4, 5)); + Arguments tc6 = Arguments.of("startIndex zero", ints(1, 2, 3, 4, 5), 0, 4, ints(4, 3, 2, 1, 5)); + Arguments tc7 = Arguments.of("endIndex zero", ints(1, 2, 3, 4, 5), 0, 0, ints(1, 2, 3, 4, 5)); + Arguments tc8 = Arguments.of("endIndex larger than array size", ints(1, 2, 3, 4, 5), 1, 10, + ints(1, 5, 4, 3, 2)); + Arguments tc9 = Arguments.of("endIndex precisely array size", ints(1, 2, 3, 4, 5), 1, 5, ints(1, 5, 4, 3, 2)); + Arguments tc10 = Arguments.of("endIndex last index", ints(1, 2, 3, 4, 5), 1, 4, ints(1, 4, 3, 2, 5)); + Arguments tc11 = Arguments.of("null array", null, 0, 1, null); + Arguments tc12 = Arguments.of("negative start index", ints(1, 2, 3, 4, 5), -1, 3, ints(3, 2, 1, 4, 5)); + Arguments tc13 = Arguments.of("negative end index", ints(1, 2, 3, 4, 5), 1, -1, ints(1, 2, 3, 4, 5)); + Arguments tc14 = Arguments.of("startIndex larger than array size", ints(1, 2, 3, 4, 5), 10, 3, + ints(1, 2, 3, 4, 5)); + Arguments tc15 = Arguments.of("empty array", ints(), 0, 0, ints()); + return Stream.of(tc1, tc2, tc3, tc4, tc5, tc6, tc7, tc8, tc9, tc10, tc11, tc12, tc13, tc14, tc15); + } + + private static int[] ints(int... nums) { + return nums; + } +} diff --git a/andy/src/test/resources/grader/fixtures/Solution/SubstringsBetweenOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/SubstringsBetweenOfficialSolution.java new file mode 100644 index 000000000..2eeeec76f --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/SubstringsBetweenOfficialSolution.java @@ -0,0 +1,78 @@ +package delft; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.*; +import java.util.stream.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.*; +import org.junit.jupiter.params.provider.*; + +class DelftStringUtilitiesTest { + + @Test + void nullCloseStringTest() { + assertThat(DelftStringUtilities.substringsBetween("hello", "e", null)).isNull(); + } + + @Test + void emptyCloseStringTest() { + assertThat(DelftStringUtilities.substringsBetween("hello", "e", "")).isNull(); + } + + @Test + void nullOpenStringTest() { + assertThat(DelftStringUtilities.substringsBetween("string", null, "n")).isNull(); + } + + @Test + void emptyOpenStringTest() { + assertThat(DelftStringUtilities.substringsBetween("hello", "", "l")).isNull(); + } + + @Test + void nullStringTest() { + assertThat(DelftStringUtilities.substringsBetween(null, "a", "b")).isNull(); + } + + @Test + void emptyStringTest() { + assertThat(DelftStringUtilities.substringsBetween("", "a", "b")).isEmpty(); + } + + @Test + void closeLargerThanStringTest() { + assertThat(DelftStringUtilities.substringsBetween("ab", "a", "abc")).isNull(); + } + + @Test + void openNotPresentInStringTest() { + assertThat(DelftStringUtilities.substringsBetween("ab", "c", "b")).isNull(); + } + + @Test + void closeNotPresentInStringTest() { + assertThat(DelftStringUtilities.substringsBetween("ab", "a", "c")).isNull(); + } + + @Test + void twoSubstringsTest() { + assertThat(DelftStringUtilities.substringsBetween("ahellocjoeabyec", "a", "c")).containsExactly("hello", "bye"); + } + + @Test + void twoSubstringsNextToEachOtherTest() { + assertThat(DelftStringUtilities.substringsBetween("ahellocabyec", "a", "c")).containsExactly("hello", "bye"); + } + + @Test + void emptySubstringTest() { + assertThat(DelftStringUtilities.substringsBetween("ahellocdefacdef", "a", "c")).containsExactly("hello", ""); + } + + @Test + void longOpenAndCloseTest() { + assertThat(DelftStringUtilities.substringsBetween("abchellodeftestabcbyedefbctestdefabctestdeghi", "abc", "def")).containsExactly("hello", "bye"); + } +} diff --git a/andy/src/test/resources/grader/fixtures/Solution/SwapCaseOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/SwapCaseOfficialSolution.java new file mode 100644 index 000000000..29792d30f --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/SwapCaseOfficialSolution.java @@ -0,0 +1,33 @@ +package delft; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.*; + +import java.util.*; +import java.util.stream.*; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.*; +import org.junit.jupiter.params.provider.*; + +class DelftWordUtilitiesTest { + + @ParameterizedTest + @MethodSource("generator") + void swapCase(String description, String originalStr, String expectedStr) { + assertThat(DelftWordUtilities.swapCase(originalStr)).isEqualTo(expectedStr); + } + + static Stream generator() { + Arguments tc1 = Arguments.of("str null", null, null); + Arguments tc2 = Arguments.of("str empty", "", ""); + Arguments tc3 = Arguments.of("1 char upper", "A", "a"); + Arguments tc4 = Arguments.of("1 char lower", "b", "B"); + Arguments tc5 = Arguments.of("1 char whitespace", " ", " "); + Arguments tc6 = Arguments.of("multiple chars upper", "ABC", "abc"); + Arguments tc7 = Arguments.of("multiple chars lower", "def", "DEF"); + Arguments tc8 = Arguments.of("multiple chars whitespace", " ", " "); + Arguments tc9 = Arguments.of("lower case after whitespace", " k", " K"); + Arguments tc10 = Arguments.of("mutliple chars mixed", "AkdLc keCtk", "aKDlC KEcTK"); + return Stream.of(tc1, tc2, tc3, tc4, tc5, tc6, tc7, tc8, tc9, tc10); + } +} diff --git a/andy/src/test/resources/grader/fixtures/Solution/ToCamelCaseOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/ToCamelCaseOfficialSolution.java new file mode 100644 index 000000000..114ba62e0 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/ToCamelCaseOfficialSolution.java @@ -0,0 +1,46 @@ +package delft; + +import static org.assertj.core.api.Assertions.*; + +import java.util.stream.*; +import org.junit.jupiter.params.*; +import org.junit.jupiter.params.provider.*; + +class DelftCaseUtilitiesTest { + + private final DelftCaseUtilities delftCaseUtilities = new DelftCaseUtilities(); + + @MethodSource("generator") + @ParameterizedTest(name = "{0}") + void domainTest(String name, String str, boolean firstLetter, char[] delimiters, String result) { + assertThat(delftCaseUtilities.toCamelCase(str, firstLetter, delimiters)).isEqualTo(result); + } + + private static Stream generator() { + return Stream.of( + Arguments.of("null", null, true, new char[]{'.'}, null), + Arguments.of("empty", "", true, new char[]{'.'}, ""), + Arguments.of("non-empty single word, capitalize first letter", "aVOcado", true, new char[]{'.'}, + "Avocado"), + Arguments.of("non-empty single word, not capitalize first letter", "aVOcado", false, new char[]{'.'}, + "avocado"), + Arguments.of("non-empty single word, capitalize first letter, no delimiters", "aVOcado", true, + new char[]{}, "Avocado"), + Arguments.of("non-empty single word, capitalize first letter, single delimiter", "aVOcado", true, + new char[]{'.'}, "Avocado"), + Arguments.of("non-empty single word, capitalize first letter, multiple delimiters", "aVOcado", true, + new char[]{'c', 'd'}, "AvoAO"), + Arguments.of("non-empty multiple words, capitalize first letter, no delimiters", "aVOcado bAnana", true, + new char[]{}, "AvocadoBanana"), + Arguments.of("non-empty multiple words, capitalize first letter, single existing delimiter", + "aVOcado-bAnana", true, new char[]{'-'}, "AvocadoBanana"), + Arguments.of("non-empty multiple words, capitalize first letter, single non-existing delimiter", + "aVOcado bAnana", true, new char[]{'x'}, "AvocadoBanana"), + Arguments.of("non-empty multiple words, capitalize first letter, multiple existing delimiters", + "aVOcado bAnana", true, new char[]{' ', 'n'}, "AvocadoBaAA"), + Arguments.of("non-empty multiple words, capitalize first letter, multiple non-existing delimiters", + "aVOcado bAnana", true, new char[]{'x', 'y'}, "AvocadoBanana"), + Arguments.of("delimiters is null", "apple", true, null, "Apple"), + Arguments.of("only delimiters in word", "apple", true, new char[]{'a', 'p', 'l', 'e'}, "apple")); + } +} diff --git a/andy/src/test/resources/grader/fixtures/Solution/ZigZagOfficialSolution.java b/andy/src/test/resources/grader/fixtures/Solution/ZigZagOfficialSolution.java new file mode 100644 index 000000000..2128a5f46 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Solution/ZigZagOfficialSolution.java @@ -0,0 +1,95 @@ +package delft; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ZigZagTest { + + + @ParameterizedTest + @CsvSource({ + "A,A", + "AB,AB" + }) + void singleRow(String s, String expected) { + assertThat(new ZigZag().zigzag(s,1)) + .isEqualTo(expected); + } + + @Test + void multipleRows() { + assertThat(new ZigZag().zigzag("PAYPALISHIRING",2)) + .isEqualTo( + "PYAIHRN\n" + + "APLSIIG" + ); + + assertThat(new ZigZag().zigzag("PAYPALISHIRING",3)) + .isEqualTo( + "P A H N\n" + + "APLSIIG\n" + + "Y I R" + ); + + assertThat(new ZigZag().zigzag("PAYPALISHIRING",4)) + .isEqualTo( + "P I N\n" + + "A LS IG\n" + + "YA HR\n" + + "P I" + ); + + assertThat(new ZigZag().zigzag("PAYPALISHIRING",5)) + .isEqualTo( + "P H\n" + + "A SI\n" + + "Y I R\n" + + "PL IG\n" + + "A N" + ); + } + + @Test + void moreRowsThanChars() { + assertThat(new ZigZag().zigzag("ABC",4)) + .isEqualTo("A\nB\nC"); + } + + @Test + void invalidStrings() { + assertThatThrownBy(() -> new ZigZag().zigzag("", 1)) + .isInstanceOf(IllegalArgumentException.class); + + StringBuilder longString = new StringBuilder(); + + // <= 1000 + for(int i = 0; i < 1000; i++) { + longString.append("a"); + } + assertThat(new ZigZag().zigzag(longString.toString(), 1)).isEqualTo(longString.toString()); + + // > 1000 + longString.append("a"); + assertThatThrownBy(() -> new ZigZag().zigzag(longString.toString(), 1)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void invalidNumberOfRows() { + assertThatThrownBy(() -> new ZigZag().zigzag("somestring", 0)) + .isInstanceOf(IllegalArgumentException.class); + + // <= 1000 + assertThat(new ZigZag().zigzag("somestring", 1000)).isEqualTo("s\no\nm\ne\ns\nt\nr\ni\nn\ng"); + // > 1000 + assertThatThrownBy(() -> new ZigZag().zigzag("somestring", 1001)) + .isInstanceOf(IllegalArgumentException.class); + + } + + +} From 1e2def1226771ee6c8e7449d68215f765a7a0e65 Mon Sep 17 00:00:00 2001 From: Afonso Vicente Nobre Date: Mon, 16 Mar 2026 13:19:16 +0100 Subject: [PATCH 23/23] Fixing bugs in tests; creating tests for contribution metric --- .../cse1110/andy/result/QualityResult.java | 9 +- .../integration/quality/CohesionTest.java | 101 ----------------- .../quality/OverviewQualityResults.java | 2 +- .../test/java/unit/quality/CohesionTest.java | 107 ++++++++++++++++++ .../java/unit/quality/ContributionTest.java | 95 ++++++++++++++++ .../quality/IsolationTest.java | 2 +- .../standard/StandardResultWriterTest.java | 2 + 7 files changed, 208 insertions(+), 110 deletions(-) delete mode 100644 andy/src/test/java/integration/quality/CohesionTest.java create mode 100644 andy/src/test/java/unit/quality/CohesionTest.java create mode 100644 andy/src/test/java/unit/quality/ContributionTest.java rename andy/src/test/java/{integration => unit}/quality/IsolationTest.java (99%) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java index d24ddde03..7d1a7c21e 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/QualityResult.java @@ -20,6 +20,7 @@ public QualityResult(int numUnitTests) { // dummy: this.score = 0; this.numUnitTests = numUnitTests; + unitTests = new HashMap<>(); metaTestReports = new LinkedList<>(); testToMetaTests = new HashMap<>(); coveragePerTest = new HashMap<>(); @@ -96,12 +97,6 @@ public void considerMetaTest(MetaTestReport metaTestReport) { String testName = metaTestReport.getName(); -// if (testName.matches(".* \\(\\d+\\)")) { -// String method = testName.replaceAll(" \\(\\d+\\)$", "").trim(); -// String invocation = testName.replaceAll(".* \\((\\d+)\\)$", "($1)").trim(); -// testName = method + " " + invocation; -// } - testToMetaTests.get(test).add(testName); } } @@ -150,7 +145,7 @@ public long countIsolatedTests() { } } - int count = unitTests.size(); + int count = numUnitTests; for (Set collisions : nonisolatedTests.values()) { if (!collisions.isEmpty()) count--; diff --git a/andy/src/test/java/integration/quality/CohesionTest.java b/andy/src/test/java/integration/quality/CohesionTest.java deleted file mode 100644 index 6458466d2..000000000 --- a/andy/src/test/java/integration/quality/CohesionTest.java +++ /dev/null @@ -1,101 +0,0 @@ -//package integration.quality; -// -//import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; -//import nl.tudelft.cse1110.andy.result.QualityResult; -//import nl.tudelft.cse1110.andy.result.TestFailureInfo; -//import org.junit.jupiter.api.Test; -//import org.junit.platform.launcher.TestIdentifier; -// -//import java.util.HashMap; -//import java.util.List; -//import java.util.Map; -// -//import static org.junit.jupiter.api.Assertions.assertEquals; -// -//public class CohesionTest { -// -// @Test -// public void testTriggersOneMetaTestTest() { -// QualityResult qualityResult = new QualityResult(1); -// qualityResult.setUnitTests(List.of("test")); -// -// TestFailureInfo failure = new TestFailureInfo("test", "error message"); -// MetaTestReport metaTestReport = new MetaTestReport(1, 0, 1, List.of(failure)); -// qualityResult.considerMetaTest(metaTestReport); -// -// assertEquals(1, qualityResult.countCohesiveTests()); -// } -// -// @Test -// public void testTriggersNoMetaTestTest() { -// QualityResult qualityResult = new QualityResult(1); -// qualityResult.setUnitTests(List.of("test")); -// -// MetaTestReport metaTestReport = new MetaTestReport(1, 1, 1, List.of()); -// qualityResult.considerMetaTest(metaTestReport); -// -// assertEquals(0, qualityResult.countCohesiveTests()); -// } -// -// @Test -// public void sameTestTriggersTwoMetaTestsTest() { -// QualityResult qualityResult = new QualityResult(1); -// qualityResult.setUnitTests(List.of("test")); -// -// TestFailureInfo failure1 = new TestFailureInfo("test", "error message"); -// TestFailureInfo failure2 = new TestFailureInfo("test", "some other error message"); -// MetaTestReport metaTestReport1 = new MetaTestReport(1, 1, 1, List.of(failure1)); -// MetaTestReport metaTestReport2 = new MetaTestReport(1, 1, 1, List.of(failure2)); -// qualityResult.considerMetaTest(metaTestReport1); -// qualityResult.considerMetaTest(metaTestReport2); -// -// assertEquals(0, qualityResult.countCohesiveTests()); -// } -// -// @Test -// public void differentTestsTriggerDifferentMetaTestsTest() { -// QualityResult qualityResult = new QualityResult(2); -// qualityResult.setUnitTests(List.of("test1", "test2")); -// -// TestFailureInfo failure1 = new TestFailureInfo("test1", "error message"); -// TestFailureInfo failure2 = new TestFailureInfo("test2", "some other error message"); -// MetaTestReport metaTestReport1 = new MetaTestReport(2, 1, 2, List.of(failure1)); -// MetaTestReport metaTestReport2 = new MetaTestReport(2, 1, 2, List.of(failure2)); -// qualityResult.considerMetaTest(metaTestReport1); -// qualityResult.considerMetaTest(metaTestReport2); -// -// assertEquals(2, qualityResult.countCohesiveTests()); -// } -// -// @Test -// public void differentTestsTriggerSameMetaTestsTest() { -// QualityResult qualityResult = new QualityResult(2); -// qualityResult.setUnitTests(List.of("test1", "test2")); -// -// TestFailureInfo failure1 = new TestFailureInfo("test1", "error message"); -// TestFailureInfo failure2 = new TestFailureInfo("test2", "some other error message"); -// MetaTestReport metaTestReport = new MetaTestReport(2, 0, 2, List.of(failure1, failure2)); -// qualityResult.considerMetaTest(metaTestReport); -// -// assertEquals(2, qualityResult.countCohesiveTests()); -// } -// -// @Test -// public void noTestsTest() { -// QualityResult qualityResult = new QualityResult(0); -// qualityResult.setUnitTests(List.of()); -// -// MetaTestReport metaTestReport = new MetaTestReport(0, 0, 0, List.of()); -// qualityResult.considerMetaTest(metaTestReport); -// -// assertEquals(0, qualityResult.countCohesiveTests()); -// } -// -// @Test -// public void noMetaTestsTest() { -// QualityResult qualityResult = new QualityResult(1); -// qualityResult.setUnitTests(List.of("test")); -// -// assertEquals(0, qualityResult.countCohesiveTests()); -// } -//} diff --git a/andy/src/test/java/integration/quality/OverviewQualityResults.java b/andy/src/test/java/integration/quality/OverviewQualityResults.java index ea93190fb..85a112d07 100644 --- a/andy/src/test/java/integration/quality/OverviewQualityResults.java +++ b/andy/src/test/java/integration/quality/OverviewQualityResults.java @@ -84,7 +84,7 @@ void metaTestQuality( static Stream domainTestingTestSuites() { return Stream.of( - // Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration") + Arguments.of("NumberUtilsAddLibrary", "NumberUtilsAddOfficialSolution", "NumberUtilsAddConfiguration") // ## Week 1 // Arguments.of("ContainsAnyLibrary", "ContainsAnyOfficialSolution", "ContainsAnyConfiguration") // Arguments.of("LastIndexOfLibrary", "LastIndexOfOfficialSolution", "LastIndexOfConfiguration") diff --git a/andy/src/test/java/unit/quality/CohesionTest.java b/andy/src/test/java/unit/quality/CohesionTest.java new file mode 100644 index 000000000..dc1267deb --- /dev/null +++ b/andy/src/test/java/unit/quality/CohesionTest.java @@ -0,0 +1,107 @@ +package unit.quality; + +import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; +import nl.tudelft.cse1110.andy.result.QualityResult; +import nl.tudelft.cse1110.andy.result.TestFailureInfo; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class CohesionTest { + + @Test + public void testTriggersOneMetaTestTest() { + QualityResult qualityResult = new QualityResult(1); + qualityResult.setUnitTests(Map.of("testID", "test")); + + TestFailureInfo failure = new TestFailureInfo("test", "error message"); + MetaTestReport metaTestReport = new MetaTestReport(1, 0, 1, List.of(failure)); + metaTestReport.setName("meta-test"); + qualityResult.considerMetaTest(metaTestReport); + + assertEquals(1, qualityResult.countCohesiveTests()); + } + + @Test + public void testTriggersNoMetaTestTest() { + QualityResult qualityResult = new QualityResult(1); + qualityResult.setUnitTests(Map.of("testID", "test")); + + MetaTestReport metaTestReport = new MetaTestReport(1, 1, 1, List.of()); + metaTestReport.setName("meta-test"); + qualityResult.considerMetaTest(metaTestReport); + + assertEquals(0, qualityResult.countCohesiveTests()); + } + + @Test + public void sameTestTriggersTwoMetaTestsTest() { + QualityResult qualityResult = new QualityResult(1); + qualityResult.setUnitTests(Map.of("testID", "test")); + + TestFailureInfo failure1 = new TestFailureInfo("test", "error message"); + TestFailureInfo failure2 = new TestFailureInfo("test", "some other error message"); + MetaTestReport metaTestReport1 = new MetaTestReport(1, 1, 1, List.of(failure1)); + metaTestReport1.setName("meta-test1"); + MetaTestReport metaTestReport2 = new MetaTestReport(1, 1, 1, List.of(failure2)); + metaTestReport2.setName("meta-test2"); + qualityResult.considerMetaTest(metaTestReport1); + qualityResult.considerMetaTest(metaTestReport2); + + assertEquals(0, qualityResult.countCohesiveTests()); + } + + @Test + public void differentTestsTriggerDifferentMetaTestsTest() { + QualityResult qualityResult = new QualityResult(2); + qualityResult.setUnitTests(Map.of("test1ID", "test1", "test2ID", "test2")); + + TestFailureInfo failure1 = new TestFailureInfo("test1", "error message"); + TestFailureInfo failure2 = new TestFailureInfo("test2", "some other error message"); + MetaTestReport metaTestReport1 = new MetaTestReport(2, 1, 2, List.of(failure1)); + metaTestReport1.setName("meta-test1"); + MetaTestReport metaTestReport2 = new MetaTestReport(2, 1, 2, List.of(failure2)); + metaTestReport2.setName("meta-test2"); + qualityResult.considerMetaTest(metaTestReport1); + qualityResult.considerMetaTest(metaTestReport2); + + assertEquals(2, qualityResult.countCohesiveTests()); + } + + @Test + public void differentTestsTriggerSameMetaTestsTest() { + QualityResult qualityResult = new QualityResult(2); + qualityResult.setUnitTests(Map.of("test1ID", "test1", "test2ID", "test2")); + + TestFailureInfo failure1 = new TestFailureInfo("test1", "error message"); + TestFailureInfo failure2 = new TestFailureInfo("test2", "some other error message"); + MetaTestReport metaTestReport = new MetaTestReport(2, 0, 2, List.of(failure1, failure2)); + metaTestReport.setName("meta-test"); + qualityResult.considerMetaTest(metaTestReport); + + assertEquals(2, qualityResult.countCohesiveTests()); + } + + @Test + public void noTestsTest() { + QualityResult qualityResult = new QualityResult(0); + qualityResult.setUnitTests(Map.of()); + + MetaTestReport metaTestReport = new MetaTestReport(0, 0, 0, List.of()); + metaTestReport.setName("meta-test"); + qualityResult.considerMetaTest(metaTestReport); + + assertEquals(0, qualityResult.countCohesiveTests()); + } + + @Test + public void noMetaTestsTest() { + QualityResult qualityResult = new QualityResult(1); + qualityResult.setUnitTests(Map.of("testID", "test")); + + assertEquals(0, qualityResult.countCohesiveTests()); + } +} diff --git a/andy/src/test/java/unit/quality/ContributionTest.java b/andy/src/test/java/unit/quality/ContributionTest.java new file mode 100644 index 000000000..70b7926e8 --- /dev/null +++ b/andy/src/test/java/unit/quality/ContributionTest.java @@ -0,0 +1,95 @@ +package unit.quality; + +import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; +import nl.tudelft.cse1110.andy.result.QualityResult; +import nl.tudelft.cse1110.andy.result.TestFailureInfo; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ContributionTest { + + @Test + public void testNoTestsContributeWhenAllMapsEmpty() { + QualityResult qualityResult = new QualityResult(3); + qualityResult.setUnitTests(Map.of("id1", "test1", "id2", "test2", "id3", "test3")); + // no meta tests, coverage, or mutations set + assertEquals(0, qualityResult.countContributingTests()); + } + + @Test + public void testContributesViaMetaTestOnly() { + QualityResult qualityResult = new QualityResult(1); + qualityResult.setUnitTests(Map.of("id1", "test1")); + + TestFailureInfo failure = new TestFailureInfo("test1", "error"); + MetaTestReport report = new MetaTestReport(1, 1, 1, List.of(failure)); + report.setName("meta-test1"); + qualityResult.considerMetaTest(report); + + assertEquals(1, qualityResult.countContributingTests()); + } + + @Test + public void testContributesViaCoverageOnly() { + QualityResult qualityResult = new QualityResult(1); + qualityResult.setUnitTests(Map.of("id1", "test1")); + qualityResult.setCoveragePerTest(Map.of("test1", Set.of(10, 11, 12))); + + assertEquals(1, qualityResult.countContributingTests()); + } + + @Test + public void testContributesViaMutationOnly() { + QualityResult qualityResult = new QualityResult(1); + qualityResult.setUnitTests(Map.of("id1", "test1")); + qualityResult.setMutationsKilledPerTest(Map.of("test1", Set.of(1, 2))); + + assertEquals(1, qualityResult.countContributingTests()); + } + + @Test + public void testSameTestContributesViaAllThree() { + QualityResult qualityResult = new QualityResult(1); + qualityResult.setUnitTests(Map.of("id1", "test1")); + + TestFailureInfo failure = new TestFailureInfo("test1", "error"); + MetaTestReport report = new MetaTestReport(1, 1, 1, List.of(failure)); + report.setName("meta-test1"); + qualityResult.considerMetaTest(report); + qualityResult.setCoveragePerTest(Map.of("test1", Set.of(10))); + qualityResult.setMutationsKilledPerTest(Map.of("test1", Set.of(1))); + + // still counts as 1 contributing test, not 3 + assertEquals(1, qualityResult.countContributingTests()); + } + + @Test + public void testMultipleTestsEachContributingDifferently() { + QualityResult qualityResult = new QualityResult(3); + qualityResult.setUnitTests(Map.of("id1", "test1", "id2", "test2", "id3", "test3")); + + TestFailureInfo failure = new TestFailureInfo("test1", "error"); + MetaTestReport report = new MetaTestReport(1, 1, 1, List.of(failure)); + report.setName("meta-test1"); + qualityResult.considerMetaTest(report); + qualityResult.setCoveragePerTest(Map.of("test2", Set.of(10))); + qualityResult.setMutationsKilledPerTest(Map.of("test3", Set.of(1))); + + assertEquals(3, qualityResult.countContributingTests()); + } + + @Test + public void testTestWithEmptySetsDoesNotContribute() { + QualityResult qualityResult = new QualityResult(1); + qualityResult.setUnitTests(Map.of("id1", "test1")); + qualityResult.setCoveragePerTest(Map.of("test1", Set.of())); + qualityResult.setMutationsKilledPerTest(Map.of("test1", Set.of())); + + assertEquals(0, qualityResult.countContributingTests()); + } +} diff --git a/andy/src/test/java/integration/quality/IsolationTest.java b/andy/src/test/java/unit/quality/IsolationTest.java similarity index 99% rename from andy/src/test/java/integration/quality/IsolationTest.java rename to andy/src/test/java/unit/quality/IsolationTest.java index 59f479818..004c139b6 100644 --- a/andy/src/test/java/integration/quality/IsolationTest.java +++ b/andy/src/test/java/unit/quality/IsolationTest.java @@ -1,4 +1,4 @@ -package integration.quality; +package unit.quality; import nl.tudelft.cse1110.andy.execution.metatest.MetaTestReport; import nl.tudelft.cse1110.andy.result.QualityResult; diff --git a/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java b/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java index 9c873d18b..7310df28e 100644 --- a/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java +++ b/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java @@ -50,6 +50,8 @@ void setupMocks() throws FileNotFoundException { when(ctx.getDirectoryConfiguration()).thenReturn(dirs); when(asciiArtGenerator.getRandomAsciiArt()).thenReturn("random ascii art"); when(codeSnippetGenerator.generateCodeSnippetFromSolution(any(), anyInt())).thenReturn("arbitrary code snippet"); + ModeActionSelector mas = new ModeActionSelector(Mode.PRACTICE, Action.TESTS); + when(ctx.getModeActionSelector()).thenReturn(mas); } @BeforeEach