Remove overlapping generated tests, closes #535 (#726)

* add comments

* add new pit results to baseline

* fix checkstyle error

* fix code style

* modify selection in CloverCoverageSelector

* fix PitScoreMutantSelectorTest error

* fix test errors in CloverCoverageSelectorTest

* change variable name in CloverCoverageSelector

* make baseline the only improvement criteria

* make comments more precise

* fix checkstyle error

* add unit tests for checking removal of overlapping amplified tests

* add project for testing overlapping test removal

* rename test files

* fix npe in report()

* modify assertion arguments

* fix checkstyle error

* fix checkstyle error

* delete test folder

* add test folder

* modify gitignore

* fix failing test

.gitignore

@@ -29,6 +29,7 @@ dspot/src/test/resources/multiple-pom_1/module-1/module-2-1/.dspot__junit5_pom.x
 dspot/src/test/resources/project-with-resources/.dspot__junit5_pom.xml
 dspot/src/test/resources/regression/test-projects_0/.dspot__junit5_pom.xml
 dspot/src/test/resources/regression/test-projects_1/.dspot__junit5_pom.xml
+dspot/src/test/resources/regression/test-projects_2/.dspot__junit5_pom.xml
 dspot/src/test/resources/sample/.dspot__junit5_pom.xml
 dspot/src/test/resources/test-projects/.dspot__junit5_pom.xml
 dspot-maven/src/test/resources/multi-module/module/.dspot__junit5_pom.xml

dspot/src/main/java/eu/stamp_project/dspot/Amplification.java

@@ -96,14 +96,20 @@ public void amplification(CtType<?> classTest, List<CtMethod<?>> tests, int maxI
         }
         final List<CtMethod<?>> selectedToBeAmplified;
         try {
+
+            // set up the selector with tests to amplify
             selectedToBeAmplified = this.testSelector.selectToAmplify(classTest, passingTests);
         } catch (Exception | java.lang.Error e) {
             Main.GLOBAL_REPORT.addError(new Error(ERROR_PRE_SELECTION, e));
             return;
         }
+
+        // generate tests with additional assertions
         final List<CtMethod<?>> assertionAmplifiedTestMethods = this.assertionsAmplification(classTest, selectedToBeAmplified);
         final List<CtMethod<?>> amplifiedTestMethodsToKeep;
         try {
+
+            // keep tests that improve the test suite
             amplifiedTestMethodsToKeep = this.testSelector.selectToKeep(assertionAmplifiedTestMethods);
         } catch (Exception | java.lang.Error e) {
             Main.GLOBAL_REPORT.addError(new Error(ERROR_SELECTION, e));
@@ -115,6 +121,8 @@ public void amplification(CtType<?> classTest, List<CtMethod<?>> tests, int maxI
         if (this.amplifiers.isEmpty()) {
             return;
         }
+
+        // generate tests with input modification and associated new assertions
         LOGGER.info("Applying Input-amplification and Assertion-amplification test by test.");
         this.resetAmplifiers(classTest);
         for (int i = 0; i < tests.size(); i++) {
@@ -149,6 +157,8 @@ protected List<CtMethod<?>> amplification(CtType<?> classTest, CtMethod test, in
             LOGGER.info("iteration {} / {}", i, maxIteration);
             final List<CtMethod<?>> selectedToBeAmplified;
             try {
+
+                // set up the selector with tests to amplify
                 selectedToBeAmplified = this.testSelector.selectToAmplify(classTest, currentTestList);
             } catch (Exception | java.lang.Error e) {
                 Main.GLOBAL_REPORT.addError(new Error(ERROR_PRE_SELECTION, e));
@@ -164,26 +174,35 @@ protected List<CtMethod<?>> amplification(CtType<?> classTest, CtMethod test, in
             );
             final List<CtMethod<?>> inputAmplifiedTests;
             try {
+
+                // amplify tests and shrink amplified set with budgetizer
                 inputAmplifiedTests = this.budgetizer.inputAmplify(selectedToBeAmplified, i);
             } catch (Exception | java.lang.Error e) {
                 Main.GLOBAL_REPORT.addError(new Error(ERROR_INPUT_AMPLIFICATION, e));
                 return Collections.emptyList();
             }
+
+            // add assertions to input modified tests
             final List<CtMethod<?>> testsWithAssertions = this.assertionsAmplification(classTest, inputAmplifiedTests);
+
             // in case no test with assertions could be generated, we go for the next iteration.
             if (testsWithAssertions.isEmpty()) {
                 currentTestList = inputAmplifiedTests;
                 continue;
             }
             final List<CtMethod<?>> amplifiedTestMethodsToKeep;
             try {
+
+                // keep tests that improve the test suite
                 amplifiedTestMethodsToKeep = this.testSelector.selectToKeep(testsWithAssertions);
             } catch (Exception | java.lang.Error e) {
                 Main.GLOBAL_REPORT.addError(new Error(ERROR_SELECTION, e));
                 return Collections.emptyList();
             }
             amplifiedTests.addAll(amplifiedTestMethodsToKeep);
             LOGGER.info("{} amplified test methods has been selected to be kept.", amplifiedTestMethodsToKeep.size());
+
+            // new amplified tests will be the basis for further amplification
             currentTestList = testsWithAssertions;
         }
         return amplifiedTests;

dspot/src/main/java/eu/stamp_project/dspot/assertgenerator/AssertGenerator.java

@@ -58,10 +58,14 @@ public List<CtMethod<?>> assertionAmplification(CtType<?> testClass, List<CtMeth
         }
         CtType cloneClass = testClass.clone();
         cloneClass.setParent(testClass.getParent());
+
+        // remove existing assertions from cloned test methods
         List<CtMethod<?>> testsWithoutAssertions = tests.stream()
                 .map(this.assertionRemover::removeAssertion)
                 .collect(Collectors.toList());
         testsWithoutAssertions.forEach(cloneClass::addMethod);
+
+        // set up methodsAssertGenerator for use in innerAssertionAmplification
         this.methodsAssertGenerator = new MethodsAssertGenerator(
                 testClass,
                 this.configuration,
@@ -99,6 +103,8 @@ private boolean checkMethodName(String patternMethodName, String methodNameToBeC
      * @return New tests with new assertions
      */
     private List<CtMethod<?>> innerAssertionAmplification(CtType testClass, List<CtMethod<?>> tests) {
+
+        // check if input amplified tests throw new exceptions
         LOGGER.info("Run tests. ({})", tests.size());
         final TestResult testResult;
         try {
@@ -112,16 +118,14 @@ private List<CtMethod<?>> innerAssertionAmplification(CtType testClass, List<CtM
             e.printStackTrace();
             return Collections.emptyList();
         }
-
         final List<String> failuresMethodName = testResult.getFailingTests()
                 .stream()
                 .map(failure -> failure.testCaseName)
                 .collect(Collectors.toList());
-
         final List<String> passingTestsName = testResult.getPassingTests();
-
         final List<CtMethod<?>> generatedTestWithAssertion = new ArrayList<>();
-        // add assertion on passing tests
+
+        // add assertions on passing tests
         if (!passingTestsName.isEmpty()) {
             LOGGER.info("{} test pass, generating assertion...", passingTestsName.size());
             final List<CtMethod<?>> passingTestMethods = tests.stream()
@@ -137,7 +141,7 @@ private List<CtMethod<?>> innerAssertionAmplification(CtType testClass, List<CtM
             generatedTestWithAssertion.addAll(passingTests);
         }
 
-        // add try/catch/fail on failing/error tests
+        // add try/catch block with fail statement in failing tests
         if (!failuresMethodName.isEmpty()) {
             LOGGER.info("{} test fail, generating try/catch/fail blocks...", failuresMethodName.size());
             final List<CtMethod<?>> failingTests = tests.stream()

dspot/src/main/java/eu/stamp_project/dspot/assertgenerator/MethodsAssertGenerator.java

@@ -79,6 +79,8 @@ public List<CtMethod<?>> addAssertions(CtType<?> testClass, List<CtMethod<?>> te
         testClass.getPackage().addType(clone);
         LOGGER.info("Add observations points in passing tests.");
         LOGGER.info("Instrumentation...");
+
+        // add logs in tests to observe state of tested program
         final List<CtMethod<?>> testCasesWithLogs = testCases.stream()
                 .map(ctMethod -> {
                             DSpotUtils.printProgress(testCases.indexOf(ctMethod), testCases.size());
@@ -94,6 +96,8 @@ public List<CtMethod<?>> addAssertions(CtType<?> testClass, List<CtMethod<?>> te
             LOGGER.warn("Could not continue the assertion amplification since all the instrumented test have an empty body.");
             return testCasesWithLogs;
         }
+
+        // clone and set up tests with added logs
         final List<CtMethod<?>> testsToRun = new ArrayList<>();
         IntStream.range(0, 3).forEach(i -> testsToRun.addAll(
                 testCasesWithLogs.stream()
@@ -106,8 +110,9 @@ public List<CtMethod<?>> addAssertions(CtType<?> testClass, List<CtMethod<?>> te
                         .collect(Collectors.toList())
         ));
         ObjectLog.reset();
+
+        // compile and run tests with added logs
         LOGGER.info("Run instrumented tests. ({})", testsToRun.size());
-        //AssertGeneratorHelper.addAfterClassMethod(clone);
         TestFramework.get().generateAfterClassToSaveObservations(clone, testsToRun);
         try {
             final TestResult result = TestCompiler.compileAndRun(
@@ -123,6 +128,8 @@ public List<CtMethod<?>> addAssertions(CtType<?> testClass, List<CtMethod<?>> te
             e.printStackTrace();
             return Collections.emptyList();
         }
+
+        // add assertions with values retrieved from logs in tests
         Map<String, Observation> observations = ObjectLog.getObservations();
         LOGGER.info("Generating assertions...");
         return testCases.stream()

dspot/src/main/java/eu/stamp_project/dspot/selector/JacocoCoverageSelector.java

@@ -43,6 +43,10 @@ public class JacocoCoverageSelector extends TakeAllSelector {
 
     private Coverage initialCoverage;
 
+    private List<String> pathExecuted = new ArrayList<>();
+
+    private TestSelectorElementReport lastReport;
+
     @Override
     public boolean init() {
         super.init();
@@ -123,7 +127,6 @@ public List<CtMethod<?>> selectToKeep(List<CtMethod<?>> amplifiedTestToBeKept) {
             return amplifiedTestToBeKept;
         }
         final CoveragePerTestMethod coveragePerTestMethod = computeCoverageForGivenTestMethods(amplifiedTestToBeKept);
-        final List<String> pathExecuted = new ArrayList<>();
         final List<CtMethod<?>> methodsKept = amplifiedTestToBeKept.stream()
                 .filter(ctMethod -> {
                     final String simpleNameOfFirstParent = getFirstParentThatHasBeenRun(ctMethod).getSimpleName();
@@ -166,6 +169,9 @@ protected CtMethod<?> getFirstParentThatHasBeenRun(CtMethod<?> test) {
 
     @Override
     public TestSelectorElementReport report() {
+        if(currentClassTestToBeAmplified == null) {
+            return lastReport;
+        }
         // 1 textual report
         StringBuilder report = new StringBuilder()
                 .append("Initial instruction coverage: ")
@@ -215,7 +221,8 @@ public TestSelectorElementReport report() {
                             (double) coverageResults.getInstructionsTotal())))
                     .append("%")
                     .append(AmplificationHelper.LINE_SEPARATOR);
-            return new TestSelectorElementReportImpl(report.toString(), jsonReport(coverageResults));
+            lastReport = new TestSelectorElementReportImpl(report.toString(), jsonReport(coverageResults));
+            return lastReport;
         } catch (TimeoutException e) {
             throw new RuntimeException(e);
         } finally {

dspot/src/main/java/eu/stamp_project/dspot/selector/PitMutantScoreSelector.java

@@ -40,6 +40,8 @@ public class PitMutantScoreSelector extends TakeAllSelector {
 
     private List<AbstractPitResult> originalKilledMutants;
 
+    private List<AbstractPitResult> baselineKilledMutants;
+
     private Map<CtMethod, Set<AbstractPitResult>> testThatKilledMutants;
 
     private List<AbstractPitResult> mutantNotTestedByOriginal;
@@ -48,6 +50,8 @@ public class PitMutantScoreSelector extends TakeAllSelector {
 
     public enum OutputFormat {XML, CSV}
 
+    private TestSelectorElementReport lastReport;
+
     public PitMutantScoreSelector() {
         this(OutputFormat.XML);
     }
@@ -96,6 +100,11 @@ public boolean init() {
                 }
             }
             initOriginalPitResult(parser.parseAndDelete(InputConfiguration.get().getAbsolutePathToProjectRoot() + automaticBuilder.getOutputDirectoryPit()));
+        } else {
+            baselineKilledMutants = new ArrayList<>();
+            for(AbstractPitResult r : originalKilledMutants) {
+                baselineKilledMutants.add(r.clone());
+            }
         }
         return true;
     }
@@ -111,6 +120,10 @@ private void initOriginalPitResult(List<AbstractPitResult> results) {
                 .filter(result -> result.getStateOfMutant() == AbstractPitResult.State.KILLED)
                 .collect(Collectors.toList());
         LOGGER.info("The original test suite kill {} / {}", this.originalKilledMutants.size(), results.size());
+        baselineKilledMutants = new ArrayList<>();
+        for(AbstractPitResult r : originalKilledMutants) {
+            baselineKilledMutants.add(r.clone());
+        }
     }
 
     @Override
@@ -127,55 +140,62 @@ public List<CtMethod<?>> selectToKeep(List<CtMethod<?>> amplifiedTestToBeKept) {
         if (amplifiedTestToBeKept.isEmpty()) {
             return amplifiedTestToBeKept;
         }
+
+        // prepare clone of the test class
         CtType clone = this.currentClassTestToBeAmplified.clone();
         clone.setParent(this.currentClassTestToBeAmplified.getParent());
+
+        // remove test methods from clone that are in original test class and add all amplified methods
         this.currentClassTestToBeAmplified.getMethods().stream()
                 .filter(TestFramework.get()::isTest)
                 .forEach(clone::removeMethod);
         amplifiedTestToBeKept.forEach(clone::addMethod);
 
+        // print clone to file and run pit on it
         DSpotUtils.printCtTypeToGivenDirectory(clone, new File(DSpotCompiler.getPathToAmplifiedTestSrc()));
         final AutomaticBuilder automaticBuilder = InputConfiguration.get().getBuilder();
         final String classpath = InputConfiguration.get().getBuilder()
                 .buildClasspath()
                 + AmplificationHelper.PATH_SEPARATOR +
                 InputConfiguration.get().getClasspathClassesProject()
                 + AmplificationHelper.PATH_SEPARATOR + DSpotUtils.getAbsolutePathToDSpotDependencies();
-
         DSpotCompiler.compile(InputConfiguration.get(), DSpotCompiler.getPathToAmplifiedTestSrc(), classpath,
                 new File(InputConfiguration.get().getAbsolutePathToTestClasses()));
 
         InputConfiguration.get().getBuilder().runPit(clone);
         final List<AbstractPitResult> results = parser.parseAndDelete(InputConfiguration.get().getAbsolutePathToProjectRoot() + automaticBuilder.getOutputDirectoryPit());
-
         Set<CtMethod<?>> selectedTests = new HashSet<>();
         if (results != null) {
             LOGGER.info("{} mutants has been generated ({})", results.size(), this.numberOfMutant);
             if (results.size() != this.numberOfMutant) {
                 LOGGER.warn("Number of generated mutant is different than the original one.");
             }
+
+            // keep results where amplified tests kill a mutant not killed (but tested) by original test
             results.stream()
                     .filter(result -> result.getStateOfMutant() == AbstractPitResult.State.KILLED &&
-                            !this.originalKilledMutants.contains(result) &&
+                            !this.baselineKilledMutants.contains(result) &&
                             !this.mutantNotTestedByOriginal.contains(result))
                     .forEach(result -> {
                         CtMethod method = result.getMethod(clone);
-                        if (killsMoreMutantThanParents(method, result)) {
+
+                        // keep methods that kill mutants not killed before
+                        if (killsNewMutant(result)) {
                             if (!testThatKilledMutants.containsKey(method)) {
                                 testThatKilledMutants.put(method, new HashSet<>());
                             }
                             testThatKilledMutants.get(method).add(result);
                             if (method == null) {
-                                selectedTests.addAll(amplifiedTestToBeKept);// output of pit test does not allow us to know which test case kill new mutants... we keep them all...
+
+                                // output of pit test does not allow us to know which test case kill new mutants... we keep them all...
+                                selectedTests.addAll(amplifiedTestToBeKept);
                             } else {
                                 selectedTests.add(method);
                             }
                         }
                     });
         }
-
         this.selectedAmplifiedTest.addAll(selectedTests);
-
         selectedTests.forEach(selectedTest ->
                 LOGGER.info("{} kills {} more mutants",
                         selectedTest == null ?
@@ -186,27 +206,29 @@ public List<CtMethod<?>> selectToKeep(List<CtMethod<?>> amplifiedTestToBeKept) {
         return new ArrayList<>(selectedTests);
     }
 
-    private boolean killsMoreMutantThanParents(CtMethod test, AbstractPitResult result) {
-        CtMethod parent = AmplificationHelper.getAmpTestParent(test);
-        while (parent != null) {
-            if (this.testThatKilledMutants.get(parent) != null &&
-                    this.testThatKilledMutants.get(parent).contains(result)) {
-                return false;
-            }
-            parent = AmplificationHelper.getAmpTestParent(parent);
+    private boolean killsNewMutant(AbstractPitResult result) {
+        if (baselineKilledMutants.contains(result)) {
+            return false;
         }
+
+        // add result to baseline to prohibit selection of identical amplified tests
+        baselineKilledMutants.add(result);
         return true;
     }
 
     @Override
     public TestSelectorElementReport report() {
+        if(currentClassTestToBeAmplified == null) {
+            return lastReport;
+        }
         final String reportStdout = reportStdout();
         final TestClassJSON testClassJSON = reportJSONMutants();
         //clean up for the next class
         this.currentClassTestToBeAmplified = null;
         this.testThatKilledMutants.clear();
         this.selectedAmplifiedTest.clear();
-        return new TestSelectorElementReportImpl(reportStdout, testClassJSON);
+        lastReport = new TestSelectorElementReportImpl(reportStdout, testClassJSON);
+        return lastReport;
     }
 
     private String reportStdout() {

dspot/src/main/java/eu/stamp_project/utils/pit/AbstractPitResult.java

@@ -5,7 +5,7 @@
 
 import java.util.List;
 
-public class AbstractPitResult {
+public abstract class AbstractPitResult {
     public enum State {SURVIVED, KILLED, NO_COVERAGE, TIMED_OUT, NON_VIABLE, MEMORY_ERROR}
 
     protected final String fullQualifiedNameOfMutatedClass;
@@ -77,4 +77,6 @@ public CtMethod getMethod(CtType<?> ctClass) {
             return this.testCase;
         }
     }
+
+    public abstract AbstractPitResult clone();
 }

dspot/src/main/java/eu/stamp_project/utils/pit/PitCSVResult.java

@@ -55,4 +55,10 @@ public String toString() {
                 ", testCase=" + testCase +
                 '}';
     }
+
+    @Override
+    public PitCSVResult clone() {
+        return new PitCSVResult(fullQualifiedNameOfMutatedClass, stateOfMutant, fullQualifiedNameMutantOperator,
+                simpleNameMethod, fullQualifiedNameOfKiller, lineNumber, nameOfMutatedMethod);
+    }
 }